1//===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements hazard recognizers for scheduling on GCN processors.
10//
11//===----------------------------------------------------------------------===//
12
13#include "GCNHazardRecognizer.h"
14#include "AMDGPUTargetMachine.h"
15#include "AMDGPUWaitcntUtils.h"
16#include "GCNSubtarget.h"
17#include "SIMachineFunctionInfo.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/CodeGen/MachineFrameInfo.h"
20#include "llvm/CodeGen/MachineFunction.h"
21#include "llvm/CodeGen/MachineInstrBuilder.h"
22#include "llvm/CodeGen/ScheduleDAG.h"
23#include "llvm/Support/Debug.h"
24#include "llvm/TargetParser/AMDGPUTargetParser.h"
25
26using namespace llvm;
27
28#define DEBUG_TYPE "gcn-hazard-recognizer"
29// Opt-in debug type for the per-candidate co-execution slot traces, which are
30// far too noisy for the normal debug output. Pass both types to get everything.
31#define DEBUG_TYPE_VERBOSE "gcn-hazard-recognizer-verbose"
32
33STATISTIC(NumWMMANopsHoisted,
34 "Number of WMMA hazard V_NOPs hoisted from loops");
35STATISTIC(NumWMMAHoistingBailed,
36 "Number of WMMA hazards where V_NOP hoisting was not possible");
37
38namespace {
39
40struct MFMAPaddingRatioParser : public cl::parser<unsigned> {
41 MFMAPaddingRatioParser(cl::Option &O) : cl::parser<unsigned>(O) {}
42
43 bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) {
44 if (Arg.getAsInteger(Radix: 0, Result&: Value))
45 return O.error(Message: "'" + Arg + "' value invalid for uint argument!");
46
47 if (Value > 100)
48 return O.error(Message: "'" + Arg + "' value must be in the range [0, 100]!");
49
50 return false;
51 }
52};
53
54} // end anonymous namespace
55
56static cl::opt<unsigned, false, MFMAPaddingRatioParser>
57 MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(Val: 0), cl::Hidden,
58 cl::desc("Fill a percentage of the latency between "
59 "neighboring MFMA with s_nops."));
60
61// This is intended for debugging purposes only.
62static cl::opt<unsigned>
63 NopPadding("amdgpu-snop-padding", cl::init(Val: 0), cl::Hidden,
64 cl::desc("Insert a s_nop x before every instruction"));
65
66static cl::opt<bool> EnableWMMAVnopHoisting(
67 "amdgpu-wmma-vnop-hoisting", cl::init(Val: true), cl::Hidden,
68 cl::desc("Hoist WMMA hazard V_NOPs from loops to preheaders"));
69
70//===----------------------------------------------------------------------===//
71// Hazard Recognizer Implementation
72//===----------------------------------------------------------------------===//
73
74static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF,
75 const GCNSubtarget &ST);
76
77GCNHazardRecognizer::GCNHazardRecognizer(
78 const MachineFunction &MF, GCNHazardRecognizer::OperatingMode Mode,
79 MachineLoopInfo *MLI)
80 : Mode(Mode), CurrCycleInstr(nullptr), MF(MF),
81 ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()),
82 TRI(TII.getRegisterInfo()), TSchedModel(TII.getSchedModel()), MLI(MLI),
83 ClauseUses(TRI.getNumRegUnits()), ClauseDefs(TRI.getNumRegUnits()) {
84 MaxLookAhead = MF.getRegInfo().isPhysRegUsed(PhysReg: AMDGPU::AGPR0) ? 19 : 5;
85 RunLdsBranchVmemWARHazardFixup = shouldRunLdsBranchVmemWARHazardFixup(MF, ST);
86 LLVM_DEBUG({
87 if (isPreRA())
88 dbgs() << " PreRA hazard recognizer: " << MF.getName() << "\n";
89 });
90}
91
92GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF,
93 MachineLoopInfo *MLI)
94 : GCNHazardRecognizer(MF, OperatingMode::PostRA, MLI) {}
95
96GCNHazardRecognizer::~GCNHazardRecognizer() {
97 // Dump any active co-execution window that did not complete naturally
98 // (e.g. region ended before the window expired).
99 LLVM_DEBUG({
100 if (CurrentCoExecStage.has_value()) {
101 unsigned Stage = *CurrentCoExecStage;
102 if (Stage < AMDGPU::MaxCoExecStages)
103 CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
104 dbgs() << " CoExec window ended at stage " << Stage << ":\n";
105 dumpCoExecWindow();
106 }
107 });
108}
109
110void GCNHazardRecognizer::Reset() {
111 EmittedInstrs.clear();
112 EmittedVALUInstrs.clear();
113 HasPendingWMMACoexecHazard = false;
114 if (isSchedulerMode())
115 schedulerReset();
116}
117
118void GCNHazardRecognizer::schedulerReset() {
119 LLVM_DEBUG({
120 if (CurrentCoExecStage.has_value() || CyclesUntilTRANS > 0 ||
121 CyclesUntilVALU > 0)
122 dbgs() << " Scheduler Reset: clearing co-exec window, TRANS="
123 << CyclesUntilTRANS << ", VALU=" << CyclesUntilVALU << "\n";
124 });
125 CurrentCoExecStage = std::nullopt;
126 CoExecWindowStartCycle = 0;
127 CyclesUntilTRANS = 0;
128 CyclesUntilVALU = 0;
129 ActiveCoExecInfo = AMDGPU::CoExecInfo();
130 CoExecWindowLog.fill(u: '.');
131}
132
133void GCNHazardRecognizer::dumpCoExecWindow() const {
134 unsigned W = ActiveCoExecInfo.TotalWindow;
135 if (W == 0)
136 return;
137
138 // Print the stage numbers row.
139 dbgs() << " Stages: ";
140 for (unsigned I = 0; I < W; ++I)
141 dbgs() << I % 10 << ' ';
142 dbgs() << '\n';
143
144 // Print the pattern row.
145 dbgs() << " Slots: ";
146 for (unsigned I = 0; I < W; ++I)
147 dbgs() << ActiveCoExecInfo.Pattern[I] << ' ';
148 dbgs() << '\n';
149
150 // Print the scheduled row.
151 dbgs() << " Scheduled: ";
152 for (unsigned I = 0; I < W; ++I)
153 dbgs() << CoExecWindowLog[I] << ' ';
154 dbgs() << '\n';
155}
156
157void GCNHazardRecognizer::schedulerAdvanceCycle() {
158 // Record what happened at the current stage of the co-exec window.
159 if (CurrentCoExecStage.has_value()) {
160 unsigned Stage = *CurrentCoExecStage;
161 if (Stage < AMDGPU::MaxCoExecStages) {
162 if (CurrCycleInstr)
163 CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
164 else
165 CoExecWindowLog[Stage] = '-';
166 }
167 }
168
169 LLVM_DEBUG({
170 bool HasState = CurrentCoExecStage.has_value() || CyclesUntilTRANS > 0 ||
171 CyclesUntilVALU > 0;
172 if (HasState) {
173 dbgs() << " Scheduler AdvanceCycle:";
174 if (CurrentCoExecStage.has_value()) {
175 unsigned Stage = *CurrentCoExecStage;
176 unsigned Next = Stage + 1;
177 if (Next >= ActiveCoExecInfo.TotalWindow)
178 dbgs() << " stage " << Stage << "->expired";
179 else
180 dbgs() << " stage " << Stage << "->" << Next;
181 }
182 if (CyclesUntilTRANS > 0)
183 dbgs() << " TRANS=" << CyclesUntilTRANS << "->"
184 << (CyclesUntilTRANS - 1);
185 if (CyclesUntilVALU > 0)
186 dbgs() << " VALU=" << CyclesUntilVALU << "->" << (CyclesUntilVALU - 1);
187 dbgs() << "\n";
188 }
189 });
190
191 // Decrement hazard counters.
192 if (CyclesUntilTRANS > 0)
193 --CyclesUntilTRANS;
194 if (CyclesUntilVALU > 0)
195 --CyclesUntilVALU;
196
197 // Advance WMMA co-execution window.
198 if (CurrentCoExecStage.has_value()) {
199 unsigned Stage = *CurrentCoExecStage + 1;
200 if (Stage >= ActiveCoExecInfo.TotalWindow) {
201 // Window expired.
202 LLVM_DEBUG({
203 dbgs() << " CoExec window complete:\n";
204 dumpCoExecWindow();
205 });
206 CurrentCoExecStage = std::nullopt;
207 } else {
208 CurrentCoExecStage = Stage;
209 }
210 }
211}
212
213bool GCNHazardRecognizer::hasCoExecWindowModel() const {
214 // The co-execution slot patterns returned by getCoExecInfo() are derived from
215 // gfx1250 timings, so the window model is restricted to gfx1250 for now.
216 // gfx1251 and gfx12.5-generic report the same co-execution hazard features
217 // but have different WMMA latencies, so they need their own slot patterns
218 // before they can be modeled here.
219 if (ST.hasWMMACoexecutionHazards() && ST.hasTransCoexecutionHazard() &&
220 AMDGPU::isGFX1250(STI: ST))
221 return true;
222
223 if (ST.hasGFX950Insts() &&
224 AMDGPU::getSchedStrategy(F: MF.getFunction()) == "coexec")
225 return true;
226
227 return false;
228}
229
230void GCNHazardRecognizer::updateWMMAWindowState(const MachineInstr &MI) {
231 if (!hasCoExecWindowModel())
232 return;
233
234 if (!SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI) &&
235 !SIInstrInfo::isMFMA(MI))
236 return;
237
238 // If a previous window was still active, dump it before starting a new one.
239 // Record the current stage (filled by this new WMMA) before dumping.
240 LLVM_DEBUG({
241 if (CurrentCoExecStage.has_value()) {
242 unsigned Stage = *CurrentCoExecStage;
243 if (Stage < AMDGPU::MaxCoExecStages)
244 CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
245 dbgs() << " CoExec window interrupted at stage " << Stage << ":\n";
246 dumpCoExecWindow();
247 }
248 });
249
250 // Start a new co-execution window.
251 ActiveCoExecInfo = AMDGPU::getCoExecInfo(MI, TII);
252 CurrentCoExecStage = 0;
253 CoExecWindowLog.fill(u: '.');
254
255 LLVM_DEBUG(dbgs() << " WMMA window started: " << ActiveCoExecInfo.Pattern
256 << " (window=" << ActiveCoExecInfo.TotalWindow << ")\n"
257 << " " << MI);
258}
259
260void GCNHazardRecognizer::updateTRANSState(const MachineInstr &MI) {
261 if (!hasCoExecWindowModel())
262 return;
263 if (!SIInstrInfo::isTRANS(MI))
264 return;
265
266 // Back-to-back TRANS instructions have a 1-cycle hazard.
267 // This is checked via checkTRANSHazard() and does not create a co-exec
268 // window. The TRANS shadow slot allows anything except TRANS and
269 // multi-cycle VALU.
270 // Set to 2: bumpCycle advances to the next pick's cycle (decrementing
271 // by 1 via AdvanceCycle) before the next instruction's hazard check, so
272 // the counter is observed at 1 there. That 1-cycle stall lets the
273 // strategy pick a non-TRANS, non-multi-cycle-VALU candidate to fill the
274 // shadow slot.
275 CyclesUntilTRANS = 2;
276 LLVM_DEBUG(dbgs() << " TRANS hazard set: CyclesUntilTRANS=2\n");
277}
278
279void GCNHazardRecognizer::updateMultiCycleVALUState(const MachineInstr &MI) {
280 if (!hasCoExecWindowModel())
281 return;
282 // Multi-cycle VALU (CVT, etc.) blocks subsequent VALU for repeat rate cycles.
283 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
284 return;
285
286 // Skip WMMA, MFMA, and TRANS - they have their own tracking.
287 if (SIInstrInfo::isWMMA(MI) || SIInstrInfo::isSWMMAC(MI) ||
288 SIInstrInfo::isMFMA(MI) || SIInstrInfo::isTRANS(MI))
289 return;
290
291 unsigned RepeatRate = TII.getRepeatRate(MI);
292 if (RepeatRate > 1) {
293 // bumpCycle's AdvanceCycle decrements once before the next pick's
294 // hazard check (same convention as CyclesUntilTRANS), so to expose
295 // RepeatRate-1 cycles of shadow we must seed with RepeatRate.
296 CyclesUntilVALU = RepeatRate;
297 LLVM_DEBUG(dbgs() << " Multi-cycle VALU: repeat=" << RepeatRate
298 << ", CyclesUntilVALU=" << CyclesUntilVALU << "\n");
299 }
300}
301
302AMDGPU::CoExecMaskT
303GCNHazardRecognizer::getCoExecMaskForMI(const MachineInstr &MI,
304 const SIInstrInfo &TII) {
305 return AMDGPU::getCoExecMask(F: AMDGPU::classifyFlavor(MI, SII: TII));
306}
307
308unsigned GCNHazardRecognizer::checkTRANSHazard(const MachineInstr &MI) const {
309 if (!CyclesUntilTRANS)
310 return 0;
311
312 // Only TRANS and multi-cycle VALU are blocked by the TRANS shadow.
313 if (SIInstrInfo::isTRANS(MI))
314 return CyclesUntilTRANS;
315
316 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
317 !SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI) &&
318 TII.getRepeatRate(MI) > 1)
319 return CyclesUntilTRANS;
320
321 return 0;
322}
323
324unsigned
325GCNHazardRecognizer::checkMultiCycleVALUHazard(const MachineInstr &MI) const {
326 if (!CyclesUntilVALU)
327 return 0;
328
329 // Multi-cycle VALU blocks anything on the VALU pipe - VALU, WMMA, SWMMAC,
330 // and TRANS - for RepeatRate-1 cycles. Only off-pipe instructions (MEM,
331 // SALU, control) can fill the shadow.
332 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
333 !SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI) &&
334 !SIInstrInfo::isTRANS(MI))
335 return 0;
336
337 return CyclesUntilVALU;
338}
339
340unsigned
341GCNHazardRecognizer::checkWMMACoexecSlot(const MachineInstr &MI) const {
342 // No hazard if not in a WMMA window.
343 if (!CurrentCoExecStage.has_value())
344 return 0;
345
346 unsigned Stage = *CurrentCoExecStage;
347 AMDGPU::CoExecMaskT InstMask = getCoExecMaskForMI(MI, TII);
348 unsigned StallCycles = ActiveCoExecInfo.getStallCycles(InstMask, Stage);
349
350 // No stall required if the instruction can co-execute at the current stage.
351 if (StallCycles == 0)
352 return 0;
353
354 // Stall for the required number of cycles until the next allowed stage.
355 unsigned NextStage = Stage + StallCycles;
356 if (NextStage < ActiveCoExecInfo.TotalWindow) {
357 DEBUG_WITH_TYPE(
358 DEBUG_TYPE_VERBOSE,
359 dbgs() << " CoExec stall: stage=" << Stage << "("
360 << AMDGPU::getStageTypeName(ActiveCoExecInfo.getType(Stage))
361 << ") mask=" << AMDGPU::getCoExecMaskName(InstMask)
362 << " -> stall " << StallCycles << " (next allowed=" << NextStage
363 << ")\n"
364 << " " << MI);
365 return StallCycles;
366 }
367
368 // No compatible slot in window - stall until window ends.
369 DEBUG_WITH_TYPE(
370 DEBUG_TYPE_VERBOSE,
371 dbgs() << " CoExec stall: stage=" << Stage << "("
372 << AMDGPU::getStageTypeName(ActiveCoExecInfo.getType(Stage))
373 << ") mask=" << AMDGPU::getCoExecMaskName(InstMask) << " -> stall "
374 << StallCycles << " (window ends)\n"
375 << " " << MI);
376 return StallCycles;
377}
378
379unsigned
380GCNHazardRecognizer::checkMultiShadowHazard(const MachineInstr &MI) const {
381 // This models a VALU caught in both a WMMA and a TRANS shadow.
382 if (!hasCoExecWindowModel())
383 return 0;
384
385 // No hazard if not in a WMMA window.
386 if (!CurrentCoExecStage.has_value())
387 return 0;
388
389 if (!CyclesUntilTRANS)
390 return 0;
391
392 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ||
393 SIInstrInfo::isLDSDMA(MI))
394 return 0;
395
396 // We have a VALU instruction that is under both a TRANS and WMMA shadow.
397 // We need to wait for at least one to clear.
398
399 unsigned LookAheadStage = *CurrentCoExecStage + CyclesUntilTRANS;
400 AMDGPU::CoExecMaskT InstMask = getCoExecMaskForMI(MI, TII);
401 return CyclesUntilTRANS +
402 ActiveCoExecInfo.getStallCycles(InstMask, Stage: LookAheadStage);
403}
404
405void GCNHazardRecognizer::schedulerEmitInstruction(MachineInstr *MI) {
406 LLVM_DEBUG({
407 bool InWindow = CurrentCoExecStage.has_value();
408 bool HasActiveState =
409 InWindow || CyclesUntilTRANS > 0 || CyclesUntilVALU > 0;
410 if (HasActiveState) {
411 if (InWindow) {
412 unsigned Stage = *CurrentCoExecStage;
413 dbgs() << " Stage " << Stage << "("
414 << AMDGPU::getStageTypeName(ActiveCoExecInfo.getType(Stage))
415 << ") Emit ["
416 << AMDGPU::getCoExecMaskName(getCoExecMaskForMI(*MI, TII))
417 << "]: " << *MI;
418 } else {
419 dbgs() << " Emit ["
420 << AMDGPU::getCoExecMaskName(getCoExecMaskForMI(*MI, TII))
421 << "]: " << *MI;
422 }
423 }
424 });
425 DEBUG_WITH_TYPE(DEBUG_TYPE_VERBOSE, {
426 bool HasActiveState = CurrentCoExecStage.has_value() ||
427 CyclesUntilTRANS > 0 || CyclesUntilVALU > 0;
428 if (!HasActiveState)
429 dbgs() << " Emit ["
430 << AMDGPU::getCoExecMaskName(getCoExecMaskForMI(*MI, TII))
431 << "]: " << *MI;
432 });
433 updateWMMAWindowState(MI: *MI);
434 updateTRANSState(MI: *MI);
435 updateMultiCycleVALUState(MI: *MI);
436}
437
438void GCNHazardRecognizer::EmitInstruction(SUnit *SU) {
439 EmitInstruction(MI: SU->getInstr());
440}
441
442void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) {
443 CurrCycleInstr = MI;
444 if (isSchedulerMode())
445 schedulerEmitInstruction(MI);
446}
447
448static bool isDivFMas(unsigned Opcode) {
449 return Opcode == AMDGPU::V_DIV_FMAS_F32_e64 || Opcode == AMDGPU::V_DIV_FMAS_F64_e64;
450}
451
452static bool isSGetReg(unsigned Opcode) {
453 return Opcode == AMDGPU::S_GETREG_B32 || Opcode == AMDGPU::S_GETREG_B32_const;
454}
455
456static bool isSSetReg(unsigned Opcode) {
457 switch (Opcode) {
458 case AMDGPU::S_SETREG_B32:
459 case AMDGPU::S_SETREG_B32_mode:
460 case AMDGPU::S_SETREG_IMM32_B32:
461 case AMDGPU::S_SETREG_IMM32_B32_mode:
462 return true;
463 }
464 return false;
465}
466
467static bool isRWLane(unsigned Opcode) {
468 return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32;
469}
470
471static bool isRFE(unsigned Opcode) {
472 return Opcode == AMDGPU::S_RFE_B64;
473}
474
475static bool isSMovRel(unsigned Opcode) {
476 switch (Opcode) {
477 case AMDGPU::S_MOVRELS_B32:
478 case AMDGPU::S_MOVRELS_B64:
479 case AMDGPU::S_MOVRELD_B32:
480 case AMDGPU::S_MOVRELD_B64:
481 return true;
482 default:
483 return false;
484 }
485}
486
487static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII,
488 const MachineInstr &MI) {
489 if (TII.isAlwaysGDS(Opcode: MI.getOpcode()))
490 return true;
491
492 switch (MI.getOpcode()) {
493 case AMDGPU::S_SENDMSG:
494 case AMDGPU::S_SENDMSGHALT:
495 case AMDGPU::S_TTRACEDATA:
496 return true;
497 // These DS opcodes don't support GDS.
498 case AMDGPU::DS_NOP:
499 case AMDGPU::DS_PERMUTE_B32:
500 case AMDGPU::DS_BPERMUTE_B32:
501 return false;
502 default:
503 if (TII.isDS(Opcode: MI.getOpcode())) {
504 int GDS = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
505 Name: AMDGPU::OpName::gds);
506 if (MI.getOperand(i: GDS).getImm())
507 return true;
508 }
509 return false;
510 }
511}
512
513static bool isPermlane(const MachineInstr &MI) {
514 unsigned Opcode = MI.getOpcode();
515 return Opcode == AMDGPU::V_PERMLANE16_B32_e64 ||
516 Opcode == AMDGPU::V_PERMLANE64_B32 ||
517 Opcode == AMDGPU::V_PERMLANEX16_B32_e64 ||
518 Opcode == AMDGPU::V_PERMLANE16_VAR_B32_e64 ||
519 Opcode == AMDGPU::V_PERMLANEX16_VAR_B32_e64 ||
520 Opcode == AMDGPU::V_PERMLANE16_SWAP_B32_e32 ||
521 Opcode == AMDGPU::V_PERMLANE16_SWAP_B32_e64 ||
522 Opcode == AMDGPU::V_PERMLANE32_SWAP_B32_e32 ||
523 Opcode == AMDGPU::V_PERMLANE32_SWAP_B32_e64 ||
524 Opcode == AMDGPU::V_PERMLANE_BCAST_B32_e64 ||
525 Opcode == AMDGPU::V_PERMLANE_UP_B32_e64 ||
526 Opcode == AMDGPU::V_PERMLANE_DOWN_B32_e64 ||
527 Opcode == AMDGPU::V_PERMLANE_XOR_B32_e64 ||
528 Opcode == AMDGPU::V_PERMLANE_IDX_GEN_B32_e64;
529}
530
531static bool isLdsDma(const MachineInstr &MI) {
532 return SIInstrInfo::isLDSDMA(MI);
533}
534
535static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) {
536 const MachineOperand *RegOp = TII->getNamedOperand(MI: RegInstr,
537 OperandName: AMDGPU::OpName::simm16);
538 return std::get<0>(t: AMDGPU::Hwreg::HwregEncoding::decode(Encoded: RegOp->getImm()));
539}
540
541ScheduleHazardRecognizer::HazardType
542GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
543 MachineInstr *MI = SU->getInstr();
544 // If we are not in "HazardRecognizerMode" and therefore not being run from
545 // the scheduler, track possible stalls from hazards but don't insert noops.
546 auto HazardType = isHazardRecognizerMode() ? NoopHazard : Hazard;
547
548 if (MI->isBundle())
549 return NoHazard;
550
551 // Check co-execution slot hazards and pipeline stalls in scheduler modes.
552 if (isSchedulerMode()) {
553 if (checkMultiShadowHazard(MI: *MI) > 0)
554 return Hazard;
555 if (checkWMMACoexecSlot(MI: *MI) > 0)
556 return Hazard;
557 if (checkTRANSHazard(MI: *MI) > 0)
558 return Hazard;
559 if (checkMultiCycleVALUHazard(MI: *MI) > 0)
560 return Hazard;
561 // The remaining checks are all defined by register dependences.
562 if (!hasPhysRegs())
563 return NoHazard;
564 }
565
566 if (SIInstrInfo::isSMRD(MI: *MI) && checkSMRDHazards(SMRD: MI) > 0)
567 return HazardType;
568
569 if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0)
570 return HazardType;
571
572 if (checkFPAtomicToDenormModeHazard(MI) > 0)
573 return HazardType;
574
575 // Hazards which cannot be mitigated with S_NOPs.
576 if (!isHazardRecognizerMode()) {
577 if (checkWMMACoexecutionHazards(MI) > 0) {
578 HasPendingWMMACoexecHazard = true;
579 return Hazard;
580 }
581 }
582
583 if (ST.hasNoDataDepHazard())
584 return NoHazard;
585
586 if (SIInstrInfo::isVMEM(MI: *MI) && checkVMEMHazards(VMEM: MI) > 0)
587 return HazardType;
588
589 if (SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true) &&
590 checkVALUHazards(VALU: MI) > 0)
591 return HazardType;
592
593 if (SIInstrInfo::isDPP(MI: *MI) && checkDPPHazards(DPP: MI) > 0)
594 return HazardType;
595
596 if (isDivFMas(Opcode: MI->getOpcode()) && checkDivFMasHazards(DivFMas: MI) > 0)
597 return HazardType;
598
599 if (isRWLane(Opcode: MI->getOpcode()) && checkRWLaneHazards(RWLane: MI) > 0)
600 return HazardType;
601
602 if ((SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true) ||
603 SIInstrInfo::isVMEM(MI: *MI) || SIInstrInfo::isDS(MI: *MI) ||
604 SIInstrInfo::isEXP(MI: *MI)) &&
605 checkMAIVALUHazards(MI) > 0)
606 return HazardType;
607
608 if (isSGetReg(Opcode: MI->getOpcode()) && checkGetRegHazards(GetRegInstr: MI) > 0)
609 return HazardType;
610
611 if (isSSetReg(Opcode: MI->getOpcode()) && checkSetRegHazards(SetRegInstr: MI) > 0)
612 return HazardType;
613
614 if (isRFE(Opcode: MI->getOpcode()) && checkRFEHazards(RFE: MI) > 0)
615 return HazardType;
616
617 if (((ST.hasReadM0MovRelInterpHazard() &&
618 (TII.isVINTRP(MI: *MI) || isSMovRel(Opcode: MI->getOpcode()) ||
619 MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 ||
620 MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) ||
621 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, MI: *MI)) ||
622 (ST.hasReadM0LdsDmaHazard() && isLdsDma(MI: *MI)) ||
623 (ST.hasReadM0LdsDirectHazard() &&
624 MI->readsRegister(Reg: AMDGPU::LDS_DIRECT, /*TRI=*/nullptr))) &&
625 checkReadM0Hazards(SMovRel: MI) > 0)
626 return HazardType;
627
628 if (SIInstrInfo::isMAI(MI: *MI) && checkMAIHazards(MI) > 0)
629 return HazardType;
630
631 if ((SIInstrInfo::isVMEM(MI: *MI) || SIInstrInfo::isDS(MI: *MI)) &&
632 checkMAILdStHazards(MI) > 0)
633 return HazardType;
634
635 if (ST.hasPermlaneForwardingHazard() && isPermlane(MI: *MI) &&
636 checkPermlaneHazards(MI) > 0)
637 return HazardType;
638
639 if (MI->isInlineAsm() && checkInlineAsmHazards(IA: MI) > 0)
640 return HazardType;
641
642 return NoHazard;
643}
644
645static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII,
646 unsigned Quantity) {
647 while (Quantity > 0) {
648 unsigned Arg = std::min(a: Quantity, b: 8u);
649 Quantity -= Arg;
650 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_NOP))
651 .addImm(Val: Arg - 1);
652 }
653}
654
655unsigned
656GCNHazardRecognizer::getMFMAPipelineWaitStates(const MachineInstr &MI) const {
657 const MCSchedClassDesc *SC = TSchedModel.resolveSchedClass(MI: &MI);
658 assert(TSchedModel.getWriteProcResBegin(SC) !=
659 TSchedModel.getWriteProcResEnd(SC));
660 return TSchedModel.getWriteProcResBegin(SC)->ReleaseAtCycle;
661}
662
663void GCNHazardRecognizer::processBundle() {
664 MachineBasicBlock::instr_iterator MI = std::next(x: CurrCycleInstr->getIterator());
665 MachineBasicBlock::instr_iterator E = CurrCycleInstr->getParent()->instr_end();
666 // Check bundled MachineInstr's for hazards.
667 for (; MI != E && MI->isInsideBundle(); ++MI) {
668 CurrCycleInstr = &*MI;
669 unsigned WaitStates = PreEmitNoopsCommon(CurrCycleInstr);
670
671 if (isHazardRecognizerMode()) {
672 fixHazards(MI: CurrCycleInstr);
673
674 insertNoopsInBundle(MI: CurrCycleInstr, TII, Quantity: WaitStates);
675 }
676
677 // It’s unnecessary to track more than MaxLookAhead instructions. Since we
678 // include the bundled MI directly after, only add a maximum of
679 // (MaxLookAhead - 1) noops to EmittedInstrs.
680 for (unsigned i = 0, e = std::min(a: WaitStates, b: MaxLookAhead - 1); i < e; ++i)
681 EmittedInstrs.push_front(x: nullptr);
682
683 EmittedInstrs.push_front(x: CurrCycleInstr);
684 EmittedInstrs.resize(new_size: MaxLookAhead);
685 }
686 CurrCycleInstr = nullptr;
687}
688
689void GCNHazardRecognizer::runOnInstruction(MachineInstr *MI) {
690 assert(isHazardRecognizerMode());
691
692 unsigned NumPreNoops = PreEmitNoops(MI);
693 EmitNoops(Quantity: NumPreNoops);
694 if (MI->isInsideBundle())
695 insertNoopsInBundle(MI, TII, Quantity: NumPreNoops);
696 else
697 TII.insertNoops(MBB&: *MI->getParent(), MI: MachineBasicBlock::iterator(MI),
698 Quantity: NumPreNoops);
699 EmitInstruction(MI);
700 AdvanceCycle();
701}
702
703unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
704 assert(isHazardRecognizerMode());
705 CurrCycleInstr = MI;
706 unsigned W = PreEmitNoopsCommon(MI);
707 fixHazards(MI);
708 CurrCycleInstr = nullptr;
709 return std::max(a: W, b: NopPadding.getValue());
710}
711
712unsigned GCNHazardRecognizer::getHazardWaitStates(MachineInstr *MI) const {
713 unsigned W = 0;
714
715 // Check co-execution slot hazards and pipeline stalls in scheduler modes.
716 if (isSchedulerMode()) {
717 W = checkWMMACoexecSlot(MI: *MI);
718 W = std::max(a: W, b: checkTRANSHazard(MI: *MI));
719 W = std::max(a: W, b: checkMultiCycleVALUHazard(MI: *MI));
720 W = std::max(a: W, b: checkMultiShadowHazard(MI: *MI));
721 // The remaining checks are all defined by register dependences.
722 if (!hasPhysRegs())
723 return W;
724 }
725
726 return std::max(a: W, b: PreEmitNoopsCommon(MI));
727}
728
729unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) const {
730 if (MI->isBundle())
731 return 0;
732
733 int WaitStates = 0;
734
735 if (SIInstrInfo::isSMRD(MI: *MI))
736 return std::max(a: WaitStates, b: checkSMRDHazards(SMRD: MI));
737
738 if (ST.hasNSAtoVMEMBug())
739 WaitStates = std::max(a: WaitStates, b: checkNSAtoVMEMHazard(MI));
740
741 WaitStates = std::max(a: WaitStates, b: checkFPAtomicToDenormModeHazard(MI));
742
743 if (ST.hasNoDataDepHazard())
744 return WaitStates;
745
746 if (SIInstrInfo::isVMEM(MI: *MI))
747 WaitStates = std::max(a: WaitStates, b: checkVMEMHazards(VMEM: MI));
748
749 if (SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true))
750 WaitStates = std::max(a: WaitStates, b: checkVALUHazards(VALU: MI));
751
752 if (SIInstrInfo::isDPP(MI: *MI))
753 WaitStates = std::max(a: WaitStates, b: checkDPPHazards(DPP: MI));
754
755 if (isDivFMas(Opcode: MI->getOpcode()))
756 WaitStates = std::max(a: WaitStates, b: checkDivFMasHazards(DivFMas: MI));
757
758 if (isRWLane(Opcode: MI->getOpcode()))
759 WaitStates = std::max(a: WaitStates, b: checkRWLaneHazards(RWLane: MI));
760
761 if (SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true) ||
762 SIInstrInfo::isVMEM(MI: *MI) || SIInstrInfo::isDS(MI: *MI) ||
763 SIInstrInfo::isEXP(MI: *MI))
764 WaitStates = std::max(a: WaitStates, b: checkMAIVALUHazards(MI));
765
766 if (MI->isInlineAsm())
767 return std::max(a: WaitStates, b: checkInlineAsmHazards(IA: MI));
768
769 if (isSGetReg(Opcode: MI->getOpcode()))
770 return std::max(a: WaitStates, b: checkGetRegHazards(GetRegInstr: MI));
771
772 if (isSSetReg(Opcode: MI->getOpcode()))
773 return std::max(a: WaitStates, b: checkSetRegHazards(SetRegInstr: MI));
774
775 if (isRFE(Opcode: MI->getOpcode()))
776 return std::max(a: WaitStates, b: checkRFEHazards(RFE: MI));
777
778 if ((ST.hasReadM0MovRelInterpHazard() &&
779 (TII.isVINTRP(MI: *MI) || isSMovRel(Opcode: MI->getOpcode()) ||
780 MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 ||
781 MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) ||
782 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, MI: *MI)) ||
783 (ST.hasReadM0LdsDmaHazard() && isLdsDma(MI: *MI)) ||
784 (ST.hasReadM0LdsDirectHazard() &&
785 MI->readsRegister(Reg: AMDGPU::LDS_DIRECT, /*TRI=*/nullptr)))
786 return std::max(a: WaitStates, b: checkReadM0Hazards(SMovRel: MI));
787
788 if (SIInstrInfo::isMAI(MI: *MI))
789 return std::max(a: WaitStates, b: checkMAIHazards(MI));
790
791 if (SIInstrInfo::isVMEM(MI: *MI) || SIInstrInfo::isDS(MI: *MI))
792 return std::max(a: WaitStates, b: checkMAILdStHazards(MI));
793
794 if (ST.hasPermlaneForwardingHazard() && isPermlane(MI: *MI))
795 return std::max(a: WaitStates, b: checkPermlaneHazards(MI));
796
797 return WaitStates;
798}
799
800void GCNHazardRecognizer::EmitNoop() {
801 EmittedInstrs.push_front(x: nullptr);
802}
803
804void GCNHazardRecognizer::AdvanceCycle() {
805 if (isSchedulerMode())
806 schedulerAdvanceCycle();
807
808 // When the scheduler detects a stall, it will call AdvanceCycle() without
809 // emitting any instructions.
810 if (!CurrCycleInstr) {
811 EmittedInstrs.push_front(x: nullptr);
812
813 if (HasPendingWMMACoexecHazard)
814 EmittedVALUInstrs.push_front(x: nullptr);
815 return;
816 }
817
818 HasPendingWMMACoexecHazard = false;
819
820 if (CurrCycleInstr->isBundle()) {
821 processBundle();
822 return;
823 }
824
825 unsigned NumWaitStates = TII.getNumWaitStates(MI: *CurrCycleInstr);
826 if (!NumWaitStates) {
827 CurrCycleInstr = nullptr;
828 return;
829 }
830
831 // Keep track of emitted instructions
832 EmittedInstrs.push_front(x: CurrCycleInstr);
833
834 bool IsVALUOrWMMA =
835 SIInstrInfo::isVALU(MI: *CurrCycleInstr, /*AllowLDSDMA=*/true) ||
836 SIInstrInfo::isWMMA(MI: *CurrCycleInstr) ||
837 SIInstrInfo::isSWMMAC(MI: *CurrCycleInstr);
838 if (IsVALUOrWMMA) {
839 EmittedVALUInstrs.push_front(x: CurrCycleInstr);
840 } else {
841 // A pending WMMA co-execution hazard optimistically records stall cycles as
842 // future V_NOPs. If the scheduler instead stalls for a different
843 // (S_NOP-resolvable) hazard and schedules a non-VALU into those cycles,
844 // they will not resolve the VALU-pipe hazard, so drop them here.
845 while (!EmittedVALUInstrs.empty() && EmittedVALUInstrs.front() == nullptr)
846 EmittedVALUInstrs.pop_front();
847 }
848
849 // Add a nullptr for each additional wait state after the first. Make sure
850 // not to add more than getMaxLookAhead() items to the list, since we
851 // truncate the list to that size right after this loop.
852 for (unsigned i = 1, e = std::min(a: NumWaitStates, b: getMaxLookAhead());
853 i < e; ++i) {
854 EmittedInstrs.push_front(x: nullptr);
855 }
856
857 // getMaxLookahead() is the largest number of wait states we will ever need
858 // to insert, so there is no point in keeping track of more than that many
859 // wait states.
860 EmittedInstrs.resize(new_size: getMaxLookAhead());
861 if (EmittedVALUInstrs.size() > MaxVALULookAhead)
862 EmittedVALUInstrs.resize(new_size: MaxVALULookAhead);
863
864 CurrCycleInstr = nullptr;
865}
866
867void GCNHazardRecognizer::RecedeCycle() {
868 assert(!isHazardRecognizerMode() &&
869 "Bottom-up scheduling shouldn't run in hazard recognizer mode");
870}
871
872//===----------------------------------------------------------------------===//
873// Helper Functions
874//===----------------------------------------------------------------------===//
875
876enum HazardFnResult { HazardFound, HazardExpired, NoHazardFound };
877
878// Search for a hazard in a block and its predecessors.
879template <typename StateT>
880static bool
881hasHazard(StateT InitialState,
882 function_ref<HazardFnResult(StateT &, const MachineInstr &)> IsHazard,
883 function_ref<void(StateT &, const MachineInstr &)> UpdateState,
884 const MachineBasicBlock *InitialMBB,
885 MachineBasicBlock::const_reverse_instr_iterator InitialI) {
886 struct StateMapKey {
887 SmallVectorImpl<StateT> *States;
888 unsigned Idx;
889 static bool isEqual(const StateMapKey &LHS, const StateMapKey &RHS) {
890 return LHS.States == RHS.States && LHS.Idx == RHS.Idx;
891 }
892 };
893 struct StateMapKeyTraits : DenseMapInfo<StateMapKey> {
894 static unsigned getHashValue(const StateMapKey &Key) {
895 return StateT::getHashValue((*Key.States)[Key.Idx]);
896 }
897 static unsigned getHashValue(const StateT &State) {
898 return StateT::getHashValue(State);
899 }
900 static bool isEqual(const StateMapKey &LHS, const StateMapKey &RHS) {
901 return StateT::isEqual((*LHS.States)[LHS.Idx], (*RHS.States)[RHS.Idx]);
902 }
903 static bool isEqual(const StateT &LHS, const StateMapKey &RHS) {
904 return StateT::isEqual(LHS, (*RHS.States)[RHS.Idx]);
905 }
906 };
907
908 SmallDenseMap<StateMapKey, unsigned, 8, StateMapKeyTraits> StateMap;
909 SmallVector<StateT, 8> States;
910
911 MachineBasicBlock::const_reverse_instr_iterator I = InitialI;
912 const MachineBasicBlock *MBB = InitialMBB;
913 StateT State = InitialState;
914
915 SmallSetVector<std::pair<const MachineBasicBlock *, unsigned>, 16> Worklist;
916 unsigned WorkIdx = 0;
917 for (;;) {
918 bool Expired = false;
919 for (auto E = MBB->instr_rend(); I != E; ++I) {
920 // No need to look at parent BUNDLE instructions.
921 if (I->isBundle())
922 continue;
923
924 auto Result = IsHazard(State, *I);
925 if (Result == HazardFound)
926 return true;
927 if (Result == HazardExpired) {
928 Expired = true;
929 break;
930 }
931
932 if (I->isInlineAsm() || I->isMetaInstruction())
933 continue;
934
935 UpdateState(State, *I);
936 }
937
938 if (!Expired) {
939 unsigned StateIdx = States.size();
940 StateMapKey Key = {&States, StateIdx};
941 auto Insertion = StateMap.insert_as(std::pair(Key, StateIdx), State);
942 if (Insertion.second) {
943 States.emplace_back(State);
944 } else {
945 StateIdx = Insertion.first->second;
946 }
947 for (MachineBasicBlock *Pred : MBB->predecessors())
948 Worklist.insert(X: std::pair(Pred, StateIdx));
949 }
950
951 if (WorkIdx == Worklist.size())
952 break;
953
954 unsigned StateIdx;
955 std::tie(args&: MBB, args&: StateIdx) = Worklist[WorkIdx++];
956 State = States[StateIdx];
957 I = MBB->instr_rbegin();
958 }
959
960 return false;
961}
962
963// Number of trailing terminators of \p Pred not executed on the edge to \p MBB:
964// control leaves at the first terminator branching to \p MBB, so the ones after
965// it are dead on that edge. A fallthrough executes them all.
966static unsigned countUnexecutedTerminators(const MachineBasicBlock &Pred,
967 const MachineBasicBlock *MBB) {
968 const SIInstrInfo &TII =
969 *Pred.getParent()->getSubtarget<GCNSubtarget>().getInstrInfo();
970 unsigned Unexecuted = 0, Passed = 0;
971 for (auto I = Pred.instr_rbegin(), E = Pred.instr_rend();
972 I != E && I->isTerminator(); ++I, ++Passed) {
973 // Only an explicit MBB destination identifies this edge; the last match is
974 // the earliest such branch in program order.
975 if (I->isBranch() && I->getNumOperands() > 0 && I->getOperand(i: 0).isMBB() &&
976 TII.getBranchDestBlock(MI: *I) == MBB)
977 Unexecuted = Passed;
978 }
979 return Unexecuted;
980}
981
982// Wait states \p MI adds to a distance. Inline asm may be empty, so it adds
983// none, which is the worst case.
984static unsigned
985getCountedWaitStates(const MachineInstr &MI,
986 GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates) {
987 return MI.isInlineAsm() ? 0 : GetNumWaitStates(MI);
988}
989
990namespace {
991
992// Answers GCNHazardRecognizer::getMax*WindowDeficit, which documents the
993// result, by walking the CFG backwards from the instruction being checked,
994// counting distances with GetNumWaitStates. That needs the instruction in its
995// final place, so hazard recognizer mode only. Each object answers one query.
996class WindowDeficitSearch {
997public:
998 WindowDeficitSearch(
999 function_ref<std::optional<int>(const MachineInstr &)> WindowFor,
1000 int MaxWindow,
1001 GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates =
1002 SIInstrInfo::getNumWaitStates)
1003 : WindowFor(WindowFor), GetNumWaitStates(GetNumWaitStates),
1004 MaxWindow(MaxWindow) {}
1005
1006 int run(const MachineInstr &MI);
1007
1008private:
1009 // A block to scan upwards from Start, which is Distance away from the
1010 // instruction being checked. In a predecessor, Start is the last instruction
1011 // above the terminators, and Distance includes the ones the edge executes.
1012 struct Arrival {
1013 const MachineBasicBlock *MBB;
1014 MachineBasicBlock::const_reverse_instr_iterator Start;
1015 int Distance;
1016 };
1017
1018 void arrive(const MachineBasicBlock *MBB,
1019 MachineBasicBlock::const_reverse_instr_iterator Start,
1020 int Distance);
1021 void scan(Arrival A);
1022
1023 const function_ref<std::optional<int>(const MachineInstr &)> WindowFor;
1024 const GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates;
1025 const int MaxWindow;
1026 int Deficit = 0;
1027 // Pending[D] holds the arrivals at distance D, and Best the smallest distance
1028 // each block has been queued at. Buckets are emptied in increasing distance,
1029 // so the first arrival scanned at a block is the shortest way into it, and
1030 // any later one is stale.
1031 SmallVector<SmallVector<Arrival, 2>, 4> Pending;
1032 DenseMap<const MachineBasicBlock *, int> Best;
1033};
1034
1035} // end anonymous namespace
1036
1037void WindowDeficitSearch::arrive(
1038 const MachineBasicBlock *MBB,
1039 MachineBasicBlock::const_reverse_instr_iterator Start, int Distance) {
1040 // No window exceeds MaxWindow, so nothing this far back can beat the deficit
1041 // already found.
1042 if (MaxWindow - Distance <= Deficit)
1043 return;
1044
1045 if (Pending.empty())
1046 Pending.resize(N: MaxWindow);
1047
1048 auto [It, Inserted] = Best.try_emplace(Key: MBB, Args&: Distance);
1049 if (!Inserted) {
1050 if (It->second <= Distance)
1051 return;
1052 It->second = Distance;
1053 }
1054
1055 assert(Distance < MaxWindow && "arrival outside the window");
1056 Pending[Distance].push_back(Elt: {.MBB: MBB, .Start: Start, .Distance: Distance});
1057}
1058
1059// \p A is a copy because arrive() may append to the bucket it came from, which
1060// can move that bucket's elements.
1061void WindowDeficitSearch::scan(Arrival A) {
1062 // A shorter way into this block has been queued since.
1063 if (Best.lookup(Val: A.MBB) < A.Distance)
1064 return;
1065
1066 int Distance = A.Distance;
1067 for (auto I = A.Start, E = A.MBB->instr_rend(); I != E; ++I) {
1068 if (I->isBundle())
1069 continue;
1070
1071 if (std::optional<int> Window = WindowFor(*I)) {
1072 assert(*Window >= 0 && *Window <= MaxWindow && "window out of bounds");
1073 Deficit = std::max(a: Deficit, b: *Window - Distance);
1074 }
1075
1076 Distance += getCountedWaitStates(MI: *I, GetNumWaitStates);
1077
1078 if (Distance >= MaxWindow)
1079 return;
1080 }
1081
1082 for (const MachineBasicBlock *Pred : A.MBB->predecessors()) {
1083 auto Start = Pred->instr_rbegin(), End = Pred->instr_rend();
1084 for (unsigned N = countUnexecutedTerminators(Pred: *Pred, MBB: A.MBB); N; --N)
1085 ++Start;
1086
1087 int Executed = 0;
1088 for (; Start != End && Start->isTerminator(); ++Start) {
1089 assert(!WindowFor(*Start) && "terminators must not ask for a window");
1090 Executed += getCountedWaitStates(MI: *Start, GetNumWaitStates);
1091 }
1092
1093 arrive(MBB: Pred, Start, Distance: Distance + Executed);
1094 }
1095}
1096
1097int WindowDeficitSearch::run(const MachineInstr &MI) {
1098 // Only the part above MI is scanned here, so the block stays out of Best: a
1099 // backedge into it must still scan it from its end.
1100 scan(A: {.MBB: MI.getParent(), .Start: std::next(x: MI.getReverseIterator()), .Distance: 0});
1101
1102 // Distances never shrink along a path, so a scan only queues arrivals at the
1103 // distance being swept or further, and one pass over the buckets is enough.
1104 // The inner loop re-reads the size because an edge that adds nothing appends
1105 // to the bucket in hand; Best keeps a cycle that adds nothing from looping.
1106 for (int Distance = 0; Distance < static_cast<int>(Pending.size());
1107 ++Distance)
1108 for (unsigned I = 0; I != Pending[Distance].size(); ++I)
1109 scan(A: Pending[Distance][I]);
1110
1111 return Deficit;
1112}
1113
1114// Returns a minimum wait states since \p I walking all predecessors.
1115// Only scans until \p IsExpired does not return true.
1116// Can only be run in a hazard recognizer mode.
1117static int
1118getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
1119 const MachineBasicBlock *MBB,
1120 MachineBasicBlock::const_reverse_instr_iterator I,
1121 int WaitStates, GCNHazardRecognizer::IsExpiredFn IsExpired,
1122 DenseSet<const MachineBasicBlock *> &Visited,
1123 GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates =
1124 SIInstrInfo::getNumWaitStates) {
1125 for (auto E = MBB->instr_rend(); I != E; ++I) {
1126 // Don't add WaitStates for parent BUNDLE instructions.
1127 if (I->isBundle())
1128 continue;
1129
1130 if (IsHazard(*I))
1131 return WaitStates;
1132
1133 if (I->isInlineAsm())
1134 continue;
1135
1136 WaitStates += GetNumWaitStates(*I);
1137
1138 if (IsExpired(*I, WaitStates))
1139 return std::numeric_limits<int>::max();
1140 }
1141
1142 int MinWaitStates = std::numeric_limits<int>::max();
1143 for (MachineBasicBlock *Pred : MBB->predecessors()) {
1144 if (!Visited.insert(V: Pred).second)
1145 continue;
1146
1147 int W = getWaitStatesSince(IsHazard, MBB: Pred, I: Pred->instr_rbegin(), WaitStates,
1148 IsExpired, Visited, GetNumWaitStates);
1149
1150 MinWaitStates = std::min(a: MinWaitStates, b: W);
1151 }
1152
1153 return MinWaitStates;
1154}
1155
1156static int
1157getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
1158 const MachineInstr *MI,
1159 GCNHazardRecognizer::IsExpiredFn IsExpired,
1160 GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates =
1161 SIInstrInfo::getNumWaitStates) {
1162 DenseSet<const MachineBasicBlock *> Visited;
1163 return getWaitStatesSince(IsHazard, MBB: MI->getParent(),
1164 I: std::next(x: MI->getReverseIterator()), WaitStates: 0, IsExpired,
1165 Visited, GetNumWaitStates);
1166}
1167
1168int GCNHazardRecognizer::getMaxWindowDeficit(int MaxWindow,
1169 WindowForFn WindowFor) const {
1170 if (isHazardRecognizerMode())
1171 return WindowDeficitSearch(WindowFor, MaxWindow).run(MI: *CurrCycleInstr);
1172
1173 // EmittedInstrs is capped and can be shorter than the widest window, which
1174 // costs scheduling quality only: the standalone pass still pads.
1175 int Deficit = 0, Distance = 0;
1176 for (MachineInstr *MI : EmittedInstrs) {
1177 if (MI) {
1178 if (std::optional<int> Window = WindowFor(*MI)) {
1179 assert(*Window >= 0 && *Window <= MaxWindow && "window out of bounds");
1180 Deficit = std::max(a: Deficit, b: *Window - Distance);
1181 }
1182 }
1183
1184 Distance +=
1185 MI ? getCountedWaitStates(MI: *MI, GetNumWaitStates: SIInstrInfo::getNumWaitStates) : 1;
1186
1187 if (Distance >= MaxWindow)
1188 break;
1189 }
1190 return Deficit;
1191}
1192
1193int GCNHazardRecognizer::getWaitStatesSince(
1194 IsHazardFn IsHazard, int Limit, GetNumWaitStatesFn GetNumWaitStates) const {
1195 if (isHazardRecognizerMode()) {
1196 auto IsExpiredFn = [Limit](const MachineInstr &, int WaitStates) {
1197 return WaitStates >= Limit;
1198 };
1199 return ::getWaitStatesSince(IsHazard, MI: CurrCycleInstr, IsExpired: IsExpiredFn,
1200 GetNumWaitStates);
1201 }
1202
1203 int WaitStates = 0;
1204 for (MachineInstr *MI : EmittedInstrs) {
1205 if (MI) {
1206 if (IsHazard(*MI))
1207 return WaitStates;
1208
1209 if (MI->isInlineAsm())
1210 continue;
1211 }
1212 WaitStates += MI ? GetNumWaitStates(*MI) : 1;
1213
1214 if (WaitStates >= Limit)
1215 break;
1216 }
1217 return std::numeric_limits<int>::max();
1218}
1219
1220int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard,
1221 int Limit) const {
1222 return getWaitStatesSince(IsHazard, Limit, GetNumWaitStates: SIInstrInfo::getNumWaitStates);
1223}
1224
1225int GCNHazardRecognizer::getMaxVALUWindowDeficit(int MaxWindow,
1226 WindowForFn WindowFor) const {
1227 if (isHazardRecognizerMode()) {
1228 auto GetVALUWaitStates = [](const MachineInstr &MI) -> unsigned {
1229 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ? 1 : 0;
1230 };
1231 return WindowDeficitSearch(WindowFor, MaxWindow, GetVALUWaitStates)
1232 .run(MI: *CurrCycleInstr);
1233 }
1234
1235 // EmittedVALUInstrs is capped at MaxVALULookAhead, so a MaxWindow beyond that
1236 // window could miss a hazard. Keep the cap in sync with the wait-state
1237 // tables.
1238 assert(MaxWindow <= static_cast<int>(MaxVALULookAhead) &&
1239 "MaxWindow exceeds the EmittedVALUInstrs lookahead window");
1240 int WaitStates = 0;
1241 for (MachineInstr *MI : EmittedVALUInstrs) {
1242 if (MI) {
1243 if (std::optional<int> Window = WindowFor(*MI)) {
1244 assert(*Window >= 0 && *Window <= MaxWindow && "window out of bounds");
1245 return *Window - WaitStates;
1246 }
1247 }
1248
1249 ++WaitStates;
1250
1251 if (WaitStates >= MaxWindow)
1252 break;
1253 }
1254 return 0;
1255}
1256
1257int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg,
1258 IsHazardFn IsHazardDef,
1259 int Limit) const {
1260 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1261
1262 auto IsHazardFn = [IsHazardDef, TRI, Reg](const MachineInstr &MI) {
1263 return IsHazardDef(MI) && MI.modifiesRegister(Reg, TRI);
1264 };
1265
1266 return getWaitStatesSince(IsHazard: IsHazardFn, Limit);
1267}
1268
1269int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard,
1270 int Limit) const {
1271 auto IsHazardFn = [IsHazard](const MachineInstr &MI) {
1272 return isSSetReg(Opcode: MI.getOpcode()) && IsHazard(MI);
1273 };
1274
1275 return getWaitStatesSince(IsHazard: IsHazardFn, Limit);
1276}
1277
1278//===----------------------------------------------------------------------===//
1279// No-op Hazard Detection
1280//===----------------------------------------------------------------------===//
1281
1282static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV,
1283 MCRegister Reg) {
1284 for (MCRegUnit Unit : TRI.regunits(Reg))
1285 BV.set(static_cast<unsigned>(Unit));
1286}
1287
1288static void addRegsToSet(const SIRegisterInfo &TRI,
1289 iterator_range<MachineInstr::const_mop_iterator> Ops,
1290 BitVector &DefSet, BitVector &UseSet) {
1291 for (const MachineOperand &Op : Ops) {
1292 if (Op.isReg())
1293 addRegUnits(TRI, BV&: Op.isDef() ? DefSet : UseSet, Reg: Op.getReg().asMCReg());
1294 }
1295}
1296
1297void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) const {
1298 addRegsToSet(TRI, Ops: MI.operands(), DefSet&: ClauseDefs, UseSet&: ClauseUses);
1299}
1300
1301static bool breaksSMEMSoftClause(MachineInstr *MI) {
1302 return !SIInstrInfo::isSMRD(MI: *MI);
1303}
1304
1305static bool breaksVMEMSoftClause(MachineInstr *MI) {
1306 return !SIInstrInfo::isVMEM(MI: *MI);
1307}
1308
1309int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) const {
1310 // SMEM soft clause are only present on VI+, and only matter if xnack is
1311 // enabled.
1312 if (!ST.isXNACKEnabled())
1313 return 0;
1314
1315 bool IsSMRD = TII.isSMRD(MI: *MEM);
1316
1317 resetClause();
1318
1319 // A soft-clause is any group of consecutive SMEM instructions. The
1320 // instructions in this group may return out of order and/or may be
1321 // replayed (i.e. the same instruction issued more than once).
1322 //
1323 // In order to handle these situations correctly we need to make sure that
1324 // when a clause has more than one instruction, no instruction in the clause
1325 // writes to a register that is read by another instruction in the clause
1326 // (including itself). If we encounter this situation, we need to break the
1327 // clause by inserting a non SMEM instruction.
1328
1329 for (MachineInstr *MI : EmittedInstrs) {
1330 // When we hit a non-SMEM instruction then we have passed the start of the
1331 // clause and we can stop.
1332 if (!MI)
1333 break;
1334
1335 if (IsSMRD ? breaksSMEMSoftClause(MI) : breaksVMEMSoftClause(MI))
1336 break;
1337
1338 addClauseInst(MI: *MI);
1339 }
1340
1341 if (ClauseDefs.none())
1342 return 0;
1343
1344 // We need to make sure not to put loads and stores in the same clause if they
1345 // use the same address. For now, just start a new clause whenever we see a
1346 // store.
1347 if (MEM->mayStore())
1348 return 1;
1349
1350 addClauseInst(MI: *MEM);
1351
1352 // If the set of defs and uses intersect then we cannot add this instruction
1353 // to the clause, so we have a hazard.
1354 return ClauseDefs.anyCommon(RHS: ClauseUses) ? 1 : 0;
1355}
1356
1357int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) const {
1358 int WaitStatesNeeded = 0;
1359
1360 WaitStatesNeeded = checkSoftClauseHazards(MEM: SMRD);
1361
1362 // This SMRD hazard only affects SI.
1363 if (!ST.hasSMRDReadVALUDefHazard())
1364 return WaitStatesNeeded;
1365
1366 // A read of an SGPR by SMRD instruction requires 4 wait states when the
1367 // SGPR was written by a VALU instruction.
1368 int SmrdSgprWaitStates = 4;
1369 auto IsHazardDefFn = [this](const MachineInstr &MI) {
1370 return TII.isVALU(MI, /*AllowLDSDMA=*/true);
1371 };
1372 auto IsBufferHazardDefFn = [this](const MachineInstr &MI) {
1373 return TII.isSALU(MI);
1374 };
1375
1376 bool IsBufferSMRD = TII.isBufferSMRD(MI: *SMRD);
1377
1378 for (const MachineOperand &Use : SMRD->uses()) {
1379 if (!Use.isReg())
1380 continue;
1381 int WaitStatesNeededForUse =
1382 SmrdSgprWaitStates - getWaitStatesSinceDef(Reg: Use.getReg(), IsHazardDef: IsHazardDefFn,
1383 Limit: SmrdSgprWaitStates);
1384 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
1385
1386 // This fixes what appears to be undocumented hardware behavior in SI where
1387 // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor
1388 // needs some number of nops in between. We don't know how many we need, but
1389 // let's use 4. This wasn't discovered before probably because the only
1390 // case when this happens is when we expand a 64-bit pointer into a full
1391 // descriptor and use s_buffer_load_dword instead of s_load_dword, which was
1392 // probably never encountered in the closed-source land.
1393 if (IsBufferSMRD) {
1394 int WaitStatesNeededForUse =
1395 SmrdSgprWaitStates - getWaitStatesSinceDef(Reg: Use.getReg(),
1396 IsHazardDef: IsBufferHazardDefFn,
1397 Limit: SmrdSgprWaitStates);
1398 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
1399 }
1400 }
1401
1402 return WaitStatesNeeded;
1403}
1404
1405int GCNHazardRecognizer::checkVMEMHazards(MachineInstr *VMEM) const {
1406 if (!ST.hasVMEMReadSGPRVALUDefHazard())
1407 return 0;
1408
1409 int WaitStatesNeeded = checkSoftClauseHazards(MEM: VMEM);
1410
1411 // A read of an SGPR by a VMEM instruction requires 5 wait states when the
1412 // SGPR was written by a VALU Instruction.
1413 const int VmemSgprWaitStates = 5;
1414 auto IsHazardDefFn = [this](const MachineInstr &MI) {
1415 return TII.isVALU(MI, /*AllowLDSDMA=*/true);
1416 };
1417 for (const MachineOperand &Use : VMEM->uses()) {
1418 if (!Use.isReg() || TRI.isVectorRegister(MRI: MF.getRegInfo(), Reg: Use.getReg()))
1419 continue;
1420
1421 int WaitStatesNeededForUse =
1422 VmemSgprWaitStates - getWaitStatesSinceDef(Reg: Use.getReg(), IsHazardDef: IsHazardDefFn,
1423 Limit: VmemSgprWaitStates);
1424 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
1425 }
1426 return WaitStatesNeeded;
1427}
1428
1429int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) const {
1430 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1431 const SIInstrInfo *TII = ST.getInstrInfo();
1432
1433 // Check for DPP VGPR read after VALU VGPR write and EXEC write.
1434 int DppVgprWaitStates = 2;
1435 int DppExecWaitStates = 5;
1436 int WaitStatesNeeded = 0;
1437 auto IsHazardDefFn = [TII](const MachineInstr &MI) {
1438 return TII->isVALU(MI, /*AllowLDSDMA=*/true);
1439 };
1440
1441 for (const MachineOperand &Use : DPP->uses()) {
1442 if (!Use.isReg() || !TRI->isVGPR(MRI: MF.getRegInfo(), Reg: Use.getReg()))
1443 continue;
1444 int WaitStatesNeededForUse =
1445 DppVgprWaitStates - getWaitStatesSinceDef(
1446 Reg: Use.getReg(),
1447 IsHazardDef: [](const MachineInstr &) { return true; },
1448 Limit: DppVgprWaitStates);
1449 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
1450 }
1451
1452 WaitStatesNeeded = std::max(
1453 a: WaitStatesNeeded,
1454 b: DppExecWaitStates - getWaitStatesSinceDef(Reg: AMDGPU::EXEC, IsHazardDef: IsHazardDefFn,
1455 Limit: DppExecWaitStates));
1456
1457 return WaitStatesNeeded;
1458}
1459
1460int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) const {
1461 const SIInstrInfo *TII = ST.getInstrInfo();
1462
1463 // v_div_fmas requires 4 wait states after a write to vcc from a VALU
1464 // instruction.
1465 const int DivFMasWaitStates = 4;
1466 auto IsHazardDefFn = [TII](const MachineInstr &MI) {
1467 return TII->isVALU(MI, /*AllowLDSDMA=*/true);
1468 };
1469 int WaitStatesNeeded = getWaitStatesSinceDef(Reg: AMDGPU::VCC, IsHazardDef: IsHazardDefFn,
1470 Limit: DivFMasWaitStates);
1471
1472 return DivFMasWaitStates - WaitStatesNeeded;
1473}
1474
1475int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) const {
1476 const SIInstrInfo *TII = ST.getInstrInfo();
1477 unsigned GetRegHWReg = getHWReg(TII, RegInstr: *GetRegInstr);
1478
1479 const int GetRegWaitStates = 2;
1480 auto IsHazardFn = [TII, GetRegHWReg](const MachineInstr &MI) {
1481 return GetRegHWReg == getHWReg(TII, RegInstr: MI);
1482 };
1483 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazard: IsHazardFn, Limit: GetRegWaitStates);
1484
1485 return GetRegWaitStates - WaitStatesNeeded;
1486}
1487
1488int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) const {
1489 const SIInstrInfo *TII = ST.getInstrInfo();
1490 unsigned HWReg = getHWReg(TII, RegInstr: *SetRegInstr);
1491
1492 const int SetRegWaitStates = ST.getSetRegWaitStates();
1493 auto IsHazardFn = [TII, HWReg](const MachineInstr &MI) {
1494 return HWReg == getHWReg(TII, RegInstr: MI);
1495 };
1496 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazard: IsHazardFn, Limit: SetRegWaitStates);
1497 return SetRegWaitStates - WaitStatesNeeded;
1498}
1499
1500int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) const {
1501 if (!MI.mayStore())
1502 return -1;
1503
1504 const SIInstrInfo *TII = ST.getInstrInfo();
1505 unsigned Opcode = MI.getOpcode();
1506 const MCInstrDesc &Desc = MI.getDesc();
1507
1508 int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::vdata);
1509 int VDataRCID = -1;
1510 if (VDataIdx != -1)
1511 VDataRCID = TII->getOpRegClassID(OpInfo: Desc.operands()[VDataIdx]);
1512
1513 if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) {
1514 // There is no hazard if the instruction does not use vector regs
1515 // (like wbinvl1)
1516 if (VDataIdx == -1)
1517 return -1;
1518 if (AMDGPU::getRegBitWidth(RCID: VDataRCID) > 64) {
1519 // When SOFFSET-dependent wide-store windows apply, the BUFFER_STORE
1520 // source-vgpr WAR hazard exists for every SOFFSET shape; the wait-state
1521 // count differs by SOFFSET and is computed in checkVALUHazardsHelper.
1522 // Otherwise the hazard only exists if soffset is not an SGPR.
1523 if (ST.hasVDecCoExecHazard())
1524 return VDataIdx;
1525 const MachineOperand *SOffset =
1526 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
1527 if (!SOffset || !SOffset->isReg())
1528 return VDataIdx;
1529 }
1530 }
1531
1532 // MIMG instructions create a hazard if they don't use a 256-bit T# and
1533 // the store size is greater than 8 bytes and they have more than two bits
1534 // of their dmask set.
1535 // All our MIMG definitions use a 256-bit T#, so we can skip checking for them.
1536 if (TII->isMIMG(MI)) {
1537 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::srsrc);
1538 assert(SRsrcIdx != -1 && AMDGPU::getRegBitWidth(TII->getOpRegClassID(
1539 Desc.operands()[SRsrcIdx])) == 256);
1540 (void)SRsrcIdx;
1541 }
1542
1543 if (TII->isFLAT(MI)) {
1544 // There is no hazard if the instruction does not use vector regs
1545 if (VDataIdx == -1)
1546 return -1;
1547
1548 if (AMDGPU::getRegBitWidth(RCID: VDataRCID) > 64)
1549 return VDataIdx;
1550 }
1551
1552 return -1;
1553}
1554
1555int GCNHazardRecognizer::checkUniformWindowVALUHazardsHelper(
1556 Register Reg) const {
1557 // Wide stores need a single wait-state bubble before a VALU that overwrites
1558 // store data. createsVALUHazard already excludes MUBUF/MTBUF stores with an
1559 // SGPR SOFFSET.
1560 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1561
1562 auto IsHazard = [&](const MachineInstr &MI) {
1563 int DataIdx = createsVALUHazard(MI);
1564 return DataIdx >= 0 &&
1565 TRI->regsOverlap(RegA: MI.getOperand(i: DataIdx).getReg(), RegB: Reg);
1566 };
1567
1568 return std::max(a: 0, b: 1 - getWaitStatesSince(IsHazard, /*Limit=*/1));
1569}
1570
1571int GCNHazardRecognizer::checkSOFFSETWindowVALUHazardsHelper(
1572 Register Reg) const {
1573 // The required wait-state window depends on the producer's SOFFSET shape:
1574 // - MUBUF/MTBUF wide store with sgpr SOFFSET: 1 wait state.
1575 // - MUBUF/MTBUF wide store with literal/absent SOFFSET, and FLAT wide
1576 // store: 2 wait states.
1577 // The 1-cycle sgpr-SOFFSET window was measured on gfx950.
1578 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1579 const SIInstrInfo *TII = ST.getInstrInfo();
1580
1581 int WaitStatesNeeded = 0;
1582
1583 // Scan each wait-state window separately and take the max padding needed.
1584 // getWaitStatesSince supplies the minimum distance to a producer over paths.
1585 for (int Window = 1; Window <= 2; ++Window) {
1586 auto IsHazard = [&](const MachineInstr &MI) {
1587 int DataIdx = createsVALUHazard(MI);
1588 if (DataIdx < 0 ||
1589 !TRI->regsOverlap(RegA: MI.getOperand(i: DataIdx).getReg(), RegB: Reg))
1590 return false;
1591
1592 // Window 1 matches every hazard producer. Window 2 excludes BUF stores
1593 // with an SGPR SOFFSET, which only require a single wait state.
1594 if (Window == 1 || !TII->isBUF(MI))
1595 return true;
1596
1597 const MachineOperand *SOffset =
1598 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
1599 return !SOffset || !SOffset->isReg();
1600 };
1601 WaitStatesNeeded = std::max(a: WaitStatesNeeded,
1602 b: Window - getWaitStatesSince(IsHazard, Limit: Window));
1603 }
1604
1605 return WaitStatesNeeded;
1606}
1607
1608int GCNHazardRecognizer::checkVALUHazardsHelper(
1609 const MachineOperand &Def, const MachineRegisterInfo &MRI) const {
1610 // Helper to check for the hazard where VMEM instructions that store more
1611 // than 8 bytes can have their store data overwritten by the next
1612 // instruction.
1613 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1614
1615 if (!TRI->isVectorRegister(MRI, Reg: Def.getReg()))
1616 return 0;
1617
1618 if (ST.hasVDecCoExecHazard())
1619 return checkSOFFSETWindowVALUHazardsHelper(Reg: Def.getReg());
1620
1621 return checkUniformWindowVALUHazardsHelper(Reg: Def.getReg());
1622}
1623
1624/// Dest sel forwarding issue occurs if additional logic is needed to swizzle /
1625/// pack the computed value into correct bit position of the dest register. This
1626/// occurs if we have SDWA with dst_sel != DWORD or if we have op_sel with
1627/// dst_sel that is not aligned to the register. This function analayzes the \p
1628/// MI and \returns an operand with dst forwarding issue, or nullptr if
1629/// none exists.
1630static const MachineOperand *
1631getDstSelForwardingOperand(const MachineInstr &MI, const GCNSubtarget &ST) {
1632 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false))
1633 return nullptr;
1634
1635 const SIInstrInfo *TII = ST.getInstrInfo();
1636
1637 unsigned Opcode = MI.getOpcode();
1638
1639 // There are three different types of instructions
1640 // which produce forwarded dest: 1. SDWA with dst_sel != DWORD, 2. VOP3
1641 // which write hi bits (e.g. op_sel[3] == 1), and 3. FP8DstSelInst
1642 // (instructions with dest byte sel, e.g. CVT_SR_BF8_F32) and
1643 // op_sel[3:2]
1644 // != 0
1645 if (SIInstrInfo::isSDWA(MI)) {
1646 // Type 1: SDWA with dst_sel != DWORD
1647 if (auto *DstSel = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::dst_sel))
1648 if (DstSel->getImm() != AMDGPU::SDWA::DWORD)
1649 return TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1650 }
1651
1652 AMDGPU::FPType IsFP4OrFP8ConvOpc = AMDGPU::getFPDstSelType(Opc: Opcode);
1653 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::op_sel)) {
1654 // Type 2: VOP3 which write the hi bits. DST_OP_SEL is a destination
1655 // selector in the VOP3 layout; VOP3P uses this bit for source op_sel_hi.
1656 if (SIInstrInfo::isVOP3(MI) && !SIInstrInfo::isVOP3P(MI) &&
1657 (TII->getNamedImmOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers) &
1658 SISrcMods::DST_OP_SEL))
1659 return TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1660
1661 // Type 2b: VOP3P mix instructions with a tied destination (MIXLO/MIXHI)
1662 // write only one half of vdst and preserve the other half, so treat them
1663 // like a VOP3 destination op_sel write regardless of their source
1664 // op_sel_hi bits.
1665 if (SIInstrInfo::isVOP3P(MI) &&
1666 AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::vdst_in))
1667 return TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1668
1669 // Type 3: FP8DstSelInst with op_sel[3:2] != 0)
1670 if (IsFP4OrFP8ConvOpc == AMDGPU::FPType::FP8 &&
1671 (TII->getNamedImmOperand(MI, OperandName: AMDGPU::OpName::src2_modifiers) &
1672 SISrcMods::OP_SEL_0))
1673 return TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1674 }
1675
1676 // Special case: nop is required for all the opsel values for fp4 sr variant
1677 // cvt scale instructions
1678 if (IsFP4OrFP8ConvOpc == AMDGPU::FPType::FP4)
1679 return TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1680
1681 return nullptr;
1682}
1683
1684/// Checks whether the provided \p MI "consumes" the operand with a Dest sel
1685/// fowarding issue \p Dst . We may "consume" the Dst via a standard explicit
1686/// RAW, or through irregular ways (e.g implicit RAW, certain types of WAW)
1687static bool consumesDstSelForwardingOperand(const MachineInstr *VALU,
1688 const MachineOperand *Dst,
1689 const SIRegisterInfo *TRI) {
1690 // We must consider implicit reads of the VALU. SDWA with dst_sel and
1691 // UNUSED_PRESERVE will implicitly read the result from forwarded dest,
1692 // and we must account for that hazard.
1693 // We also must account for WAW hazards. In particular, WAW with dest
1694 // preserve semantics (e.g. VOP3 with op_sel, VOP2 &&
1695 // !zeroesHigh16BitsOfDest) will read the forwarded dest for parity
1696 // check for ECC. Without accounting for this hazard, the ECC will be
1697 // wrong.
1698 // TODO: limit to RAW (including implicit reads) + problematic WAW (i.e.
1699 // complete zeroesHigh16BitsOfDest)
1700 for (auto &Operand : VALU->operands()) {
1701 if (Operand.isReg() && TRI->regsOverlap(RegA: Dst->getReg(), RegB: Operand.getReg())) {
1702 return true;
1703 }
1704 }
1705 return false;
1706}
1707
1708int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) const {
1709 int WaitStatesNeeded = 0;
1710
1711 if (ST.hasTransForwardingHazard() && !SIInstrInfo::isTRANS(MI: *VALU)) {
1712 const int TransDefWaitstates = 1;
1713
1714 auto IsTransDefFn = [this, VALU](const MachineInstr &MI) {
1715 if (!SIInstrInfo::isTRANS(MI))
1716 return false;
1717 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1718 const SIInstrInfo *TII = ST.getInstrInfo();
1719 Register Def = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst)->getReg();
1720
1721 for (const MachineOperand &Use : VALU->explicit_uses()) {
1722 if (Use.isReg() && TRI->regsOverlap(RegA: Def, RegB: Use.getReg()))
1723 return true;
1724 }
1725
1726 return false;
1727 };
1728
1729 int WaitStatesNeededForDef =
1730 TransDefWaitstates -
1731 getWaitStatesSince(IsHazard: IsTransDefFn, Limit: TransDefWaitstates);
1732 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1733 }
1734
1735 if (ST.hasDstSelForwardingHazard() || ST.hasCvtScaleForwardingHazard()) {
1736 const int Shift16DefWaitstates = 1;
1737
1738 auto IsShift16BitDefFn = [this, VALU](const MachineInstr &ProducerMI) {
1739 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1740 const MachineOperand *ForwardedDst =
1741 getDstSelForwardingOperand(MI: ProducerMI, ST);
1742 if (ForwardedDst) {
1743 return consumesDstSelForwardingOperand(VALU, Dst: ForwardedDst, TRI);
1744 }
1745
1746 if (ProducerMI.isInlineAsm()) {
1747 // Assume inline asm has dst forwarding hazard
1748 for (auto &Def : ProducerMI.all_defs()) {
1749 if (consumesDstSelForwardingOperand(VALU, Dst: &Def, TRI))
1750 return true;
1751 }
1752 }
1753
1754 return false;
1755 };
1756
1757 int WaitStatesNeededForDef =
1758 Shift16DefWaitstates -
1759 getWaitStatesSince(IsHazard: IsShift16BitDefFn, Limit: Shift16DefWaitstates);
1760 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1761 }
1762
1763 if (ST.hasVDecCoExecHazard()) {
1764 const int VALUWriteSGPRVALUReadWaitstates = 2;
1765 const int VALUWriteEXECRWLane = 4;
1766 const int VALUWriteVGPRReadlaneRead = 1;
1767
1768 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1769 const MachineRegisterInfo &MRI = MF.getRegInfo();
1770 Register UseReg;
1771 auto IsVALUDefSGPRFn = [&UseReg, TRI](const MachineInstr &MI) {
1772 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
1773 return false;
1774 return MI.modifiesRegister(Reg: UseReg, TRI);
1775 };
1776
1777 for (const MachineOperand &Use : VALU->explicit_uses()) {
1778 if (!Use.isReg())
1779 continue;
1780
1781 UseReg = Use.getReg();
1782 if (TRI->isSGPRReg(MRI, Reg: UseReg)) {
1783 int WaitStatesNeededForDef =
1784 VALUWriteSGPRVALUReadWaitstates -
1785 getWaitStatesSince(IsHazard: IsVALUDefSGPRFn,
1786 Limit: VALUWriteSGPRVALUReadWaitstates);
1787 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1788 }
1789 }
1790
1791 if (VALU->readsRegister(Reg: AMDGPU::VCC, TRI)) {
1792 UseReg = AMDGPU::VCC;
1793 int WaitStatesNeededForDef =
1794 VALUWriteSGPRVALUReadWaitstates -
1795 getWaitStatesSince(IsHazard: IsVALUDefSGPRFn, Limit: VALUWriteSGPRVALUReadWaitstates);
1796 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1797 }
1798
1799 switch (VALU->getOpcode()) {
1800 case AMDGPU::V_READLANE_B32:
1801 case AMDGPU::V_READFIRSTLANE_B32: {
1802 MachineOperand *Src = TII.getNamedOperand(MI&: *VALU, OperandName: AMDGPU::OpName::src0);
1803 UseReg = Src->getReg();
1804 int WaitStatesNeededForDef =
1805 VALUWriteVGPRReadlaneRead -
1806 getWaitStatesSince(IsHazard: IsVALUDefSGPRFn, Limit: VALUWriteVGPRReadlaneRead);
1807 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1808 }
1809 [[fallthrough]];
1810 case AMDGPU::V_WRITELANE_B32: {
1811 UseReg = AMDGPU::EXEC;
1812 int WaitStatesNeededForDef =
1813 VALUWriteEXECRWLane -
1814 getWaitStatesSince(IsHazard: IsVALUDefSGPRFn, Limit: VALUWriteEXECRWLane);
1815 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1816 break;
1817 }
1818 default:
1819 break;
1820 }
1821 }
1822
1823 // This checks for the hazard where VMEM instructions that store more than
1824 // 8 bytes can have there store data over written by the next instruction.
1825 if (!ST.has12DWordStoreHazard())
1826 return WaitStatesNeeded;
1827
1828 const MachineRegisterInfo &MRI = MF.getRegInfo();
1829
1830 for (const MachineOperand &Def : VALU->defs()) {
1831 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: checkVALUHazardsHelper(Def, MRI));
1832 }
1833
1834 return WaitStatesNeeded;
1835}
1836
1837int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) const {
1838 // This checks for hazards associated with inline asm statements.
1839 // Since inline asms can contain just about anything, we use this
1840 // to call/leverage other check*Hazard routines. Note that
1841 // this function doesn't attempt to address all possible inline asm
1842 // hazards (good luck), but is a collection of what has been
1843 // problematic thus far.
1844
1845 // see checkVALUHazards()
1846 if (!ST.has12DWordStoreHazard() && !ST.hasDstSelForwardingHazard() &&
1847 !ST.hasCvtScaleForwardingHazard())
1848 return 0;
1849
1850 const MachineRegisterInfo &MRI = MF.getRegInfo();
1851 int WaitStatesNeeded = 0;
1852
1853 for (const MachineOperand &Op :
1854 llvm::drop_begin(RangeOrContainer: IA->operands(), N: InlineAsm::MIOp_FirstOperand)) {
1855 if (Op.isReg() && Op.isDef()) {
1856 if (!TRI.isVectorRegister(MRI, Reg: Op.getReg()))
1857 continue;
1858
1859 if (ST.has12DWordStoreHazard()) {
1860 WaitStatesNeeded =
1861 std::max(a: WaitStatesNeeded, b: checkVALUHazardsHelper(Def: Op, MRI));
1862 }
1863 }
1864 }
1865
1866 if (ST.hasDstSelForwardingHazard()) {
1867 const int Shift16DefWaitstates = 1;
1868
1869 auto IsShift16BitDefFn = [this, &IA](const MachineInstr &ProducerMI) {
1870 const MachineOperand *Dst = getDstSelForwardingOperand(MI: ProducerMI, ST);
1871 // Assume inline asm reads the dst
1872 if (Dst)
1873 return IA->modifiesRegister(Reg: Dst->getReg(), TRI: &TRI) ||
1874 IA->readsRegister(Reg: Dst->getReg(), TRI: &TRI);
1875
1876 if (ProducerMI.isInlineAsm()) {
1877 // If MI is inline asm, assume it has dst forwarding hazard
1878 for (auto &Def : ProducerMI.all_defs()) {
1879 if (IA->modifiesRegister(Reg: Def.getReg(), TRI: &TRI) ||
1880 IA->readsRegister(Reg: Def.getReg(), TRI: &TRI)) {
1881 return true;
1882 }
1883 }
1884 }
1885
1886 return false;
1887 };
1888
1889 int WaitStatesNeededForDef =
1890 Shift16DefWaitstates -
1891 getWaitStatesSince(IsHazard: IsShift16BitDefFn, Limit: Shift16DefWaitstates);
1892 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1893 }
1894
1895 return WaitStatesNeeded;
1896}
1897
1898int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) const {
1899 const SIInstrInfo *TII = ST.getInstrInfo();
1900 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1901 const MachineRegisterInfo &MRI = MF.getRegInfo();
1902
1903 const MachineOperand *LaneSelectOp =
1904 TII->getNamedOperand(MI&: *RWLane, OperandName: AMDGPU::OpName::src1);
1905
1906 if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, Reg: LaneSelectOp->getReg()))
1907 return 0;
1908
1909 Register LaneSelectReg = LaneSelectOp->getReg();
1910 auto IsHazardFn = [TII](const MachineInstr &MI) {
1911 return TII->isVALU(MI, /*AllowLDSDMA=*/true);
1912 };
1913
1914 const int RWLaneWaitStates = 4;
1915 int WaitStatesSince = getWaitStatesSinceDef(Reg: LaneSelectReg, IsHazardDef: IsHazardFn,
1916 Limit: RWLaneWaitStates);
1917 return RWLaneWaitStates - WaitStatesSince;
1918}
1919
1920int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) const {
1921 if (!ST.hasRFEHazards())
1922 return 0;
1923
1924 const SIInstrInfo *TII = ST.getInstrInfo();
1925
1926 const int RFEWaitStates = 1;
1927
1928 auto IsHazardFn = [TII](const MachineInstr &MI) {
1929 return getHWReg(TII, RegInstr: MI) == AMDGPU::Hwreg::ID_TRAPSTS;
1930 };
1931 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazard: IsHazardFn, Limit: RFEWaitStates);
1932 return RFEWaitStates - WaitStatesNeeded;
1933}
1934
1935int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) const {
1936 const SIInstrInfo *TII = ST.getInstrInfo();
1937 const int ReadM0WaitStates = 1;
1938 auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isSALU(MI); };
1939 return ReadM0WaitStates -
1940 getWaitStatesSinceDef(Reg: AMDGPU::M0, IsHazardDef: IsHazardFn, Limit: ReadM0WaitStates);
1941}
1942
1943void GCNHazardRecognizer::emitVNops(MachineBasicBlock &MBB,
1944 MachineBasicBlock::iterator InsertPt,
1945 int WaitStatesNeeded, bool IsHoisting) {
1946 const DebugLoc &DL = IsHoisting ? DebugLoc() : InsertPt->getDebugLoc();
1947 for (int I = 0; I < WaitStatesNeeded; ++I)
1948 BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_NOP_e32));
1949}
1950
1951void GCNHazardRecognizer::fixHazards(MachineInstr *MI) {
1952 fixVMEMtoScalarWriteHazards(MI);
1953 fixVcmpxPermlaneHazards(MI);
1954 fixSMEMtoVectorWriteHazards(MI);
1955 fixVcmpxExecWARHazard(MI);
1956 fixLdsBranchVmemWARHazard(MI);
1957 if (ST.hasLdsDirect()) {
1958 fixLdsDirectVALUHazard(MI);
1959 fixLdsDirectVMEMHazard(MI);
1960 }
1961 fixVALUPartialForwardingHazard(MI);
1962 fixVALUTransUseHazard(MI);
1963 fixVALUTransCoexecutionHazards(MI);
1964 fixWMMAHazards(MI); // fall-through if co-execution is enabled.
1965 fixWMMACoexecutionHazards(MI);
1966 fixShift64HighRegBug(MI);
1967 fixVALUMaskWriteHazard(MI);
1968 fixRequiredExportPriority(MI);
1969 if (ST.hasVPermPk16Hazard())
1970 fixVPermPk16Hazard(MI);
1971 if (ST.requiresWaitIdleBeforeGetReg())
1972 fixGetRegWaitIdle(MI);
1973 if (ST.hasDsAtomicAsyncBarrierArriveB64PipeBug())
1974 fixDsAtomicAsyncBarrierArriveB64(MI);
1975 if (ST.hasScratchBaseForwardingHazard())
1976 fixScratchBaseForwardingHazard(MI);
1977 if (ST.setRegModeNeedsVNOPs())
1978 fixSetRegMode(MI);
1979 if (ST.hasNeedsTDMDrain())
1980 fixTDM(MI);
1981}
1982
1983static bool isVCmpXWritesExec(const SIInstrInfo &TII, const SIRegisterInfo &TRI,
1984 const MachineInstr &MI) {
1985 return (TII.isVOPC(MI) ||
1986 (MI.isCompare() && (TII.isVOP3(MI) || TII.isSDWA(MI)))) &&
1987 MI.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &TRI);
1988}
1989
1990bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) {
1991 if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(MI: *MI))
1992 return false;
1993
1994 const SIInstrInfo *TII = ST.getInstrInfo();
1995 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1996 auto IsHazardFn = [TII, TRI](const MachineInstr &MI) {
1997 return isVCmpXWritesExec(TII: *TII, TRI: *TRI, MI);
1998 };
1999
2000 auto IsExpiredFn = [](const MachineInstr &MI, int) {
2001 unsigned Opc = MI.getOpcode();
2002 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
2003 Opc != AMDGPU::V_NOP_e32 && Opc != AMDGPU::V_NOP_e64 &&
2004 Opc != AMDGPU::V_NOP_sdwa;
2005 };
2006
2007 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2008 std::numeric_limits<int>::max())
2009 return false;
2010
2011 // V_NOP will be discarded by SQ.
2012 // Use V_MOV_B32 v?, v?. Register must be alive so use src0 of V_PERMLANE*
2013 // which is always a VGPR and available.
2014 auto *Src0 = TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src0);
2015 Register Reg = Src0->getReg();
2016 bool IsUndef = Src0->isUndef();
2017 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2018 MCID: TII->get(Opcode: AMDGPU::V_MOV_B32_e32))
2019 .addReg(RegNo: Reg, Flags: RegState::Define | getDeadRegState(B: IsUndef))
2020 .addReg(RegNo: Reg, Flags: IsUndef ? RegState::Undef : RegState::Kill);
2021
2022 return true;
2023}
2024
2025bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) {
2026 if (!ST.hasVMEMtoScalarWriteHazard())
2027 return false;
2028 assert(!ST.hasExtendedWaitCounts());
2029
2030 if (!SIInstrInfo::isSALU(MI: *MI) && !SIInstrInfo::isSMRD(MI: *MI))
2031 return false;
2032
2033 if (MI->getNumDefs() == 0)
2034 return false;
2035
2036 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2037
2038 auto IsHazardFn = [TRI, MI](const MachineInstr &I) {
2039 if (!SIInstrInfo::isVMEM(MI: I) && !SIInstrInfo::isDS(MI: I))
2040 return false;
2041
2042 for (const MachineOperand &Def : MI->defs()) {
2043 const MachineOperand *Op =
2044 I.findRegisterUseOperand(Reg: Def.getReg(), TRI, isKill: false);
2045 if (!Op)
2046 continue;
2047 return true;
2048 }
2049 return false;
2050 };
2051
2052 auto IsExpiredFn = [](const MachineInstr &MI, int) {
2053 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ||
2054 (MI.getOpcode() == AMDGPU::S_WAITCNT &&
2055 !MI.getOperand(i: 0).getImm()) ||
2056 (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2057 AMDGPU::DepCtr::decodeFieldVmVsrc(Encoded: MI.getOperand(i: 0).getImm()) == 0);
2058 };
2059
2060 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2061 std::numeric_limits<int>::max())
2062 return false;
2063
2064 const SIInstrInfo *TII = ST.getInstrInfo();
2065 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2066 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
2067 .addImm(Val: AMDGPU::DepCtr::encodeFieldVmVsrc(VmVsrc: 0, STI: ST));
2068 return true;
2069}
2070
2071bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) {
2072 if (!ST.hasSMEMtoVectorWriteHazard())
2073 return false;
2074 assert(!ST.hasExtendedWaitCounts());
2075
2076 if (!SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true))
2077 return false;
2078
2079 AMDGPU::OpName SDSTName;
2080 switch (MI->getOpcode()) {
2081 case AMDGPU::V_READLANE_B32:
2082 case AMDGPU::V_READFIRSTLANE_B32:
2083 SDSTName = AMDGPU::OpName::vdst;
2084 break;
2085 default:
2086 SDSTName = AMDGPU::OpName::sdst;
2087 break;
2088 }
2089
2090 const SIInstrInfo *TII = ST.getInstrInfo();
2091 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2092 const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(GPU: ST.getCPU());
2093 const MachineOperand *SDST = TII->getNamedOperand(MI&: *MI, OperandName: SDSTName);
2094 if (!SDST) {
2095 for (const auto &MO : MI->implicit_operands()) {
2096 if (MO.isDef() && TRI->isSGPRClass(RC: TRI->getPhysRegBaseClass(Reg: MO.getReg()))) {
2097 SDST = &MO;
2098 break;
2099 }
2100 }
2101 }
2102
2103 if (!SDST)
2104 return false;
2105
2106 const Register SDSTReg = SDST->getReg();
2107 auto IsHazardFn = [SDSTReg, TRI](const MachineInstr &I) {
2108 return SIInstrInfo::isSMRD(MI: I) && I.readsRegister(Reg: SDSTReg, TRI);
2109 };
2110
2111 auto IsExpiredFn = [TII, IV](const MachineInstr &MI, int) {
2112 if (TII->isSALU(MI)) {
2113 switch (MI.getOpcode()) {
2114 case AMDGPU::S_SETVSKIP:
2115 case AMDGPU::S_VERSION:
2116 case AMDGPU::S_WAITCNT_VSCNT:
2117 case AMDGPU::S_WAITCNT_VMCNT:
2118 case AMDGPU::S_WAITCNT_EXPCNT:
2119 // These instructions cannot not mitigate the hazard.
2120 return false;
2121 case AMDGPU::S_WAITCNT_LGKMCNT:
2122 // Reducing lgkmcnt count to 0 always mitigates the hazard.
2123 return (MI.getOperand(i: 1).getImm() == 0) &&
2124 (MI.getOperand(i: 0).getReg() == AMDGPU::SGPR_NULL);
2125 case AMDGPU::S_WAITCNT: {
2126 const int64_t Imm = MI.getOperand(i: 0).getImm();
2127 AMDGPU::Waitcnt Decoded = AMDGPU::decodeWaitcnt(Version: IV, Encoded: Imm);
2128 // DsCnt corresponds to LGKMCnt here.
2129 return Decoded.get(T: AMDGPU::DS_CNT) == 0;
2130 }
2131 default:
2132 assert((!SIInstrInfo::isWaitcnt(MI.getOpcode()) ||
2133 MI.getOpcode() == AMDGPU::S_WAIT_IDLE) &&
2134 "unexpected wait count instruction");
2135 // SOPP instructions cannot mitigate the hazard.
2136 if (TII->isSOPP(MI))
2137 return false;
2138 // At this point the SALU can be assumed to mitigate the hazard
2139 // because either:
2140 // (a) it is independent of the at risk SMEM (breaking chain),
2141 // or
2142 // (b) it is dependent on the SMEM, in which case an appropriate
2143 // s_waitcnt lgkmcnt _must_ exist between it and the at risk
2144 // SMEM instruction.
2145 return true;
2146 }
2147 }
2148 return false;
2149 };
2150
2151 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2152 std::numeric_limits<int>::max())
2153 return false;
2154
2155 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2156 MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::SGPR_NULL)
2157 .addImm(Val: 0);
2158 return true;
2159}
2160
2161bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) {
2162 if (!ST.hasVcmpxExecWARHazard())
2163 return false;
2164 assert(!ST.hasExtendedWaitCounts());
2165
2166 if (!SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true))
2167 return false;
2168
2169 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2170 if (!MI->modifiesRegister(Reg: AMDGPU::EXEC, TRI))
2171 return false;
2172
2173 auto IsHazardFn = [TRI](const MachineInstr &I) {
2174 if (SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true))
2175 return false;
2176 return I.readsRegister(Reg: AMDGPU::EXEC, TRI);
2177 };
2178
2179 const SIInstrInfo *TII = ST.getInstrInfo();
2180 auto IsExpiredFn = [TII, TRI](const MachineInstr &MI, int) {
2181 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true)) {
2182 if (TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst))
2183 return true;
2184 for (auto MO : MI.implicit_operands())
2185 if (MO.isDef() && TRI->isSGPRClass(RC: TRI->getPhysRegBaseClass(Reg: MO.getReg())))
2186 return true;
2187 }
2188 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2189 AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: MI.getOperand(i: 0).getImm()) == 0)
2190 return true;
2191 return false;
2192 };
2193
2194 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2195 std::numeric_limits<int>::max())
2196 return false;
2197
2198 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2199 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
2200 .addImm(Val: AMDGPU::DepCtr::encodeFieldSaSdst(SaSdst: 0, STI: ST));
2201 return true;
2202}
2203
2204static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF,
2205 const GCNSubtarget &ST) {
2206 if (!ST.hasLdsBranchVmemWARHazard())
2207 return false;
2208
2209 // Check if the necessary condition for the hazard is met: both LDS and VMEM
2210 // instructions need to appear in the same function.
2211 bool HasLds = false;
2212 bool HasVmem = false;
2213 for (auto &MBB : MF) {
2214 for (auto &MI : MBB) {
2215 HasLds |= SIInstrInfo::isDS(MI) || SIInstrInfo::isLDSDMA(MI);
2216 HasVmem |= SIInstrInfo::isVMEM(MI);
2217 if (HasLds && HasVmem)
2218 return true;
2219 }
2220 }
2221 return false;
2222}
2223
2224static bool isStoreCountWaitZero(const MachineInstr &I) {
2225 return I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
2226 I.getOperand(i: 0).getReg() == AMDGPU::SGPR_NULL &&
2227 !I.getOperand(i: 1).getImm();
2228}
2229
2230bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) {
2231 if (!RunLdsBranchVmemWARHazardFixup)
2232 return false;
2233
2234 assert(ST.hasLdsBranchVmemWARHazard());
2235 assert(!ST.hasExtendedWaitCounts());
2236
2237 auto IsHazardInst = [](const MachineInstr &MI) {
2238 if (SIInstrInfo::isDS(MI) || SIInstrInfo::isLDSDMA(MI))
2239 return 1;
2240 if (SIInstrInfo::isVMEM(MI))
2241 return 2;
2242 return 0;
2243 };
2244
2245 auto InstType = IsHazardInst(*MI);
2246 if (!InstType)
2247 return false;
2248
2249 auto IsExpiredFn = [&IsHazardInst](const MachineInstr &I, int) {
2250 return IsHazardInst(I) || isStoreCountWaitZero(I);
2251 };
2252
2253 auto IsHazardFn = [InstType, &IsHazardInst](const MachineInstr &I) {
2254 if (!I.isBranch())
2255 return false;
2256
2257 auto IsHazardFn = [InstType, IsHazardInst](const MachineInstr &I) {
2258 auto InstType2 = IsHazardInst(I);
2259 return InstType2 && InstType != InstType2;
2260 };
2261
2262 auto IsExpiredFn = [InstType, &IsHazardInst](const MachineInstr &I, int) {
2263 auto InstType2 = IsHazardInst(I);
2264 if (InstType == InstType2)
2265 return true;
2266
2267 return isStoreCountWaitZero(I);
2268 };
2269
2270 return ::getWaitStatesSince(IsHazard: IsHazardFn, MI: &I, IsExpired: IsExpiredFn) !=
2271 std::numeric_limits<int>::max();
2272 };
2273
2274 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2275 std::numeric_limits<int>::max())
2276 return false;
2277
2278 const SIInstrInfo *TII = ST.getInstrInfo();
2279 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2280 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_VSCNT))
2281 .addReg(RegNo: AMDGPU::SGPR_NULL, Flags: RegState::Undef)
2282 .addImm(Val: 0);
2283
2284 return true;
2285}
2286
2287bool GCNHazardRecognizer::fixLdsDirectVALUHazard(MachineInstr *MI) {
2288 if (!SIInstrInfo::isLDSDIR(MI: *MI))
2289 return false;
2290
2291 const int NoHazardWaitStates = 15;
2292 const MachineOperand *VDST = TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::vdst);
2293 const Register VDSTReg = VDST->getReg();
2294
2295 bool VisitedTrans = false;
2296 auto IsHazardFn = [this, VDSTReg, &VisitedTrans](const MachineInstr &I) {
2297 if (!SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true))
2298 return false;
2299 VisitedTrans = VisitedTrans || SIInstrInfo::isTRANS(MI: I);
2300 // Cover both WAR and WAW
2301 return I.readsRegister(Reg: VDSTReg, TRI: &TRI) || I.modifiesRegister(Reg: VDSTReg, TRI: &TRI);
2302 };
2303 auto IsExpiredFn = [&](const MachineInstr &I, int WaitStates) {
2304 if (WaitStates >= NoHazardWaitStates)
2305 return true;
2306 // Instructions which cause va_vdst==0 expire hazard
2307 return SIInstrInfo::isVMEM(MI: I) || SIInstrInfo::isDS(MI: I) ||
2308 SIInstrInfo::isEXP(MI: I);
2309 };
2310 auto GetWaitStatesFn = [](const MachineInstr &MI) {
2311 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ? 1 : 0;
2312 };
2313
2314 DenseSet<const MachineBasicBlock *> Visited;
2315 auto Count = ::getWaitStatesSince(IsHazard: IsHazardFn, MBB: MI->getParent(),
2316 I: std::next(x: MI->getReverseIterator()), WaitStates: 0,
2317 IsExpired: IsExpiredFn, Visited, GetNumWaitStates: GetWaitStatesFn);
2318
2319 // Transcendentals can execute in parallel to other VALUs.
2320 // This makes va_vdst count unusable with a mixture of VALU and TRANS.
2321 if (VisitedTrans)
2322 Count = 0;
2323
2324 MachineOperand *WaitVdstOp =
2325 TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::waitvdst);
2326 WaitVdstOp->setImm(std::min(a: Count, b: NoHazardWaitStates));
2327
2328 return true;
2329}
2330
2331bool GCNHazardRecognizer::fixLdsDirectVMEMHazard(MachineInstr *MI) {
2332 if (!SIInstrInfo::isLDSDIR(MI: *MI))
2333 return false;
2334
2335 const MachineOperand *VDST = TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::vdst);
2336 const Register VDSTReg = VDST->getReg();
2337
2338 auto IsHazardFn = [this, VDSTReg](const MachineInstr &I) {
2339 if (!SIInstrInfo::isVMEM(MI: I) && !SIInstrInfo::isDS(MI: I))
2340 return false;
2341 return I.readsRegister(Reg: VDSTReg, TRI: &TRI) || I.modifiesRegister(Reg: VDSTReg, TRI: &TRI);
2342 };
2343 bool LdsdirCanWait = ST.hasLdsWaitVMSRC();
2344 // TODO: On GFX12 the hazard should expire on S_WAIT_LOADCNT/SAMPLECNT/BVHCNT
2345 // according to the type of VMEM instruction.
2346 auto IsExpiredFn = [this, LdsdirCanWait](const MachineInstr &I, int) {
2347 return SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true) ||
2348 SIInstrInfo::isEXP(MI: I) ||
2349 (I.getOpcode() == AMDGPU::S_WAITCNT && !I.getOperand(i: 0).getImm()) ||
2350 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2351 AMDGPU::DepCtr::decodeFieldVmVsrc(Encoded: I.getOperand(i: 0).getImm()) == 0) ||
2352 (LdsdirCanWait && SIInstrInfo::isLDSDIR(MI: I) &&
2353 !TII.getNamedOperand(MI: I, OperandName: AMDGPU::OpName::waitvsrc)->getImm());
2354 };
2355
2356 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2357 std::numeric_limits<int>::max())
2358 return false;
2359
2360 if (LdsdirCanWait) {
2361 TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::waitvsrc)->setImm(0);
2362 } else {
2363 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2364 MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
2365 .addImm(Val: AMDGPU::DepCtr::encodeFieldVmVsrc(VmVsrc: 0, STI: ST));
2366 }
2367
2368 return true;
2369}
2370
2371bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(MachineInstr *MI) {
2372 if (!ST.hasVALUPartialForwardingHazard())
2373 return false;
2374 assert(!ST.hasExtendedWaitCounts());
2375
2376 if (!ST.isWave64() || !SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true))
2377 return false;
2378
2379 SmallSetVector<Register, 4> SrcVGPRs;
2380
2381 for (const MachineOperand &Use : MI->explicit_uses()) {
2382 if (Use.isReg() && TRI.isVGPR(MRI: MF.getRegInfo(), Reg: Use.getReg()))
2383 SrcVGPRs.insert(X: Use.getReg());
2384 }
2385
2386 // Only applies with >= 2 unique VGPR sources
2387 if (SrcVGPRs.size() <= 1)
2388 return false;
2389
2390 // Look for the following pattern:
2391 // Va <- VALU [PreExecPos]
2392 // intv1
2393 // Exec <- SALU [ExecPos]
2394 // intv2
2395 // Vb <- VALU [PostExecPos]
2396 // intv3
2397 // MI Va, Vb (WaitState = 0)
2398 //
2399 // Where:
2400 // intv1 + intv2 <= 2 VALUs
2401 // intv3 <= 4 VALUs
2402 //
2403 // If found, insert an appropriate S_WAITCNT_DEPCTR before MI.
2404
2405 const int Intv1plus2MaxVALUs = 2;
2406 const int Intv3MaxVALUs = 4;
2407 const int IntvMaxVALUs = 6;
2408 const int NoHazardVALUWaitStates = IntvMaxVALUs + 2;
2409
2410 struct StateType {
2411 SmallDenseMap<Register, int, 4> DefPos;
2412 int ExecPos = std::numeric_limits<int>::max();
2413 int VALUs = 0;
2414
2415 static unsigned getHashValue(const StateType &State) {
2416 hash_code H = hash_combine(args: State.ExecPos, args: State.VALUs);
2417 for (const auto &[Reg, Pos] : State.DefPos)
2418 H = hash_combine(args: H, args: Reg, args: Pos);
2419 return H;
2420 }
2421 static bool isEqual(const StateType &LHS, const StateType &RHS) {
2422 return LHS.DefPos == RHS.DefPos && LHS.ExecPos == RHS.ExecPos &&
2423 LHS.VALUs == RHS.VALUs;
2424 }
2425 };
2426
2427 StateType State;
2428
2429 // This overloads expiry testing with all the hazard detection
2430 auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) {
2431 // Too many VALU states have passed
2432 if (State.VALUs > NoHazardVALUWaitStates)
2433 return HazardExpired;
2434
2435 // Instructions which cause va_vdst==0 expire hazard
2436 if (SIInstrInfo::isVMEM(MI: I) || SIInstrInfo::isDS(MI: I) ||
2437 SIInstrInfo::isEXP(MI: I) ||
2438 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2439 AMDGPU::DepCtr::decodeFieldVaVdst(Encoded: I.getOperand(i: 0).getImm()) == 0))
2440 return HazardExpired;
2441
2442 // Track registers writes
2443 bool Changed = false;
2444 if (SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true)) {
2445 for (Register Src : SrcVGPRs) {
2446 if (!State.DefPos.count(Val: Src) && I.modifiesRegister(Reg: Src, TRI: &TRI)) {
2447 State.DefPos[Src] = State.VALUs;
2448 Changed = true;
2449 }
2450 }
2451 } else if (SIInstrInfo::isSALU(MI: I)) {
2452 if (State.ExecPos == std::numeric_limits<int>::max()) {
2453 if (!State.DefPos.empty() && I.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &TRI)) {
2454 State.ExecPos = State.VALUs;
2455 Changed = true;
2456 }
2457 }
2458 }
2459
2460 // Early expiration: too many VALUs in intv3
2461 if (State.VALUs > Intv3MaxVALUs && State.DefPos.empty())
2462 return HazardExpired;
2463
2464 // Only evaluate state if something changed
2465 if (!Changed)
2466 return NoHazardFound;
2467
2468 // Determine positions of VALUs pre/post exec change
2469 if (State.ExecPos == std::numeric_limits<int>::max())
2470 return NoHazardFound;
2471
2472 int PreExecPos = std::numeric_limits<int>::max();
2473 int PostExecPos = std::numeric_limits<int>::max();
2474
2475 for (auto Entry : State.DefPos) {
2476 int DefVALUs = Entry.second;
2477 if (DefVALUs != std::numeric_limits<int>::max()) {
2478 if (DefVALUs >= State.ExecPos)
2479 PreExecPos = std::min(a: PreExecPos, b: DefVALUs);
2480 else
2481 PostExecPos = std::min(a: PostExecPos, b: DefVALUs);
2482 }
2483 }
2484
2485 // Need a VALUs post exec change
2486 if (PostExecPos == std::numeric_limits<int>::max())
2487 return NoHazardFound;
2488
2489 // Too many VALUs in intv3?
2490 int Intv3VALUs = PostExecPos;
2491 if (Intv3VALUs > Intv3MaxVALUs)
2492 return HazardExpired;
2493
2494 // Too many VALUs in intv2?
2495 int Intv2VALUs = (State.ExecPos - PostExecPos) - 1;
2496 if (Intv2VALUs > Intv1plus2MaxVALUs)
2497 return HazardExpired;
2498
2499 // Need a VALUs pre exec change
2500 if (PreExecPos == std::numeric_limits<int>::max())
2501 return NoHazardFound;
2502
2503 // Too many VALUs in intv1?
2504 int Intv1VALUs = PreExecPos - State.ExecPos;
2505 if (Intv1VALUs > Intv1plus2MaxVALUs)
2506 return HazardExpired;
2507
2508 // Too many VALUs in intv1 + intv2
2509 if (Intv1VALUs + Intv2VALUs > Intv1plus2MaxVALUs)
2510 return HazardExpired;
2511
2512 return HazardFound;
2513 };
2514 auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) {
2515 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
2516 State.VALUs += 1;
2517 };
2518
2519 if (!hasHazard<StateType>(InitialState: State, IsHazard: IsHazardFn, UpdateState: UpdateStateFn, InitialMBB: MI->getParent(),
2520 InitialI: std::next(x: MI->getReverseIterator())))
2521 return false;
2522
2523 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2524 MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
2525 .addImm(Val: AMDGPU::DepCtr::encodeFieldVaVdst(VaVdst: 0, STI: ST));
2526
2527 return true;
2528}
2529
2530bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) {
2531 if (!ST.hasVALUTransUseHazard())
2532 return false;
2533 assert(!ST.hasExtendedWaitCounts());
2534
2535 if (!SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true))
2536 return false;
2537
2538 SmallSet<Register, 4> SrcVGPRs;
2539
2540 for (const MachineOperand &Use : MI->explicit_uses()) {
2541 if (Use.isReg() && TRI.isVGPR(MRI: MF.getRegInfo(), Reg: Use.getReg()))
2542 SrcVGPRs.insert(V: Use.getReg());
2543 }
2544
2545 // Look for the following pattern:
2546 // Va <- TRANS VALU
2547 // intv
2548 // MI Va (WaitState = 0)
2549 //
2550 // Where:
2551 // intv <= 5 VALUs / 1 TRANS
2552 //
2553 // If found, insert an appropriate S_WAITCNT_DEPCTR before MI.
2554
2555 const int IntvMaxVALUs = 5;
2556 const int IntvMaxTRANS = 1;
2557
2558 struct StateType {
2559 int VALUs = 0;
2560 int TRANS = 0;
2561
2562 static unsigned getHashValue(const StateType &State) {
2563 return hash_combine(args: State.VALUs, args: State.TRANS);
2564 }
2565 static bool isEqual(const StateType &LHS, const StateType &RHS) {
2566 return LHS.VALUs == RHS.VALUs && LHS.TRANS == RHS.TRANS;
2567 }
2568 };
2569
2570 StateType State;
2571
2572 // This overloads expiry testing with all the hazard detection
2573 auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) {
2574 // Too many VALU states have passed
2575 if (State.VALUs > IntvMaxVALUs || State.TRANS > IntvMaxTRANS)
2576 return HazardExpired;
2577
2578 // Instructions which cause va_vdst==0 expire hazard
2579 if (SIInstrInfo::isVMEM(MI: I) || SIInstrInfo::isDS(MI: I) ||
2580 SIInstrInfo::isEXP(MI: I) ||
2581 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2582 AMDGPU::DepCtr::decodeFieldVaVdst(Encoded: I.getOperand(i: 0).getImm()) == 0))
2583 return HazardExpired;
2584
2585 // Track registers writes
2586 if (SIInstrInfo::isTRANS(MI: I)) {
2587 for (Register Src : SrcVGPRs) {
2588 if (I.modifiesRegister(Reg: Src, TRI: &TRI)) {
2589 return HazardFound;
2590 }
2591 }
2592 }
2593
2594 return NoHazardFound;
2595 };
2596 auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) {
2597 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
2598 State.VALUs += 1;
2599 if (SIInstrInfo::isTRANS(MI))
2600 State.TRANS += 1;
2601 };
2602
2603 if (!hasHazard<StateType>(InitialState: State, IsHazard: IsHazardFn, UpdateState: UpdateStateFn, InitialMBB: MI->getParent(),
2604 InitialI: std::next(x: MI->getReverseIterator())))
2605 return false;
2606
2607 // Hazard is observed - insert a wait on va_dst counter to ensure hazard is
2608 // avoided.
2609 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2610 MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
2611 .addImm(Val: AMDGPU::DepCtr::encodeFieldVaVdst(VaVdst: 0, STI: ST));
2612
2613 return true;
2614}
2615
2616bool GCNHazardRecognizer::fixVALUTransCoexecutionHazards(MachineInstr *MI) {
2617 if (!ST.hasTransCoexecutionHazard() || // Coexecution disabled.
2618 !SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true) ||
2619 SIInstrInfo::isTRANS(MI: *MI))
2620 return false;
2621
2622 const SIInstrInfo *TII = ST.getInstrInfo();
2623 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2624
2625 auto IsTransHazardFn = [MI, TII, TRI](const MachineInstr &I) {
2626 if (!SIInstrInfo::isTRANS(MI: I))
2627 return false;
2628
2629 // RAW: Trans(I) writes, VALU(MI) reads.
2630 Register TransDef = TII->getNamedOperand(MI: I, OperandName: AMDGPU::OpName::vdst)->getReg();
2631 for (const MachineOperand &ValuUse : MI->explicit_uses()) {
2632 if (ValuUse.isReg() && TRI->regsOverlap(RegA: TransDef, RegB: ValuUse.getReg()))
2633 return true;
2634 }
2635
2636 auto *ValuDst = TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::vdst);
2637 if (!ValuDst || !ValuDst->isReg())
2638 return false;
2639
2640 // WAR: Trans(I) reads, VALU(MI) writes.
2641 Register ValuDef = ValuDst->getReg();
2642 for (const MachineOperand &TransUse : I.explicit_uses()) {
2643 if (TransUse.isReg() && TRI->regsOverlap(RegA: ValuDef, RegB: TransUse.getReg()))
2644 return true;
2645 }
2646
2647 return false;
2648 };
2649
2650 auto IsExpiredFn = [](const MachineInstr &I, int) {
2651 return SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true);
2652 };
2653
2654 const int HasVALU = std::numeric_limits<int>::max();
2655 if (::getWaitStatesSince(IsHazard: IsTransHazardFn, MI, IsExpired: IsExpiredFn) == HasVALU)
2656 return false;
2657
2658 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::V_NOP_e32));
2659 return true;
2660}
2661
2662bool GCNHazardRecognizer::fixWMMAHazards(MachineInstr *MI) {
2663 if (!SIInstrInfo::isWMMA(MI: *MI) && !SIInstrInfo::isSWMMAC(MI: *MI))
2664 return false;
2665
2666 const SIInstrInfo *TII = ST.getInstrInfo();
2667 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2668
2669 auto IsHazardFn = [MI, TII, TRI, this](const MachineInstr &I) {
2670 if (!SIInstrInfo::isWMMA(MI: I) && !SIInstrInfo::isSWMMAC(MI: I))
2671 return false;
2672
2673 // Src0(matrix A) or Src1(matrix B) of the current wmma instruction overlaps
2674 // with the dest(matrix D) of the previous wmma.
2675 const Register CurSrc0Reg =
2676 TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src0)->getReg();
2677 const Register CurSrc1Reg =
2678 TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src1)->getReg();
2679
2680 const Register PrevDstReg =
2681 TII->getNamedOperand(MI: I, OperandName: AMDGPU::OpName::vdst)->getReg();
2682
2683 if (TRI->regsOverlap(RegA: PrevDstReg, RegB: CurSrc0Reg) ||
2684 TRI->regsOverlap(RegA: PrevDstReg, RegB: CurSrc1Reg)) {
2685 return true;
2686 }
2687
2688 // GFX12+ allows overlap of matrix C with PrevDstReg (hardware will stall)
2689 // but Index can't overlap with PrevDstReg.
2690 if (AMDGPU::isGFX12Plus(STI: ST)) {
2691 if (SIInstrInfo::isSWMMAC(MI: *MI)) {
2692 const Register CurIndex =
2693 TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src2)->getReg();
2694 if (TRI->regsOverlap(RegA: PrevDstReg, RegB: CurIndex))
2695 return true;
2696 }
2697 return false;
2698 }
2699
2700 return false;
2701 };
2702
2703 auto IsExpiredFn = [](const MachineInstr &I, int) {
2704 return SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true);
2705 };
2706
2707 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2708 std::numeric_limits<int>::max())
2709 return false;
2710
2711 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::V_NOP_e32));
2712
2713 return true;
2714}
2715
2716static bool isCoexecutableVALUInst(const MachineInstr &MI) {
2717 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false) &&
2718 !SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI);
2719}
2720
2721// Classify XDL WMMA instructions into co-execution hazard categories
2722// (Refer to SPG 4.6.12.1), mainly based on instruction latency.
2723//
2724// Category 0: WMMA with Latency 8
2725// WMMA_*F16, WMMA_*BF16
2726// WMMA_*_16X16X128_{FP8,BF8}
2727// WMMA_*F8F6F4 if SRCA & SRCB are not both F4
2728//
2729// Category 1: WMMA Latency 16
2730// WMMA_IU8
2731//
2732// Category 2: SWMMAC with Latency 8
2733// SWMMAC_*F16, SWMMAC_*BF16,
2734// SWMMAC_*FP8FP8
2735// SWMMAC_*BF8FP8
2736// SWMMAC_*FP8BF8
2737// SWMMAC_*BF8BF8
2738//
2739// Category 3: SWMMAC with Latency 16
2740// SWMMAC_IU8
2741//
2742// Category 4: 16 Pass GFX1251 WMMA with latency 16
2743// V_WMMA_*_16X16X32_{F16,BF16}
2744// V_WMMA_{F32,F16}_16X16X64_{FP8,BF8}*
2745// V_WMMA_F32_16x16x128_F8F6F4 (F4 only)
2746// V_SWMMAC_*_16X16X64_{F16,BF16}
2747// V_SWMMAC_{F32,F16}_16X16X128_{FP8,BF8}*
2748//
2749// Category 5: 32 Pass GFX1251 WMMA with latency 32
2750// V_WMMA_F32_16x16x128_F8F6F4 (not all F4)
2751// V_WMMA_{F32,F16}_16X16X128_{FP8,BF8}*
2752// V_WMMA_F32_32X16X128_F4
2753// V_WMMA_I32_16X16X64_IU8
2754// V_WMMA_I32_16X16X64_IU8
2755//
2756// Category 6: gfx1250 WMMA with Latency 4 (one co-execution slot)
2757// WMMA_*_16X16X64_{FP8,BF8}
2758// WMMA_*F8F6F4 if SRCA & SRCB are both F4
2759static unsigned getWMMAHazardInstInCategory(const MachineInstr &MI,
2760 const SIInstrInfo *TII,
2761 const TargetSchedModel &SchedModel,
2762 const GCNSubtarget &ST) {
2763 assert(TII->isXDLWMMA(MI) && "must be xdl wmma");
2764 bool IsSWMMAC = SIInstrInfo::isSWMMAC(MI);
2765 bool IsLowestRateWMMA = ST.hasGFX125xLowestRateWMMA();
2766 unsigned Category = 0;
2767
2768 unsigned Latency = SchedModel.computeInstrLatency(MI: &MI);
2769 switch (Latency) {
2770 case 4:
2771 // Dense 4-cycle WMMA (gfx1250 16x16x64 FP8/BF8 and f8f6f4 with both
2772 // inputs F4). One co-execution slot; there is no 4-cycle SWMMAC.
2773 assert(!IsSWMMAC && "no 4-cycle SWMMAC expected");
2774 Category = 6;
2775 break;
2776 case 8:
2777 Category = IsSWMMAC ? 2 : 0;
2778 break;
2779 case 16:
2780 Category = IsLowestRateWMMA ? 4 : (IsSWMMAC ? 3 : 1);
2781 break;
2782 case 32:
2783 assert(IsLowestRateWMMA && "latency 32 is not expected");
2784 Category = 5;
2785 break;
2786 default:
2787 llvm_unreachable("unexpected xdl wmma latency");
2788 } // end switch.
2789
2790 return Category;
2791}
2792
2793int GCNHazardRecognizer::checkWMMACoexecutionHazards(MachineInstr *MI) const {
2794 if (!ST.hasWMMACoexecutionHazards())
2795 return 0;
2796
2797 const SIInstrInfo *TII = ST.getInstrInfo();
2798 if (!TII->isXDLWMMA(MI: *MI) && !isCoexecutableVALUInst(MI: *MI))
2799 return 0;
2800
2801 // WaitStates here is the number of V_NOPs or unrelated VALU instructions must
2802 // be in between the first WMMA and the second instruction to cover the hazard
2803 // (WMMAWaitStates if the second is also a WMMA, VALUWaitStates if the second
2804 // is a VALU). Refer to SPG 4.6.12.1. "Requirements for WMMA data hazards" for
2805 // numbers, which depends on the category of the first WMMA.
2806 const int WMMAWaitStates[] = {5, 9, 3, 5, 9, 17, 2};
2807 const int VALUWaitStates[] = {4, 8, 2, 4, 8, 16, 1};
2808 const bool IsWMMA = TII->isXDLWMMA(MI: *MI);
2809 const bool IsLowestRateWMMA = ST.hasGFX125xLowestRateWMMA();
2810 ArrayRef<int> RequiredWaitStates =
2811 IsWMMA ? ArrayRef<int>(WMMAWaitStates) : ArrayRef<int>(VALUWaitStates);
2812 const int MaxWaitStates =
2813 IsWMMA ? (IsLowestRateWMMA ? 17 : 9) : (IsLowestRateWMMA ? 16 : 8);
2814
2815 auto WindowFor = [=](const MachineInstr &I) -> std::optional<int> {
2816 if (!TII->isXDLWMMA(MI: I))
2817 return std::nullopt;
2818 if (IsWMMA ? !hasWMMAToWMMARegOverlap(WMMA: I, MI: *MI)
2819 : !hasWMMAToVALURegOverlap(WMMA: I, MI: *MI))
2820 return std::nullopt;
2821 return RequiredWaitStates[getWMMAHazardInstInCategory(MI: I, TII, SchedModel: TSchedModel,
2822 ST)];
2823 };
2824
2825 return getMaxVALUWindowDeficit(MaxWindow: MaxWaitStates, WindowFor);
2826}
2827
2828bool GCNHazardRecognizer::hasWMMAToWMMARegOverlap(
2829 const MachineInstr &WMMA, const MachineInstr &MI) const {
2830 Register D0 = TII.getNamedOperand(MI: WMMA, OperandName: AMDGPU::OpName::vdst)->getReg();
2831 Register A1 = TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src0)->getReg();
2832 Register B1 = TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src1)->getReg();
2833
2834 // WMMA0 writes (D0), WMMA1 reads (A1/B1/Idx1).
2835 if (TRI.regsOverlap(RegA: D0, RegB: A1) || TRI.regsOverlap(RegA: D0, RegB: B1))
2836 return true;
2837
2838 if (SIInstrInfo::isSWMMAC(MI)) {
2839 Register Idx1 = TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src2)->getReg();
2840 if (TRI.regsOverlap(RegA: D0, RegB: Idx1))
2841 return true;
2842 }
2843 return false;
2844}
2845
2846bool GCNHazardRecognizer::hasWMMAToVALURegOverlap(
2847 const MachineInstr &WMMA, const MachineInstr &MI) const {
2848 // WMMA writes, VALU reads.
2849 Register D0 = TII.getNamedOperand(MI: WMMA, OperandName: AMDGPU::OpName::vdst)->getReg();
2850 for (const MachineOperand &ValuUse : MI.explicit_uses()) {
2851 if (ValuUse.isReg() && TRI.regsOverlap(RegA: D0, RegB: ValuUse.getReg()))
2852 return true;
2853 }
2854
2855 // WMMA reads or writes, VALU writes.
2856 Register A0 = TII.getNamedOperand(MI: WMMA, OperandName: AMDGPU::OpName::src0)->getReg();
2857 Register B0 = TII.getNamedOperand(MI: WMMA, OperandName: AMDGPU::OpName::src1)->getReg();
2858 SmallVector<Register, 4> WMMARegs({D0, A0, B0});
2859
2860 const MachineOperand *Src2 = TII.getNamedOperand(MI: WMMA, OperandName: AMDGPU::OpName::src2);
2861 if (Src2->isReg())
2862 WMMARegs.push_back(Elt: Src2->getReg());
2863
2864 for (const MachineOperand &ValuDef : MI.defs()) {
2865 Register VDstReg = ValuDef.getReg();
2866 for (Register WMMAReg : WMMARegs) {
2867 if (TRI.regsOverlap(RegA: VDstReg, RegB: WMMAReg))
2868 return true;
2869 }
2870 }
2871 return false;
2872}
2873
2874bool GCNHazardRecognizer::isCoexecutionHazardFor(const MachineInstr &I,
2875 const MachineInstr &MI) const {
2876 // I is the potential WMMA hazard source, MI is the instruction being checked
2877 // for hazard.
2878 if (!TII.isXDLWMMA(MI: I))
2879 return false;
2880
2881 // Dispatch based on MI type
2882 if (TII.isXDLWMMA(MI))
2883 return hasWMMAToWMMARegOverlap(WMMA: I, MI);
2884 if (isCoexecutableVALUInst(MI))
2885 return hasWMMAToVALURegOverlap(WMMA: I, MI);
2886
2887 return false;
2888}
2889
2890bool GCNHazardRecognizer::hasWMMAHazardInLoop(MachineLoop *L, MachineInstr *MI,
2891 bool IncludeSubloops) {
2892 // Scan loop for any WMMA that hazards MI.
2893 // TODO: Avoid full loop scan when WMMA is beyond VALU distance.
2894 for (MachineBasicBlock *MBB : L->getBlocks()) {
2895 if (!IncludeSubloops && MLI->getLoopFor(BB: MBB) != L)
2896 continue;
2897 for (MachineInstr &I : *MBB) {
2898 if (&I == MI)
2899 continue;
2900 if (isCoexecutionHazardFor(I, MI: *MI))
2901 return true;
2902 }
2903 }
2904 return false;
2905}
2906
2907bool GCNHazardRecognizer::tryHoistWMMAVnopsFromLoop(MachineInstr *MI,
2908 int WaitStatesNeeded) {
2909 if (!MLI)
2910 return false;
2911
2912 MachineLoop *L = MLI->getLoopFor(BB: MI->getParent());
2913 if (!L) {
2914 ++NumWMMAHoistingBailed;
2915 return false;
2916 }
2917
2918 // If innermost loop has WMMA hazard, we can't hoist at all
2919 if (hasWMMAHazardInLoop(L, MI)) {
2920 ++NumWMMAHoistingBailed;
2921 return false;
2922 }
2923
2924 // Find outermost loop with no internal hazard
2925 MachineLoop *TargetLoop = L;
2926 while (MachineLoop *Parent = TargetLoop->getParentLoop()) {
2927 if (hasWMMAHazardInLoop(L: Parent, MI, IncludeSubloops: false))
2928 break; // Parent has hazard in its own blocks, stop here
2929 TargetLoop = Parent; // Safe to hoist further out
2930 }
2931
2932 // Need valid preheader to insert V_NOPs
2933 MachineBasicBlock *Preheader = TargetLoop->getLoopPreheader();
2934 if (!Preheader) {
2935 ++NumWMMAHoistingBailed;
2936 return false;
2937 }
2938
2939 LLVM_DEBUG(dbgs() << "WMMA V_NOP Hoisting: Moving " << WaitStatesNeeded
2940 << " V_NOPs from loop to " << printMBBReference(*Preheader)
2941 << "\n");
2942
2943 emitVNops(MBB&: *Preheader, InsertPt: Preheader->getFirstTerminator(), WaitStatesNeeded,
2944 /*IsHoisting=*/true);
2945 NumWMMANopsHoisted += WaitStatesNeeded;
2946 return true;
2947}
2948
2949bool GCNHazardRecognizer::fixWMMACoexecutionHazards(MachineInstr *MI) {
2950 int WaitStatesNeeded = checkWMMACoexecutionHazards(MI);
2951 if (WaitStatesNeeded <= 0)
2952 return false;
2953
2954 if (EnableWMMAVnopHoisting && tryHoistWMMAVnopsFromLoop(MI, WaitStatesNeeded))
2955 return true;
2956
2957 emitVNops(MBB&: *MI->getParent(), InsertPt: MI->getIterator(), WaitStatesNeeded);
2958 return true;
2959}
2960
2961bool GCNHazardRecognizer::fixShift64HighRegBug(MachineInstr *MI) {
2962 if (!ST.hasShift64HighRegBug())
2963 return false;
2964 assert(!ST.hasExtendedWaitCounts());
2965
2966 switch (MI->getOpcode()) {
2967 default:
2968 return false;
2969 case AMDGPU::V_LSHLREV_B64_e64:
2970 case AMDGPU::V_LSHRREV_B64_e64:
2971 case AMDGPU::V_ASHRREV_I64_e64:
2972 break;
2973 }
2974
2975 MachineOperand *Amt = TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src0);
2976 if (!Amt->isReg())
2977 return false;
2978
2979 Register AmtReg = Amt->getReg();
2980 const MachineRegisterInfo &MRI = MF.getRegInfo();
2981 // Check if this is a last VGPR in the allocation block.
2982 if (!TRI.isVGPR(MRI, Reg: AmtReg) || ((AmtReg - AMDGPU::VGPR0) & 7) != 7)
2983 return false;
2984
2985 if (AmtReg != AMDGPU::VGPR255 && MRI.isPhysRegUsed(PhysReg: AmtReg + 1))
2986 return false;
2987
2988 assert(ST.needsAlignedVGPRs());
2989 static_assert(AMDGPU::VGPR0 + 1 == AMDGPU::VGPR1);
2990
2991 const DebugLoc &DL = MI->getDebugLoc();
2992 MachineBasicBlock *MBB = MI->getParent();
2993 MachineOperand *Src1 = TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src1);
2994
2995 // In:
2996 //
2997 // Dst = shiftrev64 Amt, Src1
2998 //
2999 // if Dst!=Src1 then avoid the bug with:
3000 //
3001 // Dst.sub0 = Amt
3002 // Dst = shift64 Dst.sub0, Src1
3003
3004 Register DstReg = MI->getOperand(i: 0).getReg();
3005 if (!Src1->isReg() || Src1->getReg() != DstReg) {
3006 Register DstLo = TRI.getSubReg(Reg: DstReg, Idx: AMDGPU::sub0);
3007 runOnInstruction(
3008 MI: BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstLo).add(MO: *Amt));
3009 Amt->setReg(DstLo);
3010 Amt->setIsKill(true);
3011 return true;
3012 }
3013
3014 bool Overlapped = MI->modifiesRegister(Reg: AmtReg, TRI: &TRI);
3015 Register NewReg;
3016 for (MCRegister Reg : Overlapped ? AMDGPU::VReg_64_Align2RegClass
3017 : AMDGPU::VGPR_32RegClass) {
3018 if (!MI->modifiesRegister(Reg, TRI: &TRI) && !MI->readsRegister(Reg, TRI: &TRI)) {
3019 NewReg = Reg;
3020 break;
3021 }
3022 }
3023
3024 Register NewAmt = Overlapped ? (Register)TRI.getSubReg(Reg: NewReg, Idx: AMDGPU::sub1)
3025 : NewReg;
3026 Register NewAmtLo;
3027
3028 if (Overlapped)
3029 NewAmtLo = TRI.getSubReg(Reg: NewReg, Idx: AMDGPU::sub0);
3030
3031 // Insert a full wait count because found register might be pending a wait.
3032 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT))
3033 .addImm(Val: 0);
3034
3035 // Insert V_SWAP_B32 instruction(s) and run hazard recognizer on them.
3036 if (Overlapped)
3037 runOnInstruction(
3038 MI: BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SWAP_B32), DestReg: NewAmtLo)
3039 .addDef(RegNo: AmtReg - 1)
3040 .addReg(RegNo: AmtReg - 1, Flags: RegState::Undef)
3041 .addReg(RegNo: NewAmtLo, Flags: RegState::Undef));
3042 runOnInstruction(MI: BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SWAP_B32), DestReg: NewAmt)
3043 .addDef(RegNo: AmtReg)
3044 .addReg(RegNo: AmtReg, Flags: RegState::Undef)
3045 .addReg(RegNo: NewAmt, Flags: RegState::Undef));
3046
3047 // Instructions emitted after the current instruction will be processed by the
3048 // parent loop of the hazard recognizer in a natural way.
3049 BuildMI(BB&: *MBB, I: std::next(x: MI->getIterator()), MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SWAP_B32),
3050 DestReg: AmtReg)
3051 .addDef(RegNo: NewAmt)
3052 .addReg(RegNo: NewAmt)
3053 .addReg(RegNo: AmtReg);
3054 if (Overlapped)
3055 BuildMI(BB&: *MBB, I: std::next(x: MI->getIterator()), MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SWAP_B32),
3056 DestReg: AmtReg - 1)
3057 .addDef(RegNo: NewAmtLo)
3058 .addReg(RegNo: NewAmtLo)
3059 .addReg(RegNo: AmtReg - 1);
3060
3061 // Re-running hazard recognizer on the modified instruction is not necessary,
3062 // inserted V_SWAP_B32 has already both read and write new registers so
3063 // hazards related to these register has already been handled.
3064 Amt->setReg(NewAmt);
3065 Amt->setIsKill(false);
3066 // We do not update liveness, so verifier may see it as undef.
3067 Amt->setIsUndef();
3068 if (Overlapped) {
3069 MI->getOperand(i: 0).setReg(NewReg);
3070 Src1->setReg(NewReg);
3071 Src1->setIsKill(false);
3072 Src1->setIsUndef();
3073 }
3074
3075 return true;
3076}
3077
3078int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) const {
3079 int NSAtoVMEMWaitStates = 1;
3080
3081 if (!ST.hasNSAtoVMEMBug())
3082 return 0;
3083
3084 if (!SIInstrInfo::isMUBUF(MI: *MI) && !SIInstrInfo::isMTBUF(MI: *MI))
3085 return 0;
3086
3087 const SIInstrInfo *TII = ST.getInstrInfo();
3088 const auto *Offset = TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::offset);
3089 if (!Offset || (Offset->getImm() & 6) == 0)
3090 return 0;
3091
3092 auto IsHazardFn = [TII](const MachineInstr &I) {
3093 if (!SIInstrInfo::isMIMG(MI: I))
3094 return false;
3095 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc: I.getOpcode());
3096 return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA &&
3097 TII->getInstSizeInBytes(MI: I) >= 16;
3098 };
3099
3100 return NSAtoVMEMWaitStates - getWaitStatesSince(IsHazard: IsHazardFn, Limit: 1);
3101}
3102
3103int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(
3104 MachineInstr *MI) const {
3105 int FPAtomicToDenormModeWaitStates = 3;
3106
3107 if (!ST.hasFPAtomicToDenormModeHazard())
3108 return 0;
3109 assert(!ST.hasExtendedWaitCounts());
3110
3111 if (MI->getOpcode() != AMDGPU::S_DENORM_MODE)
3112 return 0;
3113
3114 auto IsHazardFn = [](const MachineInstr &I) {
3115 if (!SIInstrInfo::isVMEM(MI: I))
3116 return false;
3117 return SIInstrInfo::isFPAtomic(MI: I);
3118 };
3119
3120 auto IsExpiredFn = [](const MachineInstr &MI, int WaitStates) {
3121 if (WaitStates >= 3 || SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
3122 return true;
3123
3124 return SIInstrInfo::isWaitcnt(Opcode: MI.getOpcode());
3125 };
3126
3127 return FPAtomicToDenormModeWaitStates -
3128 ::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn);
3129}
3130
3131int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) const {
3132 assert(SIInstrInfo::isMAI(*MI));
3133
3134 return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI);
3135}
3136
3137int GCNHazardRecognizer::checkMFMAPadding(MachineInstr *MI) const {
3138 // Early exit if no padding is requested.
3139 if (MFMAPaddingRatio == 0)
3140 return 0;
3141
3142 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
3143 if (!SIInstrInfo::isMFMA(MI: *MI) || MFI->getOccupancy() < 2)
3144 return 0;
3145
3146 int NeighborMFMALatency = 0;
3147 auto IsNeighboringMFMA = [&NeighborMFMALatency,
3148 this](const MachineInstr &MI) {
3149 if (!SIInstrInfo::isMFMA(MI))
3150 return false;
3151
3152 NeighborMFMALatency = this->getMFMAPipelineWaitStates(MI);
3153 return true;
3154 };
3155
3156 const int MaxMFMAPipelineWaitStates = 16;
3157 int WaitStatesSinceNeighborMFMA =
3158 getWaitStatesSince(IsHazard: IsNeighboringMFMA, Limit: MaxMFMAPipelineWaitStates);
3159
3160 int NeighborMFMAPaddingNeeded =
3161 (NeighborMFMALatency * MFMAPaddingRatio / 100) -
3162 WaitStatesSinceNeighborMFMA;
3163
3164 return std::max(a: 0, b: NeighborMFMAPaddingNeeded);
3165}
3166
3167int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) const {
3168 int WaitStatesNeeded = 0;
3169 unsigned Opc = MI->getOpcode();
3170
3171 auto IsVALUFn = [](const MachineInstr &MI) {
3172 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) || MI.isInlineAsm();
3173 };
3174
3175 if (Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) { // MFMA or v_accvgpr_write
3176 const int LegacyVALUWritesVGPRWaitStates = 2;
3177 const int VALUWritesExecWaitStates = 4;
3178 const int MaxWaitStates = 4;
3179
3180 int WaitStatesNeededForUse = VALUWritesExecWaitStates -
3181 getWaitStatesSinceDef(Reg: AMDGPU::EXEC, IsHazardDef: IsVALUFn, Limit: MaxWaitStates);
3182 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3183
3184 if (WaitStatesNeeded < MaxWaitStates) {
3185 for (const MachineOperand &Use : MI->explicit_uses()) {
3186 const int MaxWaitStates = 2;
3187
3188 if (!Use.isReg() || !TRI.isVGPR(MRI: MF.getRegInfo(), Reg: Use.getReg()))
3189 continue;
3190
3191 int WaitStatesNeededForUse = LegacyVALUWritesVGPRWaitStates -
3192 getWaitStatesSinceDef(Reg: Use.getReg(), IsHazardDef: IsVALUFn, Limit: MaxWaitStates);
3193 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3194
3195 if (WaitStatesNeeded == MaxWaitStates)
3196 break;
3197 }
3198 }
3199 }
3200
3201 for (const MachineOperand &Op : MI->explicit_operands()) {
3202 if (!Op.isReg() || !TRI.isAGPR(MRI: MF.getRegInfo(), Reg: Op.getReg()))
3203 continue;
3204
3205 if (Op.isDef() && Opc != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
3206 continue;
3207
3208 const int MFMAWritesAGPROverlappedSrcABWaitStates = 4;
3209 const int MFMAWritesAGPROverlappedSrcCWaitStates = 2;
3210 const int MFMA4x4WritesAGPRAccVgprReadWaitStates = 4;
3211 const int MFMA16x16WritesAGPRAccVgprReadWaitStates = 10;
3212 const int MFMA32x32WritesAGPRAccVgprReadWaitStates = 18;
3213 const int MFMA4x4WritesAGPRAccVgprWriteWaitStates = 1;
3214 const int MFMA16x16WritesAGPRAccVgprWriteWaitStates = 7;
3215 const int MFMA32x32WritesAGPRAccVgprWriteWaitStates = 15;
3216 const int MaxWaitStates = 18;
3217 Register Reg = Op.getReg();
3218 unsigned HazardDefLatency = 0;
3219
3220 auto IsOverlappedMFMAFn = [Reg, &HazardDefLatency,
3221 this](const MachineInstr &MI) {
3222 if (!SIInstrInfo::isMFMA(MI))
3223 return false;
3224 Register DstReg = MI.getOperand(i: 0).getReg();
3225 if (DstReg == Reg)
3226 return false;
3227 HazardDefLatency =
3228 std::max(a: HazardDefLatency, b: TSchedModel.computeInstrLatency(MI: &MI));
3229 return TRI.regsOverlap(RegA: DstReg, RegB: Reg);
3230 };
3231
3232 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsHazardDef: IsOverlappedMFMAFn,
3233 Limit: MaxWaitStates);
3234 int NeedWaitStates = MFMAWritesAGPROverlappedSrcABWaitStates;
3235 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2);
3236 int OpNo = Op.getOperandNo();
3237 if (OpNo == SrcCIdx) {
3238 NeedWaitStates = MFMAWritesAGPROverlappedSrcCWaitStates;
3239 } else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) {
3240 switch (HazardDefLatency) {
3241 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprReadWaitStates;
3242 break;
3243 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprReadWaitStates;
3244 break;
3245 case 16: [[fallthrough]];
3246 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprReadWaitStates;
3247 break;
3248 }
3249 } else if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
3250 switch (HazardDefLatency) {
3251 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprWriteWaitStates;
3252 break;
3253 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprWriteWaitStates;
3254 break;
3255 case 16: [[fallthrough]];
3256 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprWriteWaitStates;
3257 break;
3258 }
3259 }
3260
3261 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
3262 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3263
3264 if (WaitStatesNeeded == MaxWaitStates)
3265 return WaitStatesNeeded; // Early exit.
3266
3267 auto IsAccVgprWriteFn = [Reg, this](const MachineInstr &MI) {
3268 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
3269 return false;
3270 Register DstReg = MI.getOperand(i: 0).getReg();
3271 return TRI.regsOverlap(RegA: Reg, RegB: DstReg);
3272 };
3273
3274 const int AccVGPRWriteMFMAReadSrcCWaitStates = 1;
3275 const int AccVGPRWriteMFMAReadSrcABWaitStates = 3;
3276 const int AccVGPRWriteAccVgprReadWaitStates = 3;
3277 NeedWaitStates = AccVGPRWriteMFMAReadSrcABWaitStates;
3278 if (OpNo == SrcCIdx)
3279 NeedWaitStates = AccVGPRWriteMFMAReadSrcCWaitStates;
3280 else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64)
3281 NeedWaitStates = AccVGPRWriteAccVgprReadWaitStates;
3282
3283 WaitStatesNeededForUse = NeedWaitStates -
3284 getWaitStatesSinceDef(Reg, IsHazardDef: IsAccVgprWriteFn, Limit: MaxWaitStates);
3285 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3286
3287 if (WaitStatesNeeded == MaxWaitStates)
3288 return WaitStatesNeeded; // Early exit.
3289 }
3290
3291 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
3292 const int MFMA4x4ReadSrcCAccVgprWriteWaitStates = 0;
3293 const int MFMA16x16ReadSrcCAccVgprWriteWaitStates = 5;
3294 const int MFMA32x32ReadSrcCAccVgprWriteWaitStates = 13;
3295 const int MaxWaitStates = 13;
3296 Register DstReg = MI->getOperand(i: 0).getReg();
3297 unsigned HazardDefLatency = 0;
3298
3299 auto IsSrcCMFMAFn = [DstReg, &HazardDefLatency,
3300 this](const MachineInstr &MI) {
3301 if (!SIInstrInfo::isMFMA(MI))
3302 return false;
3303 Register Reg = TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src2)->getReg();
3304 HazardDefLatency =
3305 std::max(a: HazardDefLatency, b: TSchedModel.computeInstrLatency(MI: &MI));
3306 return TRI.regsOverlap(RegA: Reg, RegB: DstReg);
3307 };
3308
3309 int WaitStatesSince = getWaitStatesSince(IsHazard: IsSrcCMFMAFn, Limit: MaxWaitStates);
3310 int NeedWaitStates;
3311 switch (HazardDefLatency) {
3312 case 2: NeedWaitStates = MFMA4x4ReadSrcCAccVgprWriteWaitStates;
3313 break;
3314 case 8: NeedWaitStates = MFMA16x16ReadSrcCAccVgprWriteWaitStates;
3315 break;
3316 case 16: [[fallthrough]];
3317 default: NeedWaitStates = MFMA32x32ReadSrcCAccVgprWriteWaitStates;
3318 break;
3319 }
3320
3321 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSince;
3322 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3323 }
3324
3325 // Pad neighboring MFMA with noops for better inter-wave performance.
3326 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: checkMFMAPadding(MI));
3327
3328 return WaitStatesNeeded;
3329}
3330
3331/// One MFMA can be written with up to four opcodes that differ only in how vdst
3332/// and src2 are encoded: both are either AGPRs or VGPRs, and the mac form ties
3333/// vdst to src2 instead of taking them as separate operands. \returns the AGPR,
3334/// non-mac opcode, so that every form of the same MFMA maps to one value.
3335static unsigned getMFMANonMacAGPRFormOp(unsigned Opc) {
3336 if (int NonMacOp = AMDGPU::getMFMAEarlyClobberOp(Opcode: Opc); NonMacOp != -1)
3337 Opc = NonMacOp;
3338 if (int AGPROp = AMDGPU::getAGPRFormOp(Opcode: Opc); AGPROp != -1)
3339 Opc = AGPROp;
3340 return Opc;
3341}
3342
3343/// \returns true if \p Opc0 and \p Opc1 are the same MFMA, ignoring the mac
3344/// form and whether vdst/src2 are AGPRs or VGPRs.
3345static bool isSameMFMA(unsigned Opc0, unsigned Opc1) {
3346 return getMFMANonMacAGPRFormOp(Opc: Opc0) == getMFMANonMacAGPRFormOp(Opc: Opc1);
3347}
3348
3349static int
3350GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates(int NumPasses,
3351 bool IsGFX950) {
3352 // xdl def cycles | gfx940 | gfx950
3353 // 2 pass | 3 4
3354 // 4 pass | 5 6
3355 // 8 pass | 9 10
3356 // 16 pass | 17 18
3357 return NumPasses + 1 + IsGFX950;
3358}
3359
3360static int
3361GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates(int NumPasses,
3362 bool IsGFX950) {
3363 // xdl def cycles | gfx940 | gfx950
3364 // 2 pass | 3 3
3365 // 4 pass | 5 6
3366 // 8 pass | 9 10
3367 // 16 pass | 17 18
3368 return NumPasses + 1 + (NumPasses != 2 && IsGFX950);
3369}
3370
3371static int
3372GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates(int NumPasses) {
3373 // 2 pass -> 2
3374 // 4 pass -> 4
3375 // 8 pass -> 8
3376 // 16 pass -> 16
3377 return NumPasses;
3378}
3379
3380static int
3381GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses) {
3382 // 2 pass -> 4
3383 // 4 pass -> 6
3384 // 8 pass -> 10
3385 // 16 pass -> 18
3386 return NumPasses + 2;
3387}
3388
3389static int GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses,
3390 bool IsGFX950) {
3391 // xdl def cycles | gfx942 | gfx950
3392 // 2 pass | 5 5
3393 // 4 pass | 7 8
3394 // 8 pass | 11 12
3395 // 16 pass | 19 20
3396 return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
3397}
3398
3399int GCNHazardRecognizer::getMFMAOverlappedSrcCWaitStates(
3400 const MachineInstr *Reader, const MachineInstr *Writer) const {
3401 constexpr int SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates = 2;
3402 constexpr int SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates = 8;
3403 constexpr int SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates = 16;
3404 constexpr int SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates = 3;
3405 constexpr int SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates = 9;
3406 constexpr int SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates = 17;
3407 constexpr int DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 9;
3408 constexpr int GFX950_DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 17;
3409 constexpr int DMFMA4x4WritesVGPROverlappedSrcCWaitStates = 4;
3410
3411 // An XDL read of a non-XDL result needs no wait states. DGEMM is never XDL,
3412 // so this also covers the f64 writers handled below.
3413 if (TII.isXDL(MI: *Reader) && !TII.isXDL(MI: *Writer))
3414 return 0;
3415
3416 switch (Writer->getOpcode()) {
3417 case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
3418 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
3419 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
3420 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
3421 return ST.hasGFX950Insts()
3422 ? GFX950_DMFMA16x16WritesVGPROverlappedSrcCWaitStates
3423 : DMFMA16x16WritesVGPROverlappedSrcCWaitStates;
3424 case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
3425 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
3426 return DMFMA4x4WritesVGPROverlappedSrcCWaitStates;
3427 default:
3428 break;
3429 }
3430
3431 int NumPasses = TSchedModel.computeInstrLatency(MI: Writer);
3432 if (ST.hasGFX940Insts()) {
3433 if (!TII.isXDL(MI: *Writer))
3434 return GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates(
3435 NumPasses);
3436 return TII.isXDL(MI: *Reader)
3437 ? GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates(
3438 NumPasses, IsGFX950: ST.hasGFX950Insts())
3439 : GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates(
3440 NumPasses, IsGFX950: ST.hasGFX950Insts());
3441 }
3442
3443 bool IsDGEMM = SIInstrInfo::isDGEMM(Opcode: Reader->getOpcode());
3444 switch (NumPasses) {
3445 case 2:
3446 return IsDGEMM ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates
3447 : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates;
3448 case 8:
3449 return IsDGEMM ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates
3450 : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates;
3451 case 16:
3452 return IsDGEMM ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates
3453 : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates;
3454 default:
3455 llvm_unreachable("unexpected number of passes");
3456 }
3457}
3458
3459int GCNHazardRecognizer::getMFMAReadWaitStates(const MachineInstr &Consumer,
3460 const MachineInstr &Producer,
3461 Register Reg,
3462 bool IsSrcC) const {
3463 unsigned Opc = Consumer.getOpcode();
3464 const int SMFMA4x4WritesVGPROverlappedSrcABWaitStates = 5;
3465 const int SMFMA16x16WritesVGPROverlappedSrcABWaitStates = 11;
3466 const int SMFMA32x32WritesVGPROverlappedSrcABWaitStates = 19;
3467 const int DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates = 6;
3468 const int DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 11;
3469 const int GFX950_DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 19;
3470 const int DMFMA4x4WritesVGPRFullSrcCWaitStates = 4;
3471 const int GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates = 2;
3472
3473 bool FullReg = Producer.getOperand(i: 0).getReg() == Reg;
3474 unsigned Opc1 = Producer.getOpcode();
3475 int NeedWaitStates = 0;
3476 if (IsSrcC) {
3477 if (!SIInstrInfo::isDGEMM(Opcode: Opc) &&
3478 (!ST.hasGFX940Insts() && SIInstrInfo::isDGEMM(Opcode: Opc1))) {
3479 NeedWaitStates = 0;
3480 } else if (FullReg) {
3481 if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
3482 Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) &&
3483 (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
3484 Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64))
3485 NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates;
3486 else if (ST.hasGFX940Insts() &&
3487 TSchedModel.computeInstrLatency(MI: &Producer) == 2)
3488 NeedWaitStates = GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates;
3489
3490 // The accumulator forwarding path that allows zero wait states is only
3491 // available while the chain stays on a single MFMA. Two different MFMAs
3492 // sharing an accumulator need the wait states of a partial overlap.
3493 if (ST.hasGFX940Insts() && !isSameMFMA(Opc0: Opc, Opc1)) {
3494 NeedWaitStates =
3495 std::max(a: NeedWaitStates,
3496 b: getMFMAOverlappedSrcCWaitStates(Reader: &Consumer, Writer: &Producer));
3497 }
3498 } else {
3499 NeedWaitStates = getMFMAOverlappedSrcCWaitStates(Reader: &Consumer, Writer: &Producer);
3500 }
3501 } else {
3502 switch (Opc1) {
3503 case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
3504 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
3505 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
3506 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
3507 NeedWaitStates =
3508 ST.hasGFX950Insts()
3509 ? GFX950_DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates
3510 : DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates;
3511 break;
3512 case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
3513 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
3514 NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates;
3515 break;
3516 default:
3517 int NumPasses = TSchedModel.computeInstrLatency(MI: &Producer);
3518
3519 if (ST.hasGFX940Insts()) {
3520 NeedWaitStates =
3521 TII.isXDL(MI: Producer)
3522 ? GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(
3523 NumPasses, IsGFX950: ST.hasGFX950Insts())
3524 : GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates(
3525 NumPasses);
3526 break;
3527 }
3528
3529 switch (NumPasses) {
3530 case 2:
3531 NeedWaitStates = SMFMA4x4WritesVGPROverlappedSrcABWaitStates;
3532 break;
3533 case 4:
3534 llvm_unreachable("unexpected number of passes for mfma");
3535 case 8:
3536 NeedWaitStates = SMFMA16x16WritesVGPROverlappedSrcABWaitStates;
3537 break;
3538 case 16:
3539 default:
3540 NeedWaitStates = SMFMA32x32WritesVGPROverlappedSrcABWaitStates;
3541 }
3542 }
3543 }
3544 return NeedWaitStates;
3545}
3546
3547int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) const {
3548 int WaitStatesNeeded = 0;
3549 unsigned Opc = MI->getOpcode();
3550
3551 auto IsLegacyVALUFn = [](const MachineInstr &MI) {
3552 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
3553 !SIInstrInfo::isMFMA(MI);
3554 };
3555
3556 auto IsLegacyVALUNotDotFn = [](const MachineInstr &MI) {
3557 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
3558 !SIInstrInfo::isMFMA(MI) && !SIInstrInfo::isDOT(MI);
3559 };
3560
3561 if (!SIInstrInfo::isMFMA(MI: *MI))
3562 return WaitStatesNeeded;
3563
3564 const int VALUWritesExecWaitStates = 4;
3565 int WaitStatesNeededForUse = VALUWritesExecWaitStates -
3566 getWaitStatesSinceDef(Reg: AMDGPU::EXEC, IsHazardDef: IsLegacyVALUFn,
3567 Limit: VALUWritesExecWaitStates);
3568 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3569
3570 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2);
3571
3572 // Loop for both DGEMM and S/HGEMM 2nd instruction.
3573 for (const MachineOperand &Use : MI->explicit_uses()) {
3574 const int LegacyVALUNotDotWritesVGPRWaitStates = 2;
3575 const int MaxWaitStates =
3576 GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(
3577 NumPasses: 16, IsGFX950: ST.hasGFX950Insts());
3578
3579 if (!Use.isReg())
3580 continue;
3581 Register Reg = Use.getReg();
3582
3583 WaitStatesNeededForUse = LegacyVALUNotDotWritesVGPRWaitStates -
3584 getWaitStatesSinceDef(Reg, IsHazardDef: IsLegacyVALUNotDotFn, Limit: MaxWaitStates);
3585 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3586
3587 int OpNo = Use.getOperandNo();
3588 WaitStatesNeeded = std::max(
3589 a: WaitStatesNeeded,
3590 b: getMaxWindowDeficit(
3591 MaxWindow: MaxWaitStates, WindowFor: [&](const MachineInstr &P) -> std::optional<int> {
3592 if (!SIInstrInfo::isMFMA(MI: P) ||
3593 !TRI.regsOverlap(RegA: P.getOperand(i: 0).getReg(), RegB: Reg))
3594 return std::nullopt;
3595 return getMFMAReadWaitStates(Consumer: *MI, Producer: P, Reg, IsSrcC: OpNo == SrcCIdx);
3596 }));
3597
3598 if (WaitStatesNeeded == MaxWaitStates)
3599 break;
3600 }
3601
3602 // Pad neighboring MFMA with noops for better inter-wave performance.
3603 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: checkMFMAPadding(MI));
3604
3605 return WaitStatesNeeded;
3606}
3607
3608int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) const {
3609 // On gfx90a+ relevant hazards are checked in checkMAIVALUHazards()
3610 if (!ST.hasMAIInsts() || ST.hasGFX90AInsts())
3611 return 0;
3612
3613 int WaitStatesNeeded = 0;
3614
3615 auto IsAccVgprReadFn = [](const MachineInstr &MI) {
3616 return MI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64;
3617 };
3618
3619 for (const MachineOperand &Op : MI->explicit_uses()) {
3620 if (!Op.isReg() || !TRI.isVGPR(MRI: MF.getRegInfo(), Reg: Op.getReg()))
3621 continue;
3622
3623 Register Reg = Op.getReg();
3624
3625 const int AccVgprReadLdStWaitStates = 2;
3626 const int VALUWriteAccVgprRdWrLdStDepVALUWaitStates = 1;
3627 const int MaxWaitStates = 2;
3628
3629 int WaitStatesNeededForUse = AccVgprReadLdStWaitStates -
3630 getWaitStatesSinceDef(Reg, IsHazardDef: IsAccVgprReadFn, Limit: MaxWaitStates);
3631 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3632
3633 if (WaitStatesNeeded == MaxWaitStates)
3634 return WaitStatesNeeded; // Early exit.
3635
3636 auto IsVALUAccVgprRdWrCheckFn = [Reg, this](const MachineInstr &MI) {
3637 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 &&
3638 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
3639 return false;
3640 auto IsVALUFn = [](const MachineInstr &MI) {
3641 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
3642 !SIInstrInfo::isMAI(MI);
3643 };
3644 return getWaitStatesSinceDef(Reg, IsHazardDef: IsVALUFn, Limit: 2 /*MaxWaitStates*/) <
3645 std::numeric_limits<int>::max();
3646 };
3647
3648 WaitStatesNeededForUse = VALUWriteAccVgprRdWrLdStDepVALUWaitStates -
3649 getWaitStatesSince(IsHazard: IsVALUAccVgprRdWrCheckFn, Limit: MaxWaitStates);
3650 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3651 }
3652
3653 return WaitStatesNeeded;
3654}
3655
3656int GCNHazardRecognizer::checkPermlaneHazards(MachineInstr *MI) const {
3657 assert(!ST.hasVcmpxPermlaneHazard() &&
3658 "this is a different vcmpx+permlane hazard");
3659 const SIRegisterInfo *TRI = ST.getRegisterInfo();
3660 const SIInstrInfo *TII = ST.getInstrInfo();
3661
3662 auto IsVCmpXWritesExecFn = [TII, TRI](const MachineInstr &MI) {
3663 return isVCmpXWritesExec(TII: *TII, TRI: *TRI, MI);
3664 };
3665
3666 auto IsVALUFn = [](const MachineInstr &MI) {
3667 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true);
3668 };
3669
3670 const int VCmpXWritesExecWaitStates = 4;
3671 const int VALUWritesVDstWaitStates = 2;
3672 int WaitStatesNeeded = 0;
3673
3674 for (const MachineOperand &Op : MI->explicit_uses()) {
3675 if (!Op.isReg() || !TRI->isVGPR(MRI: MF.getRegInfo(), Reg: Op.getReg()))
3676 continue;
3677 Register Reg = Op.getReg();
3678
3679 int WaitStatesSinceDef =
3680 VALUWritesVDstWaitStates -
3681 getWaitStatesSinceDef(Reg, IsHazardDef: IsVALUFn,
3682 /*MaxWaitStates=*/Limit: VALUWritesVDstWaitStates);
3683 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesSinceDef);
3684 if (WaitStatesNeeded >= VALUWritesVDstWaitStates)
3685 break;
3686 }
3687
3688 int VCmpXHazardWaits =
3689 VCmpXWritesExecWaitStates -
3690 getWaitStatesSince(IsHazard: IsVCmpXWritesExecFn, Limit: VCmpXWritesExecWaitStates);
3691
3692 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: VCmpXHazardWaits);
3693 return WaitStatesNeeded;
3694}
3695
3696static int GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(int NumPasses) {
3697 // 2 pass -> 4
3698 // 4 pass -> 6
3699 // 8 pass -> 10
3700 // 16 pass -> 18
3701 return NumPasses + 2;
3702}
3703
3704static int GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(int NumPasses,
3705 bool IsGFX950) {
3706 // xdl def cycles | gfx942 | gfx950
3707 // 2 pass | 5 5
3708 // 4 pass | 7 8
3709 // 8 pass | 11 12
3710 // 16 pass | 19 20
3711 return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
3712}
3713
3714static int GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses,
3715 bool IsGFX950) {
3716 // xdl def cycles | gfx942 | gfx950
3717 // 2 pass | 5 5
3718 // 4 pass | 7 8
3719 // 8 pass | 11 12
3720 // 16 pass | 19 20
3721 return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
3722}
3723
3724static int GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses) {
3725 // 2 pass -> 4
3726 // 4 pass -> 6
3727 // 8 pass -> 10
3728 // 16 pass -> 18
3729 return NumPasses + 2;
3730}
3731
3732int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const {
3733 if (!ST.hasGFX90AInsts())
3734 return 0;
3735
3736 auto IsDGEMMFn = [](const MachineInstr &MI) -> bool {
3737 return SIInstrInfo::isDGEMM(Opcode: MI.getOpcode());
3738 };
3739
3740 // This is checked in checkMAIHazards90A()
3741 if (SIInstrInfo::isMFMA(MI: *MI))
3742 return 0;
3743
3744 const MachineRegisterInfo &MRI = MF.getRegInfo();
3745
3746 int WaitStatesNeeded = 0;
3747
3748 bool IsMem = SIInstrInfo::isVMEM(MI: *MI) || SIInstrInfo::isDS(MI: *MI);
3749 bool IsMemOrExport = IsMem || SIInstrInfo::isEXP(MI: *MI);
3750 bool IsVALU = SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true);
3751
3752 unsigned Reg;
3753 auto IsMFMAWriteFn = [&Reg, this](const MachineInstr &MI) {
3754 if (!SIInstrInfo::isMFMA(MI) ||
3755 !TRI.regsOverlap(RegA: MI.getOperand(i: 0).getReg(), RegB: Reg))
3756 return false;
3757 return true;
3758 };
3759
3760 const MachineInstr *DOT = nullptr;
3761 auto IsDotWriteFn = [&Reg, &DOT, this](const MachineInstr &MI) {
3762 if (!SIInstrInfo::isDOT(MI) ||
3763 !TRI.regsOverlap(RegA: MI.getOperand(i: 0).getReg(), RegB: Reg))
3764 return false;
3765 DOT = &MI;
3766 return true;
3767 };
3768
3769 bool DGEMMAfterVALUWrite = false;
3770 auto IsDGEMMHazard = [&DGEMMAfterVALUWrite, this](const MachineInstr &MI) {
3771 // Found DGEMM on reverse traversal to def.
3772 if (SIInstrInfo::isDGEMM(Opcode: MI.getOpcode()))
3773 DGEMMAfterVALUWrite = true;
3774
3775 // Only hazard if register is defined by a VALU and a DGEMM is found after
3776 // after the def.
3777 if (!TII.isVALU(MI, /*AllowLDSDMA=*/true) || !DGEMMAfterVALUWrite)
3778 return false;
3779
3780 return true;
3781 };
3782
3783 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opcode: MI->getOpcode(),
3784 Name: AMDGPU::OpName::src2);
3785
3786 if (IsMemOrExport || IsVALU) {
3787 const int SMFMA4x4WriteVgprVALUMemExpReadWaitStates = 5;
3788 const int SMFMA16x16WriteVgprVALUMemExpReadWaitStates = 11;
3789 const int SMFMA32x32WriteVgprVALUMemExpReadWaitStates = 19;
3790 const int DMFMA4x4WriteVgprMemExpReadWaitStates = 9;
3791 const int DMFMA16x16WriteVgprMemExpReadWaitStates = 18;
3792 const int DMFMA4x4WriteVgprVALUReadWaitStates = 6;
3793 const int DMFMA16x16WriteVgprVALUReadWaitStates = 11;
3794 const int GFX950_DMFMA16x16WriteVgprVALUReadWaitStates = 19;
3795 const int DotWriteSameDotReadSrcAB = 3;
3796 const int DotWriteDifferentVALURead = 3;
3797 const int DMFMABetweenVALUWriteVMEMRead = 2;
3798 const int MaxWaitStates =
3799 GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(NumPasses: 16,
3800 IsGFX950: ST.hasGFX950Insts());
3801
3802 for (const MachineOperand &Use : MI->explicit_uses()) {
3803 if (!Use.isReg())
3804 continue;
3805 Reg = Use.getReg();
3806
3807 DOT = nullptr;
3808 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsHazardDef: IsDotWriteFn,
3809 Limit: MaxWaitStates);
3810 if (DOT) {
3811 int NeedWaitStates = 0;
3812 if (DOT->getOpcode() == MI->getOpcode()) {
3813 if (&Use - &MI->getOperand(i: 0) != SrcCIdx)
3814 NeedWaitStates = DotWriteSameDotReadSrcAB;
3815 } else {
3816 NeedWaitStates = DotWriteDifferentVALURead;
3817 }
3818
3819 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
3820 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3821 }
3822
3823 // Workaround for HW data hazard bug observed only in GFX90A. When there
3824 // is a DGEMM instruction in-between a VALU and a VMEM instruction it
3825 // causes the SQ to incorrectly not insert two wait states between the two
3826 // instructions needed to avoid data hazard.
3827 if (IsMem && ST.hasGFX90AInsts() && !ST.hasGFX940Insts()) {
3828 DGEMMAfterVALUWrite = false;
3829 if (TRI.isVectorRegister(MRI, Reg)) {
3830 int WaitStatesNeededForUse =
3831 DMFMABetweenVALUWriteVMEMRead -
3832 getWaitStatesSinceDef(Reg, IsHazardDef: IsDGEMMHazard,
3833 Limit: DMFMABetweenVALUWriteVMEMRead);
3834
3835 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3836 }
3837 }
3838
3839 auto MFMAWindow = [&](const MachineInstr &Producer) {
3840 int NumPasses = TSchedModel.computeInstrLatency(MI: &Producer);
3841 int NeedWaitStates = MaxWaitStates;
3842
3843 if (SIInstrInfo::isDGEMM(Opcode: Producer.getOpcode())) {
3844 switch (NumPasses) {
3845 case 4:
3846 NeedWaitStates = IsMemOrExport
3847 ? DMFMA4x4WriteVgprMemExpReadWaitStates
3848 : DMFMA4x4WriteVgprVALUReadWaitStates;
3849 break;
3850 case 8:
3851 case 16:
3852 NeedWaitStates =
3853 IsMemOrExport
3854 ? DMFMA16x16WriteVgprMemExpReadWaitStates
3855 : (ST.hasGFX950Insts()
3856 ? GFX950_DMFMA16x16WriteVgprVALUReadWaitStates
3857 : DMFMA16x16WriteVgprVALUReadWaitStates);
3858 break;
3859 default:
3860 llvm_unreachable("unexpected dgemm");
3861 }
3862 } else if (ST.hasGFX940Insts()) {
3863 NeedWaitStates =
3864 TII.isXDL(MI: Producer)
3865 ? GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(
3866 NumPasses, IsGFX950: ST.hasGFX950Insts())
3867 : GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates(
3868 NumPasses);
3869 } else {
3870 switch (NumPasses) {
3871 case 2:
3872 NeedWaitStates = SMFMA4x4WriteVgprVALUMemExpReadWaitStates;
3873 break;
3874 case 8:
3875 NeedWaitStates = SMFMA16x16WriteVgprVALUMemExpReadWaitStates;
3876 break;
3877 case 16:
3878 NeedWaitStates = SMFMA32x32WriteVgprVALUMemExpReadWaitStates;
3879 break;
3880 default:
3881 llvm_unreachable("unexpected number of passes for mfma");
3882 }
3883 }
3884
3885 assert(NeedWaitStates <= MaxWaitStates &&
3886 "hazard requirement exceeds the scan window");
3887 return NeedWaitStates;
3888 };
3889
3890 WaitStatesNeeded = std::max(
3891 a: WaitStatesNeeded,
3892 b: getMaxWindowDeficit(MaxWindow: MaxWaitStates,
3893 WindowFor: [&](const MachineInstr &P) -> std::optional<int> {
3894 if (!IsMFMAWriteFn(P))
3895 return std::nullopt;
3896 return MFMAWindow(P);
3897 }));
3898
3899 if (WaitStatesNeeded >= MaxWaitStates)
3900 break;
3901 }
3902 }
3903
3904 unsigned Opc = MI->getOpcode();
3905 const int DMFMAToFMA64WaitStates = 2;
3906 if ((Opc == AMDGPU::V_FMA_F64_e64 ||
3907 Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64 ||
3908 Opc == AMDGPU::V_FMAC_F64_dpp) &&
3909 WaitStatesNeeded < DMFMAToFMA64WaitStates) {
3910 int WaitStatesNeededForUse = DMFMAToFMA64WaitStates -
3911 getWaitStatesSince(IsHazard: IsDGEMMFn, Limit: DMFMAToFMA64WaitStates);
3912 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3913 }
3914
3915 if (!IsVALU && !IsMemOrExport)
3916 return WaitStatesNeeded;
3917
3918 for (const MachineOperand &Def : MI->defs()) {
3919 const int SMFMA4x4WriteVgprVALUWawWaitStates = 5;
3920 const int SMFMA16x16WriteVgprVALUWawWaitStates = 11;
3921 const int SMFMA32x32WriteVgprVALUWawWaitStates = 19;
3922 const int SMFMA4x4ReadVgprVALUWarWaitStates = 1;
3923 const int GFX940_XDL4PassReadVgprVALUWarWaitStates = 3;
3924 const int SMFMA16x16ReadVgprVALUWarWaitStates = 7;
3925 const int SMFMA32x32ReadVgprVALUWarWaitStates = 15;
3926 const int DMFMA4x4WriteVgprVALUWriteWaitStates = 6;
3927 const int DMFMA16x16WriteVgprVALUWriteWaitStates = 11;
3928 const int DotWriteDifferentVALUWrite = 3;
3929 const int MaxWaitStates =
3930 GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(NumPasses: 16, IsGFX950: ST.hasGFX950Insts());
3931 const int MaxWarWaitStates = 15;
3932
3933 Reg = Def.getReg();
3934
3935 DOT = nullptr;
3936 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsHazardDef: IsDotWriteFn,
3937 Limit: MaxWaitStates);
3938 if (DOT && DOT->getOpcode() != MI->getOpcode())
3939 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: DotWriteDifferentVALUWrite -
3940 WaitStatesSinceDef);
3941
3942 auto MFMAWawWindow = [&](const MachineInstr &Producer) {
3943 int NeedWaitStates = MaxWaitStates;
3944 int NumPasses = TSchedModel.computeInstrLatency(MI: &Producer);
3945
3946 if (SIInstrInfo::isDGEMM(Opcode: Producer.getOpcode())) {
3947 switch (NumPasses) {
3948 case 4:
3949 NeedWaitStates = DMFMA4x4WriteVgprVALUWriteWaitStates;
3950 break;
3951 case 8:
3952 case 16:
3953 NeedWaitStates = DMFMA16x16WriteVgprVALUWriteWaitStates;
3954 break;
3955 default:
3956 llvm_unreachable("unexpected number of cycles for dgemm");
3957 }
3958 } else if (ST.hasGFX940Insts()) {
3959 NeedWaitStates =
3960 TII.isXDL(MI: Producer)
3961 ? GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(
3962 NumPasses, IsGFX950: ST.hasGFX950Insts())
3963 : GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(NumPasses);
3964 } else {
3965 switch (NumPasses) {
3966 case 2:
3967 NeedWaitStates = SMFMA4x4WriteVgprVALUWawWaitStates;
3968 break;
3969 case 8:
3970 NeedWaitStates = SMFMA16x16WriteVgprVALUWawWaitStates;
3971 break;
3972 case 16:
3973 NeedWaitStates = SMFMA32x32WriteVgprVALUWawWaitStates;
3974 break;
3975 default:
3976 llvm_unreachable("Unexpected number of passes for mfma");
3977 }
3978 }
3979
3980 assert(NeedWaitStates <= MaxWaitStates &&
3981 "hazard requirement exceeds the scan window");
3982 return NeedWaitStates;
3983 };
3984
3985 WaitStatesNeeded = std::max(
3986 a: WaitStatesNeeded,
3987 b: getMaxWindowDeficit(MaxWindow: MaxWaitStates,
3988 WindowFor: [&](const MachineInstr &P) -> std::optional<int> {
3989 if (!IsMFMAWriteFn(P))
3990 return std::nullopt;
3991 return MFMAWawWindow(P);
3992 }));
3993
3994 if (WaitStatesNeeded >= MaxWaitStates)
3995 break;
3996
3997 // Wait states this write needs after an MFMA that read Reg as srcC.
3998 auto SMFMAReadAsCWindow =
3999 [&](const MachineInstr &Reader) -> std::optional<int> {
4000 if (!SIInstrInfo::isMFMA(MI: Reader) ||
4001 SIInstrInfo::isDGEMM(Opcode: Reader.getOpcode()) ||
4002 !Reader.readsRegister(Reg, TRI: &TRI))
4003 return std::nullopt;
4004
4005 if (ST.hasGFX940Insts() && !TII.isXDL(MI: Reader))
4006 return std::nullopt;
4007
4008 const MachineOperand *SrcC =
4009 TII.getNamedOperand(MI: Reader, OperandName: AMDGPU::OpName::src2);
4010 assert(SrcC && "an MFMA must have a src2 operand");
4011 if (!SrcC->isReg() || !TRI.regsOverlap(RegA: SrcC->getReg(), RegB: Reg))
4012 return std::nullopt;
4013
4014 switch (TSchedModel.computeInstrLatency(MI: &Reader)) {
4015 case 2:
4016 return SMFMA4x4ReadVgprVALUWarWaitStates;
4017 case 4:
4018 assert(ST.hasGFX940Insts() && "4-pass XDL is gfx940 or later");
4019 return GFX940_XDL4PassReadVgprVALUWarWaitStates;
4020 case 8:
4021 return SMFMA16x16ReadVgprVALUWarWaitStates;
4022 case 16:
4023 [[fallthrough]];
4024 default:
4025 return SMFMA32x32ReadVgprVALUWarWaitStates;
4026 }
4027 };
4028
4029 WaitStatesNeeded =
4030 std::max(a: WaitStatesNeeded,
4031 b: getMaxWindowDeficit(MaxWindow: MaxWarWaitStates, WindowFor: SMFMAReadAsCWindow));
4032 }
4033
4034 return WaitStatesNeeded;
4035}
4036
4037bool GCNHazardRecognizer::ShouldPreferAnother(SUnit *SU) const {
4038 if (!SU->isInstr())
4039 return false;
4040
4041 const MachineInstr *MAI = nullptr;
4042
4043 auto IsMFMAFn = [&MAI](const MachineInstr &MI) {
4044 MAI = nullptr;
4045 if (SIInstrInfo::isMFMA(MI))
4046 MAI = &MI;
4047 return MAI != nullptr;
4048 };
4049
4050 MachineInstr *MI = SU->getInstr();
4051 if (IsMFMAFn(*MI)) {
4052 int W = getWaitStatesSince(IsHazard: IsMFMAFn, Limit: 16);
4053 if (MAI)
4054 return W < (int)TSchedModel.computeInstrLatency(MI: MAI);
4055 }
4056
4057 return false;
4058}
4059
4060// Adjust global offsets for instructions bundled with S_GETPC_B64 after
4061// insertion of a new instruction.
4062static void updateGetPCBundle(MachineInstr *NewMI) {
4063 if (!NewMI->isBundled())
4064 return;
4065
4066 // Find start of bundle.
4067 auto I = NewMI->getIterator();
4068 while (I->isBundledWithPred())
4069 I--;
4070 if (I->isBundle())
4071 I++;
4072
4073 // Bail if this is not an S_GETPC bundle.
4074 if (I->getOpcode() != AMDGPU::S_GETPC_B64)
4075 return;
4076
4077 // Update offsets of any references in the bundle.
4078 const unsigned NewBytes = 4;
4079 assert(NewMI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
4080 "Unexpected instruction insertion in bundle");
4081 auto NextMI = std::next(x: NewMI->getIterator());
4082 auto End = NewMI->getParent()->end();
4083 while (NextMI != End && NextMI->isBundledWithPred()) {
4084 for (auto &Operand : NextMI->operands()) {
4085 if (Operand.isGlobal())
4086 Operand.setOffset(Operand.getOffset() + NewBytes);
4087 }
4088 NextMI++;
4089 }
4090}
4091
4092bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) {
4093 if (!ST.hasVALUMaskWriteHazard())
4094 return false;
4095 assert(!ST.hasExtendedWaitCounts());
4096
4097 if (!ST.isWave64())
4098 return false;
4099
4100 const bool IsSALU = SIInstrInfo::isSALU(MI: *MI);
4101 const bool IsVALU = SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true);
4102 if (!IsSALU && !IsVALU)
4103 return false;
4104
4105 // The hazard sequence is three instructions:
4106 // 1. VALU reads SGPR as mask
4107 // 2. VALU/SALU writes SGPR
4108 // 3. VALU/SALU reads SGPR
4109 // The hazard can expire if the distance between 2 and 3 is sufficient,
4110 // or (2) is VALU and (3) is SALU.
4111 // In practice this happens <10% of the time, hence always assume the hazard
4112 // exists if (1) and (2) are present to avoid searching all SGPR reads.
4113
4114 const SIRegisterInfo *TRI = ST.getRegisterInfo();
4115 const MachineRegisterInfo &MRI = MF.getRegInfo();
4116
4117 auto IgnoreableSGPR = [](const Register Reg) {
4118 switch (Reg) {
4119 case AMDGPU::EXEC:
4120 case AMDGPU::EXEC_LO:
4121 case AMDGPU::EXEC_HI:
4122 case AMDGPU::M0:
4123 case AMDGPU::SGPR_NULL:
4124 case AMDGPU::SGPR_NULL64:
4125 case AMDGPU::SCC:
4126 return true;
4127 default:
4128 return false;
4129 }
4130 };
4131 auto IsVCC = [](const Register Reg) {
4132 return Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::VCC_HI;
4133 };
4134
4135 struct StateType {
4136 SmallSet<Register, 2> HazardSGPRs;
4137
4138 static unsigned getHashValue(const StateType &State) {
4139 return hash_combine_range(R: State.HazardSGPRs);
4140 }
4141 static bool isEqual(const StateType &LHS, const StateType &RHS) {
4142 return LHS.HazardSGPRs == RHS.HazardSGPRs;
4143 }
4144 };
4145
4146 SmallVector<const MachineInstr *> WaitInstrs;
4147 StateType InitialState;
4148
4149 // Look for SGPR write.
4150 MachineOperand *HazardDef = nullptr;
4151 for (MachineOperand &Op : MI->all_defs()) {
4152 Register Reg = Op.getReg();
4153 if (IgnoreableSGPR(Reg))
4154 continue;
4155 if (!IsVCC(Reg)) {
4156 if (Op.isImplicit())
4157 continue;
4158 if (!TRI->isSGPRReg(MRI, Reg))
4159 continue;
4160 }
4161
4162 HazardDef = &Op;
4163 break;
4164 }
4165
4166 if (!HazardDef)
4167 return false;
4168
4169 // Setup to track writes to individual SGPRs
4170 const Register HazardReg = HazardDef->getReg();
4171 if (AMDGPU::SReg_32RegClass.contains(Reg: HazardReg)) {
4172 InitialState.HazardSGPRs.insert(V: HazardReg);
4173 } else {
4174 assert(AMDGPU::SReg_64RegClass.contains(HazardReg));
4175 InitialState.HazardSGPRs.insert(V: TRI->getSubReg(Reg: HazardReg, Idx: AMDGPU::sub0));
4176 InitialState.HazardSGPRs.insert(V: TRI->getSubReg(Reg: HazardReg, Idx: AMDGPU::sub1));
4177 }
4178
4179 auto IsHazardFn = [&](StateType &State, const MachineInstr &I) {
4180 if (State.HazardSGPRs.empty())
4181 return HazardExpired;
4182
4183 switch (I.getOpcode()) {
4184 case AMDGPU::V_ADDC_U32_e32:
4185 case AMDGPU::V_ADDC_U32_dpp:
4186 case AMDGPU::V_CNDMASK_B16_t16_e32:
4187 case AMDGPU::V_CNDMASK_B16_fake16_e32:
4188 case AMDGPU::V_CNDMASK_B16_t16_dpp:
4189 case AMDGPU::V_CNDMASK_B16_fake16_dpp:
4190 case AMDGPU::V_CNDMASK_B32_e32:
4191 case AMDGPU::V_CNDMASK_B32_dpp:
4192 case AMDGPU::V_DIV_FMAS_F32_e64:
4193 case AMDGPU::V_DIV_FMAS_F64_e64:
4194 case AMDGPU::V_SUBB_U32_e32:
4195 case AMDGPU::V_SUBB_U32_dpp:
4196 case AMDGPU::V_SUBBREV_U32_e32:
4197 case AMDGPU::V_SUBBREV_U32_dpp: {
4198 // These implicitly read VCC as mask source.
4199 return IsVCC(HazardReg) ? HazardFound : NoHazardFound;
4200 }
4201 case AMDGPU::V_ADDC_U32_e64:
4202 case AMDGPU::V_ADDC_U32_e64_dpp:
4203 case AMDGPU::V_CNDMASK_B16_t16_e64:
4204 case AMDGPU::V_CNDMASK_B16_fake16_e64:
4205 case AMDGPU::V_CNDMASK_B16_t16_e64_dpp:
4206 case AMDGPU::V_CNDMASK_B16_fake16_e64_dpp:
4207 case AMDGPU::V_CNDMASK_B32_e64:
4208 case AMDGPU::V_CNDMASK_B32_e64_dpp:
4209 case AMDGPU::V_SUBB_U32_e64:
4210 case AMDGPU::V_SUBB_U32_e64_dpp:
4211 case AMDGPU::V_SUBBREV_U32_e64:
4212 case AMDGPU::V_SUBBREV_U32_e64_dpp: {
4213 // Only check mask register overlaps.
4214 const MachineOperand *SSRCOp = TII.getNamedOperand(MI: I, OperandName: AMDGPU::OpName::src2);
4215 assert(SSRCOp);
4216 bool Result = TRI->regsOverlap(RegA: SSRCOp->getReg(), RegB: HazardReg);
4217 return Result ? HazardFound : NoHazardFound;
4218 }
4219 default:
4220 return NoHazardFound;
4221 }
4222 };
4223
4224 auto UpdateStateFn = [&](StateType &State, const MachineInstr &I) {
4225 // Update tracking of SGPR writes.
4226 for (auto &Op : I.all_defs()) {
4227 Register Reg = Op.getReg();
4228 if (IgnoreableSGPR(Reg))
4229 continue;
4230 if (!IsVCC(Reg)) {
4231 if (Op.isImplicit())
4232 continue;
4233 if (!TRI->isSGPRReg(MRI, Reg))
4234 continue;
4235 }
4236
4237 // Stop tracking any SGPRs with writes on the basis that they will
4238 // already have an appropriate wait inserted afterwards.
4239 SmallVector<Register, 2> Found;
4240 for (Register SGPR : State.HazardSGPRs) {
4241 if (Reg == SGPR || TRI->regsOverlap(RegA: Reg, RegB: SGPR))
4242 Found.push_back(Elt: SGPR);
4243 }
4244 for (Register SGPR : Found)
4245 State.HazardSGPRs.erase(V: SGPR);
4246 }
4247 };
4248
4249 // Check for hazard
4250 if (!hasHazard<StateType>(InitialState, IsHazard: IsHazardFn, UpdateState: UpdateStateFn,
4251 InitialMBB: MI->getParent(),
4252 InitialI: std::next(x: MI->getReverseIterator())))
4253 return false;
4254
4255 // Compute counter mask
4256 unsigned DepCtr =
4257 IsVALU ? (IsVCC(HazardReg) ? AMDGPU::DepCtr::encodeFieldVaVcc(VaVcc: 0, STI: ST)
4258 : AMDGPU::DepCtr::encodeFieldVaSdst(VaSdst: 0, STI: ST))
4259 : AMDGPU::DepCtr::encodeFieldSaSdst(SaSdst: 0, STI: ST);
4260
4261 // Add s_waitcnt_depctr after SGPR write.
4262 auto NextMI = std::next(x: MI->getIterator());
4263 auto NewMI = BuildMI(BB&: *MI->getParent(), I: NextMI, MIMD: MI->getDebugLoc(),
4264 MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4265 .addImm(Val: DepCtr);
4266
4267 // SALU write may be s_getpc in a bundle.
4268 updateGetPCBundle(NewMI);
4269
4270 return true;
4271}
4272
4273static bool ensureEntrySetPrio(MachineFunction *MF, int Priority,
4274 const SIInstrInfo &TII) {
4275 MachineBasicBlock &EntryMBB = MF->front();
4276 if (EntryMBB.begin() != EntryMBB.end()) {
4277 auto &EntryMI = *EntryMBB.begin();
4278 if (EntryMI.getOpcode() == AMDGPU::S_SETPRIO &&
4279 EntryMI.getOperand(i: 0).getImm() >= Priority)
4280 return false;
4281 }
4282
4283 BuildMI(BB&: EntryMBB, I: EntryMBB.begin(), MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_SETPRIO))
4284 .addImm(Val: Priority);
4285 return true;
4286}
4287
4288bool GCNHazardRecognizer::fixRequiredExportPriority(MachineInstr *MI) {
4289 if (!ST.hasRequiredExportPriority())
4290 return false;
4291
4292 // Assume the following shader types will never have exports,
4293 // and avoid adding or adjusting S_SETPRIO.
4294 MachineBasicBlock *MBB = MI->getParent();
4295 MachineFunction *MF = MBB->getParent();
4296 auto CC = MF->getFunction().getCallingConv();
4297 switch (CC) {
4298 case CallingConv::AMDGPU_CS:
4299 case CallingConv::AMDGPU_CS_Chain:
4300 case CallingConv::AMDGPU_CS_ChainPreserve:
4301 case CallingConv::AMDGPU_KERNEL:
4302 return false;
4303 default:
4304 break;
4305 }
4306
4307 const int MaxPriority = 3;
4308 const int NormalPriority = 2;
4309 const int PostExportPriority = 0;
4310
4311 auto It = MI->getIterator();
4312 switch (MI->getOpcode()) {
4313 case AMDGPU::S_ENDPGM:
4314 case AMDGPU::S_ENDPGM_SAVED:
4315 case AMDGPU::S_ENDPGM_ORDERED_PS_DONE:
4316 case AMDGPU::SI_RETURN_TO_EPILOG:
4317 // Ensure shader with calls raises priority at entry.
4318 // This ensures correct priority if exports exist in callee.
4319 if (MF->getFrameInfo().hasCalls())
4320 return ensureEntrySetPrio(MF, Priority: NormalPriority, TII);
4321 return false;
4322 case AMDGPU::S_SETPRIO: {
4323 // Raise minimum priority unless in workaround.
4324 auto &PrioOp = MI->getOperand(i: 0);
4325 int Prio = PrioOp.getImm();
4326 bool InWA = (Prio == PostExportPriority) &&
4327 (It != MBB->begin() && TII.isEXP(MI: *std::prev(x: It)));
4328 if (InWA || Prio >= NormalPriority)
4329 return false;
4330 PrioOp.setImm(std::min(a: Prio + NormalPriority, b: MaxPriority));
4331 return true;
4332 }
4333 default:
4334 if (!TII.isEXP(MI: *MI))
4335 return false;
4336 break;
4337 }
4338
4339 // Check entry priority at each export (as there will only be a few).
4340 // Note: amdgpu_gfx can only be a callee, so defer to caller setprio.
4341 bool Changed = false;
4342 if (CC != CallingConv::AMDGPU_Gfx && CC != CallingConv::AMDGPU_Gfx_WholeWave)
4343 Changed = ensureEntrySetPrio(MF, Priority: NormalPriority, TII);
4344
4345 auto NextMI = std::next(x: It);
4346 bool EndOfShader = false;
4347 if (NextMI != MBB->end()) {
4348 // Only need WA at end of sequence of exports.
4349 if (TII.isEXP(MI: *NextMI))
4350 return Changed;
4351 // Assume appropriate S_SETPRIO after export means WA already applied.
4352 if (NextMI->getOpcode() == AMDGPU::S_SETPRIO &&
4353 NextMI->getOperand(i: 0).getImm() == PostExportPriority)
4354 return Changed;
4355 EndOfShader = NextMI->getOpcode() == AMDGPU::S_ENDPGM;
4356 }
4357
4358 const DebugLoc &DL = MI->getDebugLoc();
4359
4360 // Lower priority.
4361 BuildMI(BB&: *MBB, I: NextMI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_SETPRIO))
4362 .addImm(Val: PostExportPriority);
4363
4364 if (!EndOfShader) {
4365 // Wait for exports to complete.
4366 BuildMI(BB&: *MBB, I: NextMI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_EXPCNT))
4367 .addReg(RegNo: AMDGPU::SGPR_NULL)
4368 .addImm(Val: 0);
4369 }
4370
4371 BuildMI(BB&: *MBB, I: NextMI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_NOP)).addImm(Val: 0);
4372 BuildMI(BB&: *MBB, I: NextMI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_NOP)).addImm(Val: 0);
4373
4374 if (!EndOfShader) {
4375 // Return to normal (higher) priority.
4376 BuildMI(BB&: *MBB, I: NextMI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_SETPRIO))
4377 .addImm(Val: NormalPriority);
4378 }
4379
4380 return true;
4381}
4382
4383// Advance past meta instructions (debug values, labels, CFI, KILL, etc.) to the
4384// next instruction that actually issues. Unlike skipDebugInstructionsForward /
4385// next_nodbg, this skips the full isMetaInstruction() set.
4386static MachineBasicBlock::iterator
4387skipMetaInstructionsForward(MachineBasicBlock::iterator I,
4388 MachineBasicBlock::iterator End) {
4389 while (I != End && I->isMetaInstruction())
4390 ++I;
4391 return I;
4392}
4393
4394bool GCNHazardRecognizer::fixVPermPk16Hazard(MachineInstr *MI) {
4395 // Requirement #1 of 2:
4396 // The cross-wave entry-block mitigation is delegated to the mandatory
4397 // unclaused-VMEM entry prologue (GLOBAL_PREFETCH_B8 + V_NOP).
4398 assert(ST.hasRequiresInitialUnclausedVmem() &&
4399 "V_PERM_PK16-hazard subtarget must provide the unclaused-VMEM entry "
4400 "prologue to satisfy the cross-wave entry mitigation");
4401
4402 if (!SIInstrInfo::isVPermPk16(Opcode: MI->getOpcode()))
4403 return false;
4404
4405 MachineBasicBlock *MBB = MI->getParent();
4406
4407 // Requirement #2 of 2:
4408 // V_PERM_PK16 must be immediately followed by a safe instruction.
4409 MachineBasicBlock::iterator NextI =
4410 skipMetaInstructionsForward(I: std::next(x: MI->getIterator()), End: MBB->end());
4411 if (NextI != MBB->end() && TII.isVPermPk16SafeInstr(MI: *NextI))
4412 return false;
4413
4414 // EXEC is guaranteed non-zero here: V_PERM_PK16 reports unwanted effects
4415 // when EXEC is empty, so s_cbranch_execz over this region is retained.
4416 // A plain V_NOP is therefore a real VALU nop and clears the hazard.
4417 BuildMI(BB&: *MBB, I: NextI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_NOP_e32));
4418 return true;
4419}
4420
4421bool GCNHazardRecognizer::fixGetRegWaitIdle(MachineInstr *MI) {
4422 if (!isSGetReg(Opcode: MI->getOpcode()))
4423 return false;
4424
4425 const SIInstrInfo *TII = ST.getInstrInfo();
4426 switch (getHWReg(TII, RegInstr: *MI)) {
4427 default:
4428 return false;
4429 case AMDGPU::Hwreg::ID_STATUS:
4430 case AMDGPU::Hwreg::ID_STATE_PRIV:
4431 case AMDGPU::Hwreg::ID_EXCP_FLAG_PRIV:
4432 case AMDGPU::Hwreg::ID_EXCP_FLAG_USER:
4433 break;
4434 }
4435
4436 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
4437 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4438 .addImm(Val: 0);
4439 return true;
4440}
4441
4442bool GCNHazardRecognizer::fixDsAtomicAsyncBarrierArriveB64(MachineInstr *MI) {
4443 if (MI->getOpcode() != AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
4444 return false;
4445
4446 const SIInstrInfo *TII = ST.getInstrInfo();
4447 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
4448 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4449 .addImm(Val: AMDGPU::DepCtr::encodeFieldVmVsrc(VmVsrc: 0, STI: ST));
4450 BuildMI(BB&: *MI->getParent(), I: std::next(x: MI->getIterator()), MIMD: MI->getDebugLoc(),
4451 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4452 .addImm(Val: AMDGPU::DepCtr::encodeFieldVmVsrc(VmVsrc: 0, STI: ST));
4453
4454 return true;
4455}
4456
4457bool GCNHazardRecognizer::fixScratchBaseForwardingHazard(MachineInstr *MI) {
4458 // No reason to check this in pre-RA scheduling, SGPRs have to be allocated
4459 // for hazard to trigger.
4460 if (!isHazardRecognizerMode())
4461 return false;
4462
4463 const SIRegisterInfo *TRI = ST.getRegisterInfo();
4464 const SIInstrInfo *TII = ST.getInstrInfo();
4465 // Hazard expires after 10 SGPR writes by SALU or 8 SGPR writes by VALU.
4466 const int FlatScrBaseWaitStates = 10;
4467
4468 bool ReadsFlatScrLo =
4469 MI->readsRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_LO, TRI);
4470 bool ReadsFlatScrHi =
4471 MI->readsRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_HI, TRI);
4472 if (isSGetReg(Opcode: MI->getOpcode())) {
4473 switch (getHWReg(TII, RegInstr: *MI)) {
4474 default:
4475 break;
4476 case AMDGPU::Hwreg::ID_FLAT_SCR_LO:
4477 ReadsFlatScrLo = true;
4478 break;
4479 case AMDGPU::Hwreg::ID_FLAT_SCR_HI:
4480 ReadsFlatScrHi = true;
4481 break;
4482 }
4483 }
4484
4485 const MachineRegisterInfo &MRI = MF.getRegInfo();
4486
4487 auto IsRegDefHazard = [&](Register Reg) -> bool {
4488 DenseSet<const MachineBasicBlock *> Visited;
4489 auto IsHazardFn = [TRI, Reg](const MachineInstr &MI) {
4490 return MI.modifiesRegister(Reg, TRI);
4491 };
4492
4493 // This literally abuses the idea of waitstates. Instead of waitstates it
4494 // returns 1 for SGPR written and 0 otherwise.
4495 auto IsSGPRDef = [TII, TRI, &MRI](const MachineInstr &MI) -> unsigned {
4496 if (!TII->isSALU(MI) && !TII->isVALU(MI, /*AllowLDSDMA=*/true))
4497 return 0;
4498 for (const MachineOperand &MO : MI.all_defs()) {
4499 if (TRI->isSGPRReg(MRI, Reg: MO.getReg()))
4500 return 1;
4501 }
4502 return 0;
4503 };
4504
4505 auto IsExpiredFn = [=](const MachineInstr &MI, int SgprWrites) {
4506 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR) {
4507 unsigned Wait = MI.getOperand(i: 0).getImm();
4508 if (AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: Wait) == 0 &&
4509 AMDGPU::DepCtr::decodeFieldVaSdst(Encoded: Wait) == 0)
4510 return true;
4511 }
4512 return SgprWrites >= FlatScrBaseWaitStates;
4513 };
4514
4515 return ::getWaitStatesSince(
4516 IsHazard: IsHazardFn, MBB: MI->getParent(), I: std::next(x: MI->getReverseIterator()),
4517 WaitStates: 0, IsExpired: IsExpiredFn, Visited, GetNumWaitStates: IsSGPRDef) < FlatScrBaseWaitStates;
4518 };
4519
4520 if ((!ReadsFlatScrLo || MRI.isConstantPhysReg(PhysReg: AMDGPU::SGPR102) ||
4521 !IsRegDefHazard(AMDGPU::SGPR102)) &&
4522 (!ReadsFlatScrHi || MRI.isConstantPhysReg(PhysReg: AMDGPU::SGPR103) ||
4523 !IsRegDefHazard(AMDGPU::SGPR103)))
4524 return false;
4525
4526 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
4527 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4528 .addImm(Val: AMDGPU::DepCtr::encodeFieldVaSdst(
4529 Encoded: AMDGPU::DepCtr::encodeFieldSaSdst(SaSdst: 0, STI: ST), VaSdst: 0));
4530 return true;
4531}
4532
4533bool GCNHazardRecognizer::fixSetRegMode(MachineInstr *MI) {
4534 if (!isSSetReg(Opcode: MI->getOpcode()) ||
4535 MI->getOperand(i: 1).getImm() != AMDGPU::Hwreg::ID_MODE)
4536 return false;
4537
4538 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_NOP_e32));
4539 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_NOP_e32));
4540 return true;
4541}
4542
4543bool GCNHazardRecognizer::fixTDM(MachineInstr *MI) {
4544 auto IsTDM = [&](const MachineInstr &MI) -> bool {
4545 return SIInstrInfo::usesTENSOR_CNT(MI) &&
4546 MI.getOpcode() != AMDGPU::S_WAIT_TENSORCNT;
4547 };
4548
4549 if (!IsTDM(*MI))
4550 return false;
4551
4552 auto IsExpiredFn = [](const MachineInstr &MI, int) {
4553 if (MI.getOpcode() != AMDGPU::S_WAIT_TENSORCNT)
4554 return false;
4555 return MI.getOperand(i: 0).getImm() <= 10;
4556 };
4557
4558 if (::getWaitStatesSince(IsHazard: IsTDM, MI, IsExpired: IsExpiredFn) ==
4559 std::numeric_limits<int>::max())
4560 return false;
4561
4562 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
4563 MCID: TII.get(Opcode: AMDGPU::S_WAIT_TENSORCNT))
4564 .addImm(Val: 10);
4565 return true;
4566}
4567