1//===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements hazard recognizers for scheduling on GCN processors.
10//
11//===----------------------------------------------------------------------===//
12
13#include "GCNHazardRecognizer.h"
14#include "AMDGPUTargetMachine.h"
15#include "AMDGPUWaitcntUtils.h"
16#include "GCNSubtarget.h"
17#include "SIMachineFunctionInfo.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/CodeGen/MachineFrameInfo.h"
20#include "llvm/CodeGen/MachineFunction.h"
21#include "llvm/CodeGen/MachineInstrBuilder.h"
22#include "llvm/CodeGen/ScheduleDAG.h"
23#include "llvm/Support/Debug.h"
24#include "llvm/TargetParser/AMDGPUTargetParser.h"
25
26using namespace llvm;
27
28#define DEBUG_TYPE "gcn-hazard-recognizer"
29// Opt-in debug type for the per-candidate co-execution slot traces, which are
30// far too noisy for the normal debug output. Pass both types to get everything.
31#define DEBUG_TYPE_VERBOSE "gcn-hazard-recognizer-verbose"
32
33STATISTIC(NumWMMANopsHoisted,
34 "Number of WMMA hazard V_NOPs hoisted from loops");
35STATISTIC(NumWMMAHoistingBailed,
36 "Number of WMMA hazards where V_NOP hoisting was not possible");
37
38namespace {
39
40struct MFMAPaddingRatioParser : public cl::parser<unsigned> {
41 MFMAPaddingRatioParser(cl::Option &O) : cl::parser<unsigned>(O) {}
42
43 bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) {
44 if (Arg.getAsInteger(Radix: 0, Result&: Value))
45 return O.error(Message: "'" + Arg + "' value invalid for uint argument!");
46
47 if (Value > 100)
48 return O.error(Message: "'" + Arg + "' value must be in the range [0, 100]!");
49
50 return false;
51 }
52};
53
54} // end anonymous namespace
55
56static cl::opt<unsigned, false, MFMAPaddingRatioParser>
57 MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(Val: 0), cl::Hidden,
58 cl::desc("Fill a percentage of the latency between "
59 "neighboring MFMA with s_nops."));
60
61// This is intended for debugging purposes only.
62static cl::opt<unsigned>
63 NopPadding("amdgpu-snop-padding", cl::init(Val: 0), cl::Hidden,
64 cl::desc("Insert a s_nop x before every instruction"));
65
66static cl::opt<bool> EnableWMMAVnopHoisting(
67 "amdgpu-wmma-vnop-hoisting", cl::init(Val: true), cl::Hidden,
68 cl::desc("Hoist WMMA hazard V_NOPs from loops to preheaders"));
69
70//===----------------------------------------------------------------------===//
71// Hazard Recognizer Implementation
72//===----------------------------------------------------------------------===//
73
74static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF,
75 const GCNSubtarget &ST);
76
77GCNHazardRecognizer::GCNHazardRecognizer(
78 const MachineFunction &MF, GCNHazardRecognizer::OperatingMode Mode,
79 MachineLoopInfo *MLI)
80 : Mode(Mode), CurrCycleInstr(nullptr), MF(MF),
81 ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()),
82 TRI(TII.getRegisterInfo()), TSchedModel(TII.getSchedModel()), MLI(MLI),
83 ClauseUses(TRI.getNumRegUnits()), ClauseDefs(TRI.getNumRegUnits()) {
84 MaxLookAhead = MF.getRegInfo().isPhysRegUsed(PhysReg: AMDGPU::AGPR0) ? 19 : 5;
85 RunLdsBranchVmemWARHazardFixup = shouldRunLdsBranchVmemWARHazardFixup(MF, ST);
86 LLVM_DEBUG({
87 if (isPreRA())
88 dbgs() << " PreRA hazard recognizer: " << MF.getName() << "\n";
89 });
90}
91
92GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF,
93 MachineLoopInfo *MLI)
94 : GCNHazardRecognizer(MF, OperatingMode::PostRA, MLI) {}
95
96GCNHazardRecognizer::~GCNHazardRecognizer() {
97 // Dump any active co-execution window that did not complete naturally
98 // (e.g. region ended before the window expired).
99 LLVM_DEBUG({
100 if (CurrentCoExecStage.has_value()) {
101 unsigned Stage = *CurrentCoExecStage;
102 if (Stage < AMDGPU::MaxCoExecStages)
103 CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
104 dbgs() << " CoExec window ended at stage " << Stage << ":\n";
105 dumpCoExecWindow();
106 }
107 });
108}
109
110void GCNHazardRecognizer::Reset() {
111 EmittedInstrs.clear();
112 EmittedVALUInstrs.clear();
113 HasPendingWMMACoexecHazard = false;
114 if (isSchedulerMode())
115 schedulerReset();
116}
117
118void GCNHazardRecognizer::schedulerReset() {
119 LLVM_DEBUG({
120 if (CurrentCoExecStage.has_value() || CyclesUntilTRANS > 0 ||
121 CyclesUntilVALU > 0)
122 dbgs() << " Scheduler Reset: clearing co-exec window, TRANS="
123 << CyclesUntilTRANS << ", VALU=" << CyclesUntilVALU << "\n";
124 });
125 CurrentCoExecStage = std::nullopt;
126 CoExecWindowStartCycle = 0;
127 CyclesUntilTRANS = 0;
128 CyclesUntilVALU = 0;
129 ActiveCoExecInfo = AMDGPU::CoExecInfo();
130 CoExecWindowLog.fill(u: '.');
131}
132
133void GCNHazardRecognizer::dumpCoExecWindow() const {
134 unsigned W = ActiveCoExecInfo.TotalWindow;
135 if (W == 0)
136 return;
137
138 // Print the stage numbers row.
139 dbgs() << " Stages: ";
140 for (unsigned I = 0; I < W; ++I)
141 dbgs() << I % 10 << ' ';
142 dbgs() << '\n';
143
144 // Print the pattern row.
145 dbgs() << " Slots: ";
146 for (unsigned I = 0; I < W; ++I)
147 dbgs() << ActiveCoExecInfo.Pattern[I] << ' ';
148 dbgs() << '\n';
149
150 // Print the scheduled row.
151 dbgs() << " Scheduled: ";
152 for (unsigned I = 0; I < W; ++I)
153 dbgs() << CoExecWindowLog[I] << ' ';
154 dbgs() << '\n';
155}
156
157void GCNHazardRecognizer::schedulerAdvanceCycle() {
158 // Record what happened at the current stage of the co-exec window.
159 if (CurrentCoExecStage.has_value()) {
160 unsigned Stage = *CurrentCoExecStage;
161 if (Stage < AMDGPU::MaxCoExecStages) {
162 if (CurrCycleInstr)
163 CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
164 else
165 CoExecWindowLog[Stage] = '-';
166 }
167 }
168
169 LLVM_DEBUG({
170 bool HasState = CurrentCoExecStage.has_value() || CyclesUntilTRANS > 0 ||
171 CyclesUntilVALU > 0;
172 if (HasState) {
173 dbgs() << " Scheduler AdvanceCycle:";
174 if (CurrentCoExecStage.has_value()) {
175 unsigned Stage = *CurrentCoExecStage;
176 unsigned Next = Stage + 1;
177 if (Next >= ActiveCoExecInfo.TotalWindow)
178 dbgs() << " stage " << Stage << "->expired";
179 else
180 dbgs() << " stage " << Stage << "->" << Next;
181 }
182 if (CyclesUntilTRANS > 0)
183 dbgs() << " TRANS=" << CyclesUntilTRANS << "->"
184 << (CyclesUntilTRANS - 1);
185 if (CyclesUntilVALU > 0)
186 dbgs() << " VALU=" << CyclesUntilVALU << "->" << (CyclesUntilVALU - 1);
187 dbgs() << "\n";
188 }
189 });
190
191 // Decrement hazard counters.
192 if (CyclesUntilTRANS > 0)
193 --CyclesUntilTRANS;
194 if (CyclesUntilVALU > 0)
195 --CyclesUntilVALU;
196
197 // Advance WMMA co-execution window.
198 if (CurrentCoExecStage.has_value()) {
199 unsigned Stage = *CurrentCoExecStage + 1;
200 if (Stage >= ActiveCoExecInfo.TotalWindow) {
201 // Window expired.
202 LLVM_DEBUG({
203 dbgs() << " CoExec window complete:\n";
204 dumpCoExecWindow();
205 });
206 CurrentCoExecStage = std::nullopt;
207 } else {
208 CurrentCoExecStage = Stage;
209 }
210 }
211}
212
213bool GCNHazardRecognizer::hasCoExecWindowModel() const {
214 // The co-execution slot patterns returned by getCoExecInfo() are derived from
215 // gfx1250 timings, so the window model is restricted to gfx1250 for now.
216 // gfx1251 and gfx12.5-generic report the same co-execution hazard features
217 // but have different WMMA latencies, so they need their own slot patterns
218 // before they can be modeled here.
219 if (ST.hasWMMACoexecutionHazards() && ST.hasTransCoexecutionHazard() &&
220 AMDGPU::isGFX1250(STI: ST))
221 return true;
222
223 if (ST.hasGFX950Insts() &&
224 AMDGPU::getSchedStrategy(F: MF.getFunction()) == "coexec")
225 return true;
226
227 return false;
228}
229
230void GCNHazardRecognizer::updateWMMAWindowState(const MachineInstr &MI) {
231 if (!hasCoExecWindowModel())
232 return;
233
234 if (!SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI) &&
235 !SIInstrInfo::isMFMA(MI))
236 return;
237
238 // If a previous window was still active, dump it before starting a new one.
239 // Record the current stage (filled by this new WMMA) before dumping.
240 LLVM_DEBUG({
241 if (CurrentCoExecStage.has_value()) {
242 unsigned Stage = *CurrentCoExecStage;
243 if (Stage < AMDGPU::MaxCoExecStages)
244 CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
245 dbgs() << " CoExec window interrupted at stage " << Stage << ":\n";
246 dumpCoExecWindow();
247 }
248 });
249
250 // Start a new co-execution window.
251 ActiveCoExecInfo = AMDGPU::getCoExecInfo(MI, TII);
252 CurrentCoExecStage = 0;
253 CoExecWindowLog.fill(u: '.');
254
255 LLVM_DEBUG(dbgs() << " WMMA window started: " << ActiveCoExecInfo.Pattern
256 << " (window=" << ActiveCoExecInfo.TotalWindow << ")\n"
257 << " " << MI);
258}
259
260void GCNHazardRecognizer::updateTRANSState(const MachineInstr &MI) {
261 if (!hasCoExecWindowModel())
262 return;
263 if (!SIInstrInfo::isTRANS(MI))
264 return;
265
266 // Back-to-back TRANS instructions have a 1-cycle hazard.
267 // This is checked via checkTRANSHazard() and does not create a co-exec
268 // window. The TRANS shadow slot allows anything except TRANS and
269 // multi-cycle VALU.
270 // Set to 2: bumpCycle advances to the next pick's cycle (decrementing
271 // by 1 via AdvanceCycle) before the next instruction's hazard check, so
272 // the counter is observed at 1 there. That 1-cycle stall lets the
273 // strategy pick a non-TRANS, non-multi-cycle-VALU candidate to fill the
274 // shadow slot.
275 CyclesUntilTRANS = 2;
276 LLVM_DEBUG(dbgs() << " TRANS hazard set: CyclesUntilTRANS=2\n");
277}
278
279void GCNHazardRecognizer::updateMultiCycleVALUState(const MachineInstr &MI) {
280 if (!hasCoExecWindowModel())
281 return;
282 // Multi-cycle VALU (CVT, etc.) blocks subsequent VALU for repeat rate cycles.
283 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
284 return;
285
286 // Skip WMMA, MFMA, and TRANS - they have their own tracking.
287 if (SIInstrInfo::isWMMA(MI) || SIInstrInfo::isSWMMAC(MI) ||
288 SIInstrInfo::isMFMA(MI) || SIInstrInfo::isTRANS(MI))
289 return;
290
291 unsigned RepeatRate = TII.getRepeatRate(MI);
292 if (RepeatRate > 1) {
293 // bumpCycle's AdvanceCycle decrements once before the next pick's
294 // hazard check (same convention as CyclesUntilTRANS), so to expose
295 // RepeatRate-1 cycles of shadow we must seed with RepeatRate.
296 CyclesUntilVALU = RepeatRate;
297 LLVM_DEBUG(dbgs() << " Multi-cycle VALU: repeat=" << RepeatRate
298 << ", CyclesUntilVALU=" << CyclesUntilVALU << "\n");
299 }
300}
301
302AMDGPU::CoExecMaskT
303GCNHazardRecognizer::getCoExecMaskForMI(const MachineInstr &MI,
304 const SIInstrInfo &TII) {
305 return AMDGPU::getCoExecMask(F: AMDGPU::classifyFlavor(MI, SII: TII));
306}
307
308unsigned GCNHazardRecognizer::checkTRANSHazard(const MachineInstr &MI) const {
309 if (!CyclesUntilTRANS)
310 return 0;
311
312 // Only TRANS and multi-cycle VALU are blocked by the TRANS shadow.
313 if (SIInstrInfo::isTRANS(MI))
314 return CyclesUntilTRANS;
315
316 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
317 !SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI) &&
318 TII.getRepeatRate(MI) > 1)
319 return CyclesUntilTRANS;
320
321 return 0;
322}
323
324unsigned
325GCNHazardRecognizer::checkMultiCycleVALUHazard(const MachineInstr &MI) const {
326 if (!CyclesUntilVALU)
327 return 0;
328
329 // Multi-cycle VALU blocks anything on the VALU pipe - VALU, WMMA, SWMMAC,
330 // and TRANS - for RepeatRate-1 cycles. Only off-pipe instructions (MEM,
331 // SALU, control) can fill the shadow.
332 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
333 !SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI) &&
334 !SIInstrInfo::isTRANS(MI))
335 return 0;
336
337 return CyclesUntilVALU;
338}
339
340unsigned
341GCNHazardRecognizer::checkWMMACoexecSlot(const MachineInstr &MI) const {
342 // No hazard if not in a WMMA window.
343 if (!CurrentCoExecStage.has_value())
344 return 0;
345
346 unsigned Stage = *CurrentCoExecStage;
347 AMDGPU::CoExecMaskT InstMask = getCoExecMaskForMI(MI, TII);
348 unsigned StallCycles = ActiveCoExecInfo.getStallCycles(InstMask, Stage);
349
350 // No stall required if the instruction can co-execute at the current stage.
351 if (StallCycles == 0)
352 return 0;
353
354 // Stall for the required number of cycles until the next allowed stage.
355 unsigned NextStage = Stage + StallCycles;
356 if (NextStage < ActiveCoExecInfo.TotalWindow) {
357 DEBUG_WITH_TYPE(
358 DEBUG_TYPE_VERBOSE,
359 dbgs() << " CoExec stall: stage=" << Stage << "("
360 << AMDGPU::getStageTypeName(ActiveCoExecInfo.getType(Stage))
361 << ") mask=" << AMDGPU::getCoExecMaskName(InstMask)
362 << " -> stall " << StallCycles << " (next allowed=" << NextStage
363 << ")\n"
364 << " " << MI);
365 return StallCycles;
366 }
367
368 // No compatible slot in window - stall until window ends.
369 DEBUG_WITH_TYPE(
370 DEBUG_TYPE_VERBOSE,
371 dbgs() << " CoExec stall: stage=" << Stage << "("
372 << AMDGPU::getStageTypeName(ActiveCoExecInfo.getType(Stage))
373 << ") mask=" << AMDGPU::getCoExecMaskName(InstMask) << " -> stall "
374 << StallCycles << " (window ends)\n"
375 << " " << MI);
376 return StallCycles;
377}
378
379unsigned
380GCNHazardRecognizer::checkMultiShadowHazard(const MachineInstr &MI) const {
381 // This models a VALU caught in both a WMMA and a TRANS shadow.
382 if (!hasCoExecWindowModel())
383 return 0;
384
385 // No hazard if not in a WMMA window.
386 if (!CurrentCoExecStage.has_value())
387 return 0;
388
389 if (!CyclesUntilTRANS)
390 return 0;
391
392 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ||
393 SIInstrInfo::isLDSDMA(MI))
394 return 0;
395
396 // We have a VALU instruction that is under both a TRANS and WMMA shadow.
397 // We need to wait for at least one to clear.
398
399 unsigned LookAheadStage = *CurrentCoExecStage + CyclesUntilTRANS;
400 AMDGPU::CoExecMaskT InstMask = getCoExecMaskForMI(MI, TII);
401 return CyclesUntilTRANS +
402 ActiveCoExecInfo.getStallCycles(InstMask, Stage: LookAheadStage);
403}
404
405void GCNHazardRecognizer::schedulerEmitInstruction(MachineInstr *MI) {
406 LLVM_DEBUG({
407 bool InWindow = CurrentCoExecStage.has_value();
408 bool HasActiveState =
409 InWindow || CyclesUntilTRANS > 0 || CyclesUntilVALU > 0;
410 if (HasActiveState) {
411 if (InWindow) {
412 unsigned Stage = *CurrentCoExecStage;
413 dbgs() << " Stage " << Stage << "("
414 << AMDGPU::getStageTypeName(ActiveCoExecInfo.getType(Stage))
415 << ") Emit ["
416 << AMDGPU::getCoExecMaskName(getCoExecMaskForMI(*MI, TII))
417 << "]: " << *MI;
418 } else {
419 dbgs() << " Emit ["
420 << AMDGPU::getCoExecMaskName(getCoExecMaskForMI(*MI, TII))
421 << "]: " << *MI;
422 }
423 }
424 });
425 DEBUG_WITH_TYPE(DEBUG_TYPE_VERBOSE, {
426 bool HasActiveState = CurrentCoExecStage.has_value() ||
427 CyclesUntilTRANS > 0 || CyclesUntilVALU > 0;
428 if (!HasActiveState)
429 dbgs() << " Emit ["
430 << AMDGPU::getCoExecMaskName(getCoExecMaskForMI(*MI, TII))
431 << "]: " << *MI;
432 });
433 updateWMMAWindowState(MI: *MI);
434 updateTRANSState(MI: *MI);
435 updateMultiCycleVALUState(MI: *MI);
436}
437
438void GCNHazardRecognizer::EmitInstruction(SUnit *SU) {
439 EmitInstruction(MI: SU->getInstr());
440}
441
442void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) {
443 CurrCycleInstr = MI;
444 if (isSchedulerMode())
445 schedulerEmitInstruction(MI);
446}
447
448static bool isDivFMas(unsigned Opcode) {
449 return Opcode == AMDGPU::V_DIV_FMAS_F32_e64 || Opcode == AMDGPU::V_DIV_FMAS_F64_e64;
450}
451
452static bool isSGetReg(unsigned Opcode) {
453 return Opcode == AMDGPU::S_GETREG_B32 || Opcode == AMDGPU::S_GETREG_B32_const;
454}
455
456static bool isSSetReg(unsigned Opcode) {
457 switch (Opcode) {
458 case AMDGPU::S_SETREG_B32:
459 case AMDGPU::S_SETREG_B32_mode:
460 case AMDGPU::S_SETREG_IMM32_B32:
461 case AMDGPU::S_SETREG_IMM32_B32_mode:
462 return true;
463 }
464 return false;
465}
466
467static bool isRWLane(unsigned Opcode) {
468 return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32;
469}
470
471static bool isRFE(unsigned Opcode) {
472 return Opcode == AMDGPU::S_RFE_B64;
473}
474
475static bool isSMovRel(unsigned Opcode) {
476 switch (Opcode) {
477 case AMDGPU::S_MOVRELS_B32:
478 case AMDGPU::S_MOVRELS_B64:
479 case AMDGPU::S_MOVRELD_B32:
480 case AMDGPU::S_MOVRELD_B64:
481 return true;
482 default:
483 return false;
484 }
485}
486
487static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII,
488 const MachineInstr &MI) {
489 if (TII.isAlwaysGDS(Opcode: MI.getOpcode()))
490 return true;
491
492 switch (MI.getOpcode()) {
493 case AMDGPU::S_SENDMSG:
494 case AMDGPU::S_SENDMSGHALT:
495 case AMDGPU::S_TTRACEDATA:
496 return true;
497 // These DS opcodes don't support GDS.
498 case AMDGPU::DS_NOP:
499 case AMDGPU::DS_PERMUTE_B32:
500 case AMDGPU::DS_BPERMUTE_B32:
501 return false;
502 default:
503 if (TII.isDS(Opcode: MI.getOpcode())) {
504 int GDS = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
505 Name: AMDGPU::OpName::gds);
506 if (MI.getOperand(i: GDS).getImm())
507 return true;
508 }
509 return false;
510 }
511}
512
513static bool isPermlane(const MachineInstr &MI) {
514 unsigned Opcode = MI.getOpcode();
515 return Opcode == AMDGPU::V_PERMLANE16_B32_e64 ||
516 Opcode == AMDGPU::V_PERMLANE64_B32 ||
517 Opcode == AMDGPU::V_PERMLANEX16_B32_e64 ||
518 Opcode == AMDGPU::V_PERMLANE16_VAR_B32_e64 ||
519 Opcode == AMDGPU::V_PERMLANEX16_VAR_B32_e64 ||
520 Opcode == AMDGPU::V_PERMLANE16_SWAP_B32_e32 ||
521 Opcode == AMDGPU::V_PERMLANE16_SWAP_B32_e64 ||
522 Opcode == AMDGPU::V_PERMLANE32_SWAP_B32_e32 ||
523 Opcode == AMDGPU::V_PERMLANE32_SWAP_B32_e64 ||
524 Opcode == AMDGPU::V_PERMLANE_BCAST_B32_e64 ||
525 Opcode == AMDGPU::V_PERMLANE_UP_B32_e64 ||
526 Opcode == AMDGPU::V_PERMLANE_DOWN_B32_e64 ||
527 Opcode == AMDGPU::V_PERMLANE_XOR_B32_e64 ||
528 Opcode == AMDGPU::V_PERMLANE_IDX_GEN_B32_e64;
529}
530
531static bool isLdsDma(const MachineInstr &MI) {
532 return SIInstrInfo::isLDSDMA(MI);
533}
534
535static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) {
536 const MachineOperand *RegOp = TII->getNamedOperand(MI: RegInstr,
537 OperandName: AMDGPU::OpName::simm16);
538 return std::get<0>(t: AMDGPU::Hwreg::HwregEncoding::decode(Encoded: RegOp->getImm()));
539}
540
541ScheduleHazardRecognizer::HazardType
542GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
543 MachineInstr *MI = SU->getInstr();
544 // If we are not in "HazardRecognizerMode" and therefore not being run from
545 // the scheduler, track possible stalls from hazards but don't insert noops.
546 auto HazardType = isHazardRecognizerMode() ? NoopHazard : Hazard;
547
548 if (MI->isBundle())
549 return NoHazard;
550
551 // Check co-execution slot hazards and pipeline stalls in scheduler modes.
552 if (isSchedulerMode()) {
553 if (checkMultiShadowHazard(MI: *MI) > 0)
554 return Hazard;
555 if (checkWMMACoexecSlot(MI: *MI) > 0)
556 return Hazard;
557 if (checkTRANSHazard(MI: *MI) > 0)
558 return Hazard;
559 if (checkMultiCycleVALUHazard(MI: *MI) > 0)
560 return Hazard;
561 // The remaining checks are all defined by register dependences.
562 if (!hasPhysRegs())
563 return NoHazard;
564 }
565
566 if (SIInstrInfo::isSMRD(MI: *MI) && checkSMRDHazards(SMRD: MI) > 0)
567 return HazardType;
568
569 if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0)
570 return HazardType;
571
572 if (checkFPAtomicToDenormModeHazard(MI) > 0)
573 return HazardType;
574
575 // Hazards which cannot be mitigated with S_NOPs.
576 if (!isHazardRecognizerMode()) {
577 if (checkWMMACoexecutionHazards(MI) > 0) {
578 HasPendingWMMACoexecHazard = true;
579 return Hazard;
580 }
581 }
582
583 if (ST.hasNoDataDepHazard())
584 return NoHazard;
585
586 if (SIInstrInfo::isVMEM(MI: *MI) && checkVMEMHazards(VMEM: MI) > 0)
587 return HazardType;
588
589 if (SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true) &&
590 checkVALUHazards(VALU: MI) > 0)
591 return HazardType;
592
593 if (SIInstrInfo::isDPP(MI: *MI) && checkDPPHazards(DPP: MI) > 0)
594 return HazardType;
595
596 if (isDivFMas(Opcode: MI->getOpcode()) && checkDivFMasHazards(DivFMas: MI) > 0)
597 return HazardType;
598
599 if (isRWLane(Opcode: MI->getOpcode()) && checkRWLaneHazards(RWLane: MI) > 0)
600 return HazardType;
601
602 if ((SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true) ||
603 SIInstrInfo::isVMEM(MI: *MI) || SIInstrInfo::isDS(MI: *MI) ||
604 SIInstrInfo::isEXP(MI: *MI)) &&
605 checkMAIVALUHazards(MI) > 0)
606 return HazardType;
607
608 if (isSGetReg(Opcode: MI->getOpcode()) && checkGetRegHazards(GetRegInstr: MI) > 0)
609 return HazardType;
610
611 if (isSSetReg(Opcode: MI->getOpcode()) && checkSetRegHazards(SetRegInstr: MI) > 0)
612 return HazardType;
613
614 if (isRFE(Opcode: MI->getOpcode()) && checkRFEHazards(RFE: MI) > 0)
615 return HazardType;
616
617 if (((ST.hasReadM0MovRelInterpHazard() &&
618 (TII.isVINTRP(MI: *MI) || isSMovRel(Opcode: MI->getOpcode()) ||
619 MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 ||
620 MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) ||
621 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, MI: *MI)) ||
622 (ST.hasReadM0LdsDmaHazard() && isLdsDma(MI: *MI)) ||
623 (ST.hasReadM0LdsDirectHazard() &&
624 MI->readsRegister(Reg: AMDGPU::LDS_DIRECT, /*TRI=*/nullptr))) &&
625 checkReadM0Hazards(SMovRel: MI) > 0)
626 return HazardType;
627
628 if (SIInstrInfo::isMAI(MI: *MI) && checkMAIHazards(MI) > 0)
629 return HazardType;
630
631 if ((SIInstrInfo::isVMEM(MI: *MI) || SIInstrInfo::isDS(MI: *MI)) &&
632 checkMAILdStHazards(MI) > 0)
633 return HazardType;
634
635 if (ST.hasPermlaneForwardingHazard() && isPermlane(MI: *MI) &&
636 checkPermlaneHazards(MI) > 0)
637 return HazardType;
638
639 if (MI->isInlineAsm() && checkInlineAsmHazards(IA: MI) > 0)
640 return HazardType;
641
642 return NoHazard;
643}
644
645static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII,
646 unsigned Quantity) {
647 while (Quantity > 0) {
648 unsigned Arg = std::min(a: Quantity, b: 8u);
649 Quantity -= Arg;
650 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_NOP))
651 .addImm(Val: Arg - 1);
652 }
653}
654
655unsigned
656GCNHazardRecognizer::getMFMAPipelineWaitStates(const MachineInstr &MI) const {
657 const MCSchedClassDesc *SC = TSchedModel.resolveSchedClass(MI: &MI);
658 assert(TSchedModel.getWriteProcResBegin(SC) !=
659 TSchedModel.getWriteProcResEnd(SC));
660 return TSchedModel.getWriteProcResBegin(SC)->ReleaseAtCycle;
661}
662
663void GCNHazardRecognizer::processBundle() {
664 MachineBasicBlock::instr_iterator MI = std::next(x: CurrCycleInstr->getIterator());
665 MachineBasicBlock::instr_iterator E = CurrCycleInstr->getParent()->instr_end();
666 // Check bundled MachineInstr's for hazards.
667 for (; MI != E && MI->isInsideBundle(); ++MI) {
668 CurrCycleInstr = &*MI;
669 unsigned WaitStates = PreEmitNoopsCommon(CurrCycleInstr);
670
671 if (isHazardRecognizerMode()) {
672 fixHazards(MI: CurrCycleInstr);
673
674 insertNoopsInBundle(MI: CurrCycleInstr, TII, Quantity: WaitStates);
675 }
676
677 // It’s unnecessary to track more than MaxLookAhead instructions. Since we
678 // include the bundled MI directly after, only add a maximum of
679 // (MaxLookAhead - 1) noops to EmittedInstrs.
680 for (unsigned i = 0, e = std::min(a: WaitStates, b: MaxLookAhead - 1); i < e; ++i)
681 EmittedInstrs.push_front(x: nullptr);
682
683 EmittedInstrs.push_front(x: CurrCycleInstr);
684 EmittedInstrs.resize(new_size: MaxLookAhead);
685 }
686 CurrCycleInstr = nullptr;
687}
688
689void GCNHazardRecognizer::runOnInstruction(MachineInstr *MI) {
690 assert(isHazardRecognizerMode());
691
692 unsigned NumPreNoops = PreEmitNoops(MI);
693 EmitNoops(Quantity: NumPreNoops);
694 if (MI->isInsideBundle())
695 insertNoopsInBundle(MI, TII, Quantity: NumPreNoops);
696 else
697 TII.insertNoops(MBB&: *MI->getParent(), MI: MachineBasicBlock::iterator(MI),
698 Quantity: NumPreNoops);
699 EmitInstruction(MI);
700 AdvanceCycle();
701}
702
703unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
704 assert(isHazardRecognizerMode());
705 CurrCycleInstr = MI;
706 unsigned W = PreEmitNoopsCommon(MI);
707 fixHazards(MI);
708 CurrCycleInstr = nullptr;
709 return std::max(a: W, b: NopPadding.getValue());
710}
711
712unsigned GCNHazardRecognizer::getHazardWaitStates(MachineInstr *MI) const {
713 unsigned W = 0;
714
715 // Check co-execution slot hazards and pipeline stalls in scheduler modes.
716 if (isSchedulerMode()) {
717 W = checkWMMACoexecSlot(MI: *MI);
718 W = std::max(a: W, b: checkTRANSHazard(MI: *MI));
719 W = std::max(a: W, b: checkMultiCycleVALUHazard(MI: *MI));
720 W = std::max(a: W, b: checkMultiShadowHazard(MI: *MI));
721 // The remaining checks are all defined by register dependences.
722 if (!hasPhysRegs())
723 return W;
724 }
725
726 return std::max(a: W, b: PreEmitNoopsCommon(MI));
727}
728
729unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) const {
730 if (MI->isBundle())
731 return 0;
732
733 int WaitStates = 0;
734
735 if (SIInstrInfo::isSMRD(MI: *MI))
736 return std::max(a: WaitStates, b: checkSMRDHazards(SMRD: MI));
737
738 if (ST.hasNSAtoVMEMBug())
739 WaitStates = std::max(a: WaitStates, b: checkNSAtoVMEMHazard(MI));
740
741 WaitStates = std::max(a: WaitStates, b: checkFPAtomicToDenormModeHazard(MI));
742
743 if (ST.hasNoDataDepHazard())
744 return WaitStates;
745
746 if (SIInstrInfo::isVMEM(MI: *MI))
747 WaitStates = std::max(a: WaitStates, b: checkVMEMHazards(VMEM: MI));
748
749 if (SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true))
750 WaitStates = std::max(a: WaitStates, b: checkVALUHazards(VALU: MI));
751
752 if (SIInstrInfo::isDPP(MI: *MI))
753 WaitStates = std::max(a: WaitStates, b: checkDPPHazards(DPP: MI));
754
755 if (isDivFMas(Opcode: MI->getOpcode()))
756 WaitStates = std::max(a: WaitStates, b: checkDivFMasHazards(DivFMas: MI));
757
758 if (isRWLane(Opcode: MI->getOpcode()))
759 WaitStates = std::max(a: WaitStates, b: checkRWLaneHazards(RWLane: MI));
760
761 if (SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true) ||
762 SIInstrInfo::isVMEM(MI: *MI) || SIInstrInfo::isDS(MI: *MI) ||
763 SIInstrInfo::isEXP(MI: *MI))
764 WaitStates = std::max(a: WaitStates, b: checkMAIVALUHazards(MI));
765
766 if (MI->isInlineAsm())
767 return std::max(a: WaitStates, b: checkInlineAsmHazards(IA: MI));
768
769 if (isSGetReg(Opcode: MI->getOpcode()))
770 return std::max(a: WaitStates, b: checkGetRegHazards(GetRegInstr: MI));
771
772 if (isSSetReg(Opcode: MI->getOpcode()))
773 return std::max(a: WaitStates, b: checkSetRegHazards(SetRegInstr: MI));
774
775 if (isRFE(Opcode: MI->getOpcode()))
776 return std::max(a: WaitStates, b: checkRFEHazards(RFE: MI));
777
778 if ((ST.hasReadM0MovRelInterpHazard() &&
779 (TII.isVINTRP(MI: *MI) || isSMovRel(Opcode: MI->getOpcode()) ||
780 MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 ||
781 MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) ||
782 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, MI: *MI)) ||
783 (ST.hasReadM0LdsDmaHazard() && isLdsDma(MI: *MI)) ||
784 (ST.hasReadM0LdsDirectHazard() &&
785 MI->readsRegister(Reg: AMDGPU::LDS_DIRECT, /*TRI=*/nullptr)))
786 return std::max(a: WaitStates, b: checkReadM0Hazards(SMovRel: MI));
787
788 if (SIInstrInfo::isMAI(MI: *MI))
789 return std::max(a: WaitStates, b: checkMAIHazards(MI));
790
791 if (SIInstrInfo::isVMEM(MI: *MI) || SIInstrInfo::isDS(MI: *MI))
792 return std::max(a: WaitStates, b: checkMAILdStHazards(MI));
793
794 if (ST.hasPermlaneForwardingHazard() && isPermlane(MI: *MI))
795 return std::max(a: WaitStates, b: checkPermlaneHazards(MI));
796
797 return WaitStates;
798}
799
800void GCNHazardRecognizer::EmitNoop() {
801 EmittedInstrs.push_front(x: nullptr);
802}
803
804void GCNHazardRecognizer::AdvanceCycle() {
805 if (isSchedulerMode())
806 schedulerAdvanceCycle();
807
808 // When the scheduler detects a stall, it will call AdvanceCycle() without
809 // emitting any instructions.
810 if (!CurrCycleInstr) {
811 EmittedInstrs.push_front(x: nullptr);
812
813 if (HasPendingWMMACoexecHazard)
814 EmittedVALUInstrs.push_front(x: nullptr);
815 return;
816 }
817
818 HasPendingWMMACoexecHazard = false;
819
820 if (CurrCycleInstr->isBundle()) {
821 processBundle();
822 return;
823 }
824
825 unsigned NumWaitStates = TII.getNumWaitStates(MI: *CurrCycleInstr);
826 if (!NumWaitStates) {
827 CurrCycleInstr = nullptr;
828 return;
829 }
830
831 // Keep track of emitted instructions
832 EmittedInstrs.push_front(x: CurrCycleInstr);
833
834 bool IsVALUOrWMMA =
835 SIInstrInfo::isVALU(MI: *CurrCycleInstr, /*AllowLDSDMA=*/true) ||
836 SIInstrInfo::isWMMA(MI: *CurrCycleInstr) ||
837 SIInstrInfo::isSWMMAC(MI: *CurrCycleInstr);
838 if (IsVALUOrWMMA) {
839 EmittedVALUInstrs.push_front(x: CurrCycleInstr);
840 } else {
841 // A pending WMMA co-execution hazard optimistically records stall cycles as
842 // future V_NOPs. If the scheduler instead stalls for a different
843 // (S_NOP-resolvable) hazard and schedules a non-VALU into those cycles,
844 // they will not resolve the VALU-pipe hazard, so drop them here.
845 while (!EmittedVALUInstrs.empty() && EmittedVALUInstrs.front() == nullptr)
846 EmittedVALUInstrs.pop_front();
847 }
848
849 // Add a nullptr for each additional wait state after the first. Make sure
850 // not to add more than getMaxLookAhead() items to the list, since we
851 // truncate the list to that size right after this loop.
852 for (unsigned i = 1, e = std::min(a: NumWaitStates, b: getMaxLookAhead());
853 i < e; ++i) {
854 EmittedInstrs.push_front(x: nullptr);
855 }
856
857 // getMaxLookahead() is the largest number of wait states we will ever need
858 // to insert, so there is no point in keeping track of more than that many
859 // wait states.
860 EmittedInstrs.resize(new_size: getMaxLookAhead());
861 if (EmittedVALUInstrs.size() > MaxVALULookAhead)
862 EmittedVALUInstrs.resize(new_size: MaxVALULookAhead);
863
864 CurrCycleInstr = nullptr;
865}
866
867void GCNHazardRecognizer::RecedeCycle() {
868 assert(!isHazardRecognizerMode() &&
869 "Bottom-up scheduling shouldn't run in hazard recognizer mode");
870}
871
872//===----------------------------------------------------------------------===//
873// Helper Functions
874//===----------------------------------------------------------------------===//
875
876enum HazardFnResult { HazardFound, HazardExpired, NoHazardFound };
877
878// Search for a hazard in a block and its predecessors.
879template <typename StateT>
880static bool
881hasHazard(StateT InitialState,
882 function_ref<HazardFnResult(StateT &, const MachineInstr &)> IsHazard,
883 function_ref<void(StateT &, const MachineInstr &)> UpdateState,
884 const MachineBasicBlock *InitialMBB,
885 MachineBasicBlock::const_reverse_instr_iterator InitialI) {
886 struct StateMapKey {
887 SmallVectorImpl<StateT> *States;
888 size_t Idx;
889 static bool isEqual(const StateMapKey &LHS, const StateMapKey &RHS) {
890 return LHS.States == RHS.States && LHS.Idx == RHS.Idx;
891 }
892 };
893 struct StateMapKeyTraits : DenseMapInfo<StateMapKey> {
894 static unsigned getHashValue(const StateMapKey &Key) {
895 return StateT::getHashValue((*Key.States)[Key.Idx]);
896 }
897 static unsigned getHashValue(const StateT &State) {
898 return StateT::getHashValue(State);
899 }
900 static bool isEqual(const StateMapKey &LHS, const StateMapKey &RHS) {
901 return StateT::isEqual((*LHS.States)[LHS.Idx], (*RHS.States)[RHS.Idx]);
902 }
903 static bool isEqual(const StateT &LHS, const StateMapKey &RHS) {
904 return StateT::isEqual(LHS, (*RHS.States)[RHS.Idx]);
905 }
906 };
907
908 SmallDenseMap<StateMapKey, size_t, 8, StateMapKeyTraits> StateMap;
909 SmallVector<StateT, 8> States;
910
911 MachineBasicBlock::const_reverse_instr_iterator I = InitialI;
912 const MachineBasicBlock *MBB = InitialMBB;
913 StateT State = InitialState;
914
915 SmallSetVector<std::pair<const MachineBasicBlock *, size_t>, 16> Worklist;
916 unsigned WorkIdx = 0;
917 for (;;) {
918 bool Expired = false;
919 for (auto E = MBB->instr_rend(); I != E; ++I) {
920 // No need to look at parent BUNDLE instructions.
921 if (I->isBundle())
922 continue;
923
924 auto Result = IsHazard(State, *I);
925 if (Result == HazardFound)
926 return true;
927 if (Result == HazardExpired) {
928 Expired = true;
929 break;
930 }
931
932 if (I->isInlineAsm() || I->isMetaInstruction())
933 continue;
934
935 UpdateState(State, *I);
936 }
937
938 if (!Expired) {
939 size_t StateIdx = States.size();
940 StateMapKey Key = {&States, StateIdx};
941 auto Insertion = StateMap.insert_as(std::pair(Key, StateIdx), State);
942 if (Insertion.second) {
943 States.emplace_back(State);
944 } else {
945 StateIdx = Insertion.first->second;
946 }
947 for (MachineBasicBlock *Pred : MBB->predecessors())
948 Worklist.insert(X: std::pair(Pred, StateIdx));
949 }
950
951 if (WorkIdx == Worklist.size())
952 break;
953
954 size_t StateIdx;
955 std::tie(args&: MBB, args&: StateIdx) = Worklist[WorkIdx++];
956 State = States[StateIdx];
957 I = MBB->instr_rbegin();
958 }
959
960 return false;
961}
962
963// Number of trailing terminators of \p Pred not executed on the edge to \p MBB:
964// control leaves at the first terminator branching to \p MBB, so the ones after
965// it are dead on that edge. A fallthrough executes them all.
966static unsigned countUnexecutedTerminators(const MachineBasicBlock &Pred,
967 const MachineBasicBlock *MBB) {
968 const SIInstrInfo &TII =
969 *Pred.getParent()->getSubtarget<GCNSubtarget>().getInstrInfo();
970 unsigned Unexecuted = 0, Passed = 0;
971 for (auto I = Pred.instr_rbegin(), E = Pred.instr_rend();
972 I != E && I->isTerminator(); ++I, ++Passed) {
973 // Only an explicit MBB destination identifies this edge; the last match is
974 // the earliest such branch in program order.
975 if (I->isBranch() && I->getNumOperands() > 0 && I->getOperand(i: 0).isMBB() &&
976 TII.getBranchDestBlock(MI: *I) == MBB)
977 Unexecuted = Passed;
978 }
979 return Unexecuted;
980}
981
982// Wait states \p MI adds to a distance. Inline asm may be empty, so it adds
983// none, which is the worst case.
984static unsigned
985getCountedWaitStates(const MachineInstr &MI,
986 GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates) {
987 return MI.isInlineAsm() ? 0 : GetNumWaitStates(MI);
988}
989
990namespace {
991
992// Answers GCNHazardRecognizer::getMax*WindowDeficit, which documents the
993// result, by walking the CFG backwards from the instruction being checked,
994// counting distances with GetNumWaitStates. That needs the instruction in its
995// final place, so hazard recognizer mode only. Each object answers one query.
996class WindowDeficitSearch {
997public:
998 WindowDeficitSearch(
999 function_ref<std::optional<int>(const MachineInstr &)> WindowFor,
1000 int MaxWindow,
1001 GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates =
1002 SIInstrInfo::getNumWaitStates)
1003 : WindowFor(WindowFor), GetNumWaitStates(GetNumWaitStates),
1004 MaxWindow(MaxWindow) {}
1005
1006 int run(const MachineInstr &MI);
1007
1008private:
1009 // A block to scan upwards from Start, which is Distance away from the
1010 // instruction being checked. In a predecessor, Start is the last instruction
1011 // above the terminators, and Distance includes the ones the edge executes.
1012 struct Arrival {
1013 const MachineBasicBlock *MBB;
1014 MachineBasicBlock::const_reverse_instr_iterator Start;
1015 int Distance;
1016 };
1017
1018 void arrive(const MachineBasicBlock *MBB,
1019 MachineBasicBlock::const_reverse_instr_iterator Start,
1020 int Distance);
1021 void scan(Arrival A);
1022
1023 const function_ref<std::optional<int>(const MachineInstr &)> WindowFor;
1024 const GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates;
1025 const int MaxWindow;
1026 int Deficit = 0;
1027 // Pending[D] holds the arrivals at distance D, and Best the smallest distance
1028 // each block has been queued at. Buckets are emptied in increasing distance,
1029 // so the first arrival scanned at a block is the shortest way into it, and
1030 // any later one is stale.
1031 SmallVector<SmallVector<Arrival, 2>, 4> Pending;
1032 DenseMap<const MachineBasicBlock *, int> Best;
1033};
1034
1035} // end anonymous namespace
1036
1037void WindowDeficitSearch::arrive(
1038 const MachineBasicBlock *MBB,
1039 MachineBasicBlock::const_reverse_instr_iterator Start, int Distance) {
1040 // No window exceeds MaxWindow, so nothing this far back can beat the deficit
1041 // already found.
1042 if (MaxWindow - Distance <= Deficit)
1043 return;
1044
1045 if (Pending.empty())
1046 Pending.resize(N: MaxWindow);
1047
1048 auto [It, Inserted] = Best.try_emplace(Key: MBB, Args&: Distance);
1049 if (!Inserted) {
1050 if (It->second <= Distance)
1051 return;
1052 It->second = Distance;
1053 }
1054
1055 assert(Distance < MaxWindow && "arrival outside the window");
1056 Pending[Distance].push_back(Elt: {.MBB: MBB, .Start: Start, .Distance: Distance});
1057}
1058
1059// \p A is a copy because arrive() may append to the bucket it came from, which
1060// can move that bucket's elements.
1061void WindowDeficitSearch::scan(Arrival A) {
1062 // A shorter way into this block has been queued since.
1063 if (Best.lookup(Val: A.MBB) < A.Distance)
1064 return;
1065
1066 int Distance = A.Distance;
1067 for (auto I = A.Start, E = A.MBB->instr_rend(); I != E; ++I) {
1068 if (I->isBundle())
1069 continue;
1070
1071 if (std::optional<int> Window = WindowFor(*I)) {
1072 assert(*Window >= 0 && *Window <= MaxWindow && "window out of bounds");
1073 Deficit = std::max(a: Deficit, b: *Window - Distance);
1074 }
1075
1076 Distance += getCountedWaitStates(MI: *I, GetNumWaitStates);
1077
1078 if (Distance >= MaxWindow)
1079 return;
1080 }
1081
1082 for (const MachineBasicBlock *Pred : A.MBB->predecessors()) {
1083 auto Start = Pred->instr_rbegin(), End = Pred->instr_rend();
1084 for (unsigned N = countUnexecutedTerminators(Pred: *Pred, MBB: A.MBB); N; --N)
1085 ++Start;
1086
1087 int Executed = 0;
1088 for (; Start != End && Start->isTerminator(); ++Start) {
1089 assert(!WindowFor(*Start) && "terminators must not ask for a window");
1090 Executed += getCountedWaitStates(MI: *Start, GetNumWaitStates);
1091 }
1092
1093 arrive(MBB: Pred, Start, Distance: Distance + Executed);
1094 }
1095}
1096
1097int WindowDeficitSearch::run(const MachineInstr &MI) {
1098 // Only the part above MI is scanned here, so the block stays out of Best: a
1099 // backedge into it must still scan it from its end.
1100 scan(A: {.MBB: MI.getParent(), .Start: std::next(x: MI.getReverseIterator()), .Distance: 0});
1101
1102 // Distances never shrink along a path, so a scan only queues arrivals at the
1103 // distance being swept or further, and one pass over the buckets is enough.
1104 // The inner loop re-reads the size because an edge that adds nothing appends
1105 // to the bucket in hand; Best keeps a cycle that adds nothing from looping.
1106 for (int Distance = 0; Distance < static_cast<int>(Pending.size());
1107 ++Distance)
1108 for (unsigned I = 0; I != Pending[Distance].size(); ++I)
1109 scan(A: Pending[Distance][I]);
1110
1111 return Deficit;
1112}
1113
1114// Returns a minimum wait states since \p I walking all predecessors.
1115// Only scans until \p IsExpired does not return true.
1116// Can only be run in a hazard recognizer mode.
1117static int
1118getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
1119 const MachineBasicBlock *MBB,
1120 MachineBasicBlock::const_reverse_instr_iterator I,
1121 int WaitStates, GCNHazardRecognizer::IsExpiredFn IsExpired,
1122 DenseSet<const MachineBasicBlock *> &Visited,
1123 GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates =
1124 SIInstrInfo::getNumWaitStates) {
1125 for (auto E = MBB->instr_rend(); I != E; ++I) {
1126 // Don't add WaitStates for parent BUNDLE instructions.
1127 if (I->isBundle())
1128 continue;
1129
1130 if (I->isMetaInstruction())
1131 continue;
1132
1133 if (IsHazard(*I))
1134 return WaitStates;
1135
1136 if (I->isInlineAsm())
1137 continue;
1138
1139 WaitStates += GetNumWaitStates(*I);
1140
1141 if (IsExpired(*I, WaitStates))
1142 return std::numeric_limits<int>::max();
1143 }
1144
1145 int MinWaitStates = std::numeric_limits<int>::max();
1146 for (MachineBasicBlock *Pred : MBB->predecessors()) {
1147 if (!Visited.insert(V: Pred).second)
1148 continue;
1149
1150 int W = getWaitStatesSince(IsHazard, MBB: Pred, I: Pred->instr_rbegin(), WaitStates,
1151 IsExpired, Visited, GetNumWaitStates);
1152
1153 MinWaitStates = std::min(a: MinWaitStates, b: W);
1154 }
1155
1156 return MinWaitStates;
1157}
1158
1159static int
1160getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
1161 const MachineInstr *MI,
1162 GCNHazardRecognizer::IsExpiredFn IsExpired,
1163 GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates =
1164 SIInstrInfo::getNumWaitStates) {
1165 DenseSet<const MachineBasicBlock *> Visited;
1166 return getWaitStatesSince(IsHazard, MBB: MI->getParent(),
1167 I: std::next(x: MI->getReverseIterator()), WaitStates: 0, IsExpired,
1168 Visited, GetNumWaitStates);
1169}
1170
1171int GCNHazardRecognizer::getMaxWindowDeficit(int MaxWindow,
1172 WindowForFn WindowFor) const {
1173 if (isHazardRecognizerMode())
1174 return WindowDeficitSearch(WindowFor, MaxWindow).run(MI: *CurrCycleInstr);
1175
1176 // EmittedInstrs is capped and can be shorter than the widest window, which
1177 // costs scheduling quality only: the standalone pass still pads.
1178 int Deficit = 0, Distance = 0;
1179 for (MachineInstr *MI : EmittedInstrs) {
1180 if (MI) {
1181 if (std::optional<int> Window = WindowFor(*MI)) {
1182 assert(*Window >= 0 && *Window <= MaxWindow && "window out of bounds");
1183 Deficit = std::max(a: Deficit, b: *Window - Distance);
1184 }
1185 }
1186
1187 Distance +=
1188 MI ? getCountedWaitStates(MI: *MI, GetNumWaitStates: SIInstrInfo::getNumWaitStates) : 1;
1189
1190 if (Distance >= MaxWindow)
1191 break;
1192 }
1193 return Deficit;
1194}
1195
1196int GCNHazardRecognizer::getWaitStatesSince(
1197 IsHazardFn IsHazard, int Limit, GetNumWaitStatesFn GetNumWaitStates) const {
1198 if (isHazardRecognizerMode()) {
1199 auto IsExpiredFn = [Limit](const MachineInstr &, int WaitStates) {
1200 return WaitStates >= Limit;
1201 };
1202 return ::getWaitStatesSince(IsHazard, MI: CurrCycleInstr, IsExpired: IsExpiredFn,
1203 GetNumWaitStates);
1204 }
1205
1206 int WaitStates = 0;
1207 for (MachineInstr *MI : EmittedInstrs) {
1208 if (MI) {
1209 if (IsHazard(*MI))
1210 return WaitStates;
1211
1212 if (MI->isInlineAsm())
1213 continue;
1214 }
1215 WaitStates += MI ? GetNumWaitStates(*MI) : 1;
1216
1217 if (WaitStates >= Limit)
1218 break;
1219 }
1220 return std::numeric_limits<int>::max();
1221}
1222
1223int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard,
1224 int Limit) const {
1225 return getWaitStatesSince(IsHazard, Limit, GetNumWaitStates: SIInstrInfo::getNumWaitStates);
1226}
1227
1228int GCNHazardRecognizer::getMaxVALUWindowDeficit(int MaxWindow,
1229 WindowForFn WindowFor) const {
1230 if (isHazardRecognizerMode()) {
1231 auto GetVALUWaitStates = [](const MachineInstr &MI) -> unsigned {
1232 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ? 1 : 0;
1233 };
1234 return WindowDeficitSearch(WindowFor, MaxWindow, GetVALUWaitStates)
1235 .run(MI: *CurrCycleInstr);
1236 }
1237
1238 // EmittedVALUInstrs is capped at MaxVALULookAhead, so a MaxWindow beyond that
1239 // window could miss a hazard. Keep the cap in sync with the wait-state
1240 // tables.
1241 assert(MaxWindow <= static_cast<int>(MaxVALULookAhead) &&
1242 "MaxWindow exceeds the EmittedVALUInstrs lookahead window");
1243 int WaitStates = 0;
1244 for (MachineInstr *MI : EmittedVALUInstrs) {
1245 if (MI) {
1246 if (std::optional<int> Window = WindowFor(*MI)) {
1247 assert(*Window >= 0 && *Window <= MaxWindow && "window out of bounds");
1248 return *Window - WaitStates;
1249 }
1250 }
1251
1252 ++WaitStates;
1253
1254 if (WaitStates >= MaxWindow)
1255 break;
1256 }
1257 return 0;
1258}
1259
1260int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg,
1261 IsHazardFn IsHazardDef,
1262 int Limit) const {
1263 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1264
1265 auto IsHazardFn = [IsHazardDef, TRI, Reg](const MachineInstr &MI) {
1266 return IsHazardDef(MI) && MI.modifiesRegister(Reg, TRI);
1267 };
1268
1269 return getWaitStatesSince(IsHazard: IsHazardFn, Limit);
1270}
1271
1272int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard,
1273 int Limit) const {
1274 auto IsHazardFn = [IsHazard](const MachineInstr &MI) {
1275 return isSSetReg(Opcode: MI.getOpcode()) && IsHazard(MI);
1276 };
1277
1278 return getWaitStatesSince(IsHazard: IsHazardFn, Limit);
1279}
1280
1281//===----------------------------------------------------------------------===//
1282// No-op Hazard Detection
1283//===----------------------------------------------------------------------===//
1284
1285static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV,
1286 MCRegister Reg) {
1287 for (MCRegUnit Unit : TRI.regunits(Reg))
1288 BV.set(static_cast<unsigned>(Unit));
1289}
1290
1291static void addRegsToSet(const SIRegisterInfo &TRI,
1292 iterator_range<MachineInstr::const_mop_iterator> Ops,
1293 BitVector &DefSet, BitVector &UseSet) {
1294 for (const MachineOperand &Op : Ops) {
1295 if (Op.isReg())
1296 addRegUnits(TRI, BV&: Op.isDef() ? DefSet : UseSet, Reg: Op.getReg().asMCReg());
1297 }
1298}
1299
1300void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) const {
1301 addRegsToSet(TRI, Ops: MI.operands(), DefSet&: ClauseDefs, UseSet&: ClauseUses);
1302}
1303
1304static bool breaksSMEMSoftClause(MachineInstr *MI) {
1305 return !SIInstrInfo::isSMRD(MI: *MI);
1306}
1307
1308static bool breaksVMEMSoftClause(MachineInstr *MI) {
1309 return !SIInstrInfo::isVMEM(MI: *MI);
1310}
1311
1312int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) const {
1313 // SMEM soft clause are only present on VI+, and only matter if xnack is
1314 // enabled.
1315 if (!ST.isXNACKEnabled())
1316 return 0;
1317
1318 bool IsSMRD = TII.isSMRD(MI: *MEM);
1319
1320 resetClause();
1321
1322 // A soft-clause is any group of consecutive SMEM instructions. The
1323 // instructions in this group may return out of order and/or may be
1324 // replayed (i.e. the same instruction issued more than once).
1325 //
1326 // In order to handle these situations correctly we need to make sure that
1327 // when a clause has more than one instruction, no instruction in the clause
1328 // writes to a register that is read by another instruction in the clause
1329 // (including itself). If we encounter this situation, we need to break the
1330 // clause by inserting a non SMEM instruction.
1331
1332 for (MachineInstr *MI : EmittedInstrs) {
1333 // When we hit a non-SMEM instruction then we have passed the start of the
1334 // clause and we can stop.
1335 if (!MI)
1336 break;
1337
1338 if (IsSMRD ? breaksSMEMSoftClause(MI) : breaksVMEMSoftClause(MI))
1339 break;
1340
1341 addClauseInst(MI: *MI);
1342 }
1343
1344 if (ClauseDefs.none())
1345 return 0;
1346
1347 // We need to make sure not to put loads and stores in the same clause if they
1348 // use the same address. For now, just start a new clause whenever we see a
1349 // store.
1350 if (MEM->mayStore())
1351 return 1;
1352
1353 addClauseInst(MI: *MEM);
1354
1355 // If the set of defs and uses intersect then we cannot add this instruction
1356 // to the clause, so we have a hazard.
1357 return ClauseDefs.anyCommon(RHS: ClauseUses) ? 1 : 0;
1358}
1359
1360int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) const {
1361 int WaitStatesNeeded = 0;
1362
1363 WaitStatesNeeded = checkSoftClauseHazards(MEM: SMRD);
1364
1365 // This SMRD hazard only affects SI.
1366 if (!ST.hasSMRDReadVALUDefHazard())
1367 return WaitStatesNeeded;
1368
1369 // A read of an SGPR by SMRD instruction requires 4 wait states when the
1370 // SGPR was written by a VALU instruction.
1371 int SmrdSgprWaitStates = 4;
1372 auto IsHazardDefFn = [this](const MachineInstr &MI) {
1373 return TII.isVALU(MI, /*AllowLDSDMA=*/true);
1374 };
1375 auto IsBufferHazardDefFn = [this](const MachineInstr &MI) {
1376 return TII.isSALU(MI);
1377 };
1378
1379 bool IsBufferSMRD = TII.isBufferSMRD(MI: *SMRD);
1380
1381 for (const MachineOperand &Use : SMRD->uses()) {
1382 if (!Use.isReg())
1383 continue;
1384 int WaitStatesNeededForUse =
1385 SmrdSgprWaitStates - getWaitStatesSinceDef(Reg: Use.getReg(), IsHazardDef: IsHazardDefFn,
1386 Limit: SmrdSgprWaitStates);
1387 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
1388
1389 // This fixes what appears to be undocumented hardware behavior in SI where
1390 // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor
1391 // needs some number of nops in between. We don't know how many we need, but
1392 // let's use 4. This wasn't discovered before probably because the only
1393 // case when this happens is when we expand a 64-bit pointer into a full
1394 // descriptor and use s_buffer_load_dword instead of s_load_dword, which was
1395 // probably never encountered in the closed-source land.
1396 if (IsBufferSMRD) {
1397 int WaitStatesNeededForUse =
1398 SmrdSgprWaitStates - getWaitStatesSinceDef(Reg: Use.getReg(),
1399 IsHazardDef: IsBufferHazardDefFn,
1400 Limit: SmrdSgprWaitStates);
1401 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
1402 }
1403 }
1404
1405 return WaitStatesNeeded;
1406}
1407
1408int GCNHazardRecognizer::checkVMEMHazards(MachineInstr *VMEM) const {
1409 if (!ST.hasVMEMReadSGPRVALUDefHazard())
1410 return 0;
1411
1412 int WaitStatesNeeded = checkSoftClauseHazards(MEM: VMEM);
1413
1414 // A read of an SGPR by a VMEM instruction requires 5 wait states when the
1415 // SGPR was written by a VALU Instruction.
1416 const int VmemSgprWaitStates = 5;
1417 auto IsHazardDefFn = [this](const MachineInstr &MI) {
1418 return TII.isVALU(MI, /*AllowLDSDMA=*/true);
1419 };
1420 for (const MachineOperand &Use : VMEM->uses()) {
1421 if (!Use.isReg() || TRI.isVectorRegister(MRI: MF.getRegInfo(), Reg: Use.getReg()))
1422 continue;
1423
1424 int WaitStatesNeededForUse =
1425 VmemSgprWaitStates - getWaitStatesSinceDef(Reg: Use.getReg(), IsHazardDef: IsHazardDefFn,
1426 Limit: VmemSgprWaitStates);
1427 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
1428 }
1429 return WaitStatesNeeded;
1430}
1431
1432int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) const {
1433 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1434 const SIInstrInfo *TII = ST.getInstrInfo();
1435
1436 // Check for DPP VGPR read after VALU VGPR write and EXEC write.
1437 int DppVgprWaitStates = 2;
1438 int DppExecWaitStates = 5;
1439 int WaitStatesNeeded = 0;
1440 auto IsHazardDefFn = [TII](const MachineInstr &MI) {
1441 return TII->isVALU(MI, /*AllowLDSDMA=*/true);
1442 };
1443
1444 for (const MachineOperand &Use : DPP->uses()) {
1445 if (!Use.isReg() || !TRI->isVGPR(MRI: MF.getRegInfo(), Reg: Use.getReg()))
1446 continue;
1447 int WaitStatesNeededForUse =
1448 DppVgprWaitStates - getWaitStatesSinceDef(
1449 Reg: Use.getReg(),
1450 IsHazardDef: [](const MachineInstr &) { return true; },
1451 Limit: DppVgprWaitStates);
1452 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
1453 }
1454
1455 WaitStatesNeeded = std::max(
1456 a: WaitStatesNeeded,
1457 b: DppExecWaitStates - getWaitStatesSinceDef(Reg: AMDGPU::EXEC, IsHazardDef: IsHazardDefFn,
1458 Limit: DppExecWaitStates));
1459
1460 return WaitStatesNeeded;
1461}
1462
1463int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) const {
1464 const SIInstrInfo *TII = ST.getInstrInfo();
1465
1466 // v_div_fmas requires 4 wait states after a write to vcc from a VALU
1467 // instruction.
1468 const int DivFMasWaitStates = 4;
1469 auto IsHazardDefFn = [TII](const MachineInstr &MI) {
1470 return TII->isVALU(MI, /*AllowLDSDMA=*/true);
1471 };
1472 int WaitStatesNeeded = getWaitStatesSinceDef(Reg: AMDGPU::VCC, IsHazardDef: IsHazardDefFn,
1473 Limit: DivFMasWaitStates);
1474
1475 return DivFMasWaitStates - WaitStatesNeeded;
1476}
1477
1478int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) const {
1479 const SIInstrInfo *TII = ST.getInstrInfo();
1480 unsigned GetRegHWReg = getHWReg(TII, RegInstr: *GetRegInstr);
1481
1482 const int GetRegWaitStates = 2;
1483 auto IsHazardFn = [TII, GetRegHWReg](const MachineInstr &MI) {
1484 return GetRegHWReg == getHWReg(TII, RegInstr: MI);
1485 };
1486 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazard: IsHazardFn, Limit: GetRegWaitStates);
1487
1488 return GetRegWaitStates - WaitStatesNeeded;
1489}
1490
1491int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) const {
1492 const SIInstrInfo *TII = ST.getInstrInfo();
1493 unsigned HWReg = getHWReg(TII, RegInstr: *SetRegInstr);
1494
1495 const int SetRegWaitStates = ST.getSetRegWaitStates();
1496 auto IsHazardFn = [TII, HWReg](const MachineInstr &MI) {
1497 return HWReg == getHWReg(TII, RegInstr: MI);
1498 };
1499 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazard: IsHazardFn, Limit: SetRegWaitStates);
1500 return SetRegWaitStates - WaitStatesNeeded;
1501}
1502
1503int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) const {
1504 if (!MI.mayStore())
1505 return -1;
1506
1507 const SIInstrInfo *TII = ST.getInstrInfo();
1508 unsigned Opcode = MI.getOpcode();
1509 const MCInstrDesc &Desc = MI.getDesc();
1510
1511 int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::vdata);
1512 int VDataRCID = -1;
1513 if (VDataIdx != -1)
1514 VDataRCID = TII->getOpRegClassID(OpInfo: Desc.operands()[VDataIdx]);
1515
1516 if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) {
1517 // There is no hazard if the instruction does not use vector regs
1518 // (like wbinvl1)
1519 if (VDataIdx == -1)
1520 return -1;
1521 if (AMDGPU::getRegBitWidth(RCID: VDataRCID) > 64) {
1522 // When SOFFSET-dependent wide-store windows apply, the BUFFER_STORE
1523 // source-vgpr WAR hazard exists for every SOFFSET shape; the wait-state
1524 // count differs by SOFFSET and is computed in checkVALUHazardsHelper.
1525 // Otherwise the hazard only exists if soffset is not an SGPR.
1526 if (ST.hasVDecCoExecHazard())
1527 return VDataIdx;
1528 const MachineOperand *SOffset =
1529 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
1530 if (!SOffset || !SOffset->isReg())
1531 return VDataIdx;
1532 }
1533 }
1534
1535 // MIMG instructions create a hazard if they don't use a 256-bit T# and
1536 // the store size is greater than 8 bytes and they have more than two bits
1537 // of their dmask set.
1538 // All our MIMG definitions use a 256-bit T#, so we can skip checking for them.
1539 if (TII->isMIMG(MI)) {
1540 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::srsrc);
1541 assert(SRsrcIdx != -1 && AMDGPU::getRegBitWidth(TII->getOpRegClassID(
1542 Desc.operands()[SRsrcIdx])) == 256);
1543 (void)SRsrcIdx;
1544 }
1545
1546 if (TII->isFLAT(MI)) {
1547 // There is no hazard if the instruction does not use vector regs
1548 if (VDataIdx == -1)
1549 return -1;
1550
1551 if (AMDGPU::getRegBitWidth(RCID: VDataRCID) > 64)
1552 return VDataIdx;
1553 }
1554
1555 return -1;
1556}
1557
1558int GCNHazardRecognizer::checkUniformWindowVALUHazardsHelper(
1559 Register Reg) const {
1560 // Wide stores need a single wait-state bubble before a VALU that overwrites
1561 // store data. createsVALUHazard already excludes MUBUF/MTBUF stores with an
1562 // SGPR SOFFSET.
1563 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1564
1565 auto IsHazard = [&](const MachineInstr &MI) {
1566 int DataIdx = createsVALUHazard(MI);
1567 return DataIdx >= 0 &&
1568 TRI->regsOverlap(RegA: MI.getOperand(i: DataIdx).getReg(), RegB: Reg);
1569 };
1570
1571 return std::max(a: 0, b: 1 - getWaitStatesSince(IsHazard, /*Limit=*/1));
1572}
1573
1574int GCNHazardRecognizer::checkSOFFSETWindowVALUHazardsHelper(
1575 Register Reg) const {
1576 // The required wait-state window depends on the producer's SOFFSET shape:
1577 // - MUBUF/MTBUF wide store with sgpr SOFFSET: 1 wait state.
1578 // - MUBUF/MTBUF wide store with literal/absent SOFFSET, and FLAT wide
1579 // store: 2 wait states.
1580 // The 1-cycle sgpr-SOFFSET window was measured on gfx950.
1581 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1582 const SIInstrInfo *TII = ST.getInstrInfo();
1583
1584 int WaitStatesNeeded = 0;
1585
1586 // Scan each wait-state window separately and take the max padding needed.
1587 // getWaitStatesSince supplies the minimum distance to a producer over paths.
1588 for (int Window = 1; Window <= 2; ++Window) {
1589 auto IsHazard = [&](const MachineInstr &MI) {
1590 int DataIdx = createsVALUHazard(MI);
1591 if (DataIdx < 0 ||
1592 !TRI->regsOverlap(RegA: MI.getOperand(i: DataIdx).getReg(), RegB: Reg))
1593 return false;
1594
1595 // Window 1 matches every hazard producer. Window 2 excludes BUF stores
1596 // with an SGPR SOFFSET, which only require a single wait state.
1597 if (Window == 1 || !TII->isBUF(MI))
1598 return true;
1599
1600 const MachineOperand *SOffset =
1601 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
1602 return !SOffset || !SOffset->isReg();
1603 };
1604 WaitStatesNeeded = std::max(a: WaitStatesNeeded,
1605 b: Window - getWaitStatesSince(IsHazard, Limit: Window));
1606 }
1607
1608 return WaitStatesNeeded;
1609}
1610
1611int GCNHazardRecognizer::checkVALUHazardsHelper(
1612 const MachineOperand &Def, const MachineRegisterInfo &MRI) const {
1613 // Helper to check for the hazard where VMEM instructions that store more
1614 // than 8 bytes can have their store data overwritten by the next
1615 // instruction.
1616 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1617
1618 if (!TRI->isVectorRegister(MRI, Reg: Def.getReg()))
1619 return 0;
1620
1621 if (ST.hasVDecCoExecHazard())
1622 return checkSOFFSETWindowVALUHazardsHelper(Reg: Def.getReg());
1623
1624 return checkUniformWindowVALUHazardsHelper(Reg: Def.getReg());
1625}
1626
1627/// Dest sel forwarding issue occurs if additional logic is needed to swizzle /
1628/// pack the computed value into correct bit position of the dest register. This
1629/// occurs if we have SDWA with dst_sel != DWORD or if we have op_sel with
1630/// dst_sel that is not aligned to the register. This function analayzes the \p
1631/// MI and \returns an operand with dst forwarding issue, or nullptr if
1632/// none exists.
1633static const MachineOperand *
1634getDstSelForwardingOperand(const MachineInstr &MI, const GCNSubtarget &ST) {
1635 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false))
1636 return nullptr;
1637
1638 const SIInstrInfo *TII = ST.getInstrInfo();
1639
1640 unsigned Opcode = MI.getOpcode();
1641
1642 // There are three different types of instructions
1643 // which produce forwarded dest: 1. SDWA with dst_sel != DWORD, 2. VOP3
1644 // which write hi bits (e.g. op_sel[3] == 1), and 3. FP8DstSelInst
1645 // (instructions with dest byte sel, e.g. CVT_SR_BF8_F32) and
1646 // op_sel[3:2]
1647 // != 0
1648 if (SIInstrInfo::isSDWA(MI)) {
1649 // Type 1: SDWA with dst_sel != DWORD
1650 if (auto *DstSel = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::dst_sel))
1651 if (DstSel->getImm() != AMDGPU::SDWA::DWORD)
1652 return TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1653 }
1654
1655 AMDGPU::FPType IsFP4OrFP8ConvOpc = AMDGPU::getFPDstSelType(Opc: Opcode);
1656 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::op_sel)) {
1657 // Type 2: VOP3 which write the hi bits. DST_OP_SEL is a destination
1658 // selector in the VOP3 layout; VOP3P uses this bit for source op_sel_hi.
1659 if (SIInstrInfo::isVOP3(MI) && !SIInstrInfo::isVOP3P(MI) &&
1660 (TII->getNamedImmOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers) &
1661 SISrcMods::DST_OP_SEL))
1662 return TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1663
1664 // Type 2b: VOP3P mix instructions with a tied destination (MIXLO/MIXHI)
1665 // write only one half of vdst and preserve the other half, so treat them
1666 // like a VOP3 destination op_sel write regardless of their source
1667 // op_sel_hi bits.
1668 if (SIInstrInfo::isVOP3P(MI) &&
1669 AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::vdst_in))
1670 return TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1671
1672 // Type 3: FP8DstSelInst with op_sel[3:2] != 0)
1673 if (IsFP4OrFP8ConvOpc == AMDGPU::FPType::FP8 &&
1674 (TII->getNamedImmOperand(MI, OperandName: AMDGPU::OpName::src2_modifiers) &
1675 SISrcMods::OP_SEL_0))
1676 return TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1677 }
1678
1679 // Special case: nop is required for all the opsel values for fp4 sr variant
1680 // cvt scale instructions
1681 if (IsFP4OrFP8ConvOpc == AMDGPU::FPType::FP4)
1682 return TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1683
1684 return nullptr;
1685}
1686
1687/// Checks whether the provided \p MI "consumes" the operand with a Dest sel
1688/// fowarding issue \p Dst . We may "consume" the Dst via a standard explicit
1689/// RAW, or through irregular ways (e.g implicit RAW, certain types of WAW)
1690static bool consumesDstSelForwardingOperand(const MachineInstr *VALU,
1691 const MachineOperand *Dst,
1692 const SIRegisterInfo *TRI) {
1693 // We must consider implicit reads of the VALU. SDWA with dst_sel and
1694 // UNUSED_PRESERVE will implicitly read the result from forwarded dest,
1695 // and we must account for that hazard.
1696 // We also must account for WAW hazards. In particular, WAW with dest
1697 // preserve semantics (e.g. VOP3 with op_sel, VOP2 &&
1698 // !zeroesHigh16BitsOfDest) will read the forwarded dest for parity
1699 // check for ECC. Without accounting for this hazard, the ECC will be
1700 // wrong.
1701 // TODO: limit to RAW (including implicit reads) + problematic WAW (i.e.
1702 // complete zeroesHigh16BitsOfDest)
1703 for (auto &Operand : VALU->operands()) {
1704 if (Operand.isReg() && TRI->regsOverlap(RegA: Dst->getReg(), RegB: Operand.getReg())) {
1705 return true;
1706 }
1707 }
1708 return false;
1709}
1710
1711int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) const {
1712 int WaitStatesNeeded = 0;
1713
1714 if (ST.hasTransForwardingHazard() && !SIInstrInfo::isTRANS(MI: *VALU)) {
1715 const int TransDefWaitstates = 1;
1716
1717 auto IsTransDefFn = [this, VALU](const MachineInstr &MI) {
1718 if (!SIInstrInfo::isTRANS(MI))
1719 return false;
1720 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1721 const SIInstrInfo *TII = ST.getInstrInfo();
1722 Register Def = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst)->getReg();
1723
1724 for (const MachineOperand &Use : VALU->explicit_uses()) {
1725 if (Use.isReg() && TRI->regsOverlap(RegA: Def, RegB: Use.getReg()))
1726 return true;
1727 }
1728
1729 return false;
1730 };
1731
1732 int WaitStatesNeededForDef =
1733 TransDefWaitstates -
1734 getWaitStatesSince(IsHazard: IsTransDefFn, Limit: TransDefWaitstates);
1735 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1736 }
1737
1738 if (ST.hasDstSelForwardingHazard() || ST.hasCvtScaleForwardingHazard()) {
1739 const int Shift16DefWaitstates = 1;
1740
1741 auto IsShift16BitDefFn = [this, VALU](const MachineInstr &ProducerMI) {
1742 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1743 const MachineOperand *ForwardedDst =
1744 getDstSelForwardingOperand(MI: ProducerMI, ST);
1745 if (ForwardedDst) {
1746 return consumesDstSelForwardingOperand(VALU, Dst: ForwardedDst, TRI);
1747 }
1748
1749 if (ProducerMI.isInlineAsm()) {
1750 // Assume inline asm has dst forwarding hazard
1751 for (auto &Def : ProducerMI.all_defs()) {
1752 if (consumesDstSelForwardingOperand(VALU, Dst: &Def, TRI))
1753 return true;
1754 }
1755 }
1756
1757 return false;
1758 };
1759
1760 int WaitStatesNeededForDef =
1761 Shift16DefWaitstates -
1762 getWaitStatesSince(IsHazard: IsShift16BitDefFn, Limit: Shift16DefWaitstates);
1763 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1764 }
1765
1766 if (ST.hasVDecCoExecHazard()) {
1767 const int VALUWriteSGPRVALUReadWaitstates = 2;
1768 const int VALUWriteEXECRWLane = 4;
1769 const int VALUWriteVGPRReadlaneRead = 1;
1770
1771 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1772 const MachineRegisterInfo &MRI = MF.getRegInfo();
1773 Register UseReg;
1774 auto IsVALUDefSGPRFn = [&UseReg, TRI](const MachineInstr &MI) {
1775 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
1776 return false;
1777 return MI.modifiesRegister(Reg: UseReg, TRI);
1778 };
1779
1780 for (const MachineOperand &Use : VALU->explicit_uses()) {
1781 if (!Use.isReg())
1782 continue;
1783
1784 UseReg = Use.getReg();
1785 if (TRI->isSGPRReg(MRI, Reg: UseReg)) {
1786 int WaitStatesNeededForDef =
1787 VALUWriteSGPRVALUReadWaitstates -
1788 getWaitStatesSince(IsHazard: IsVALUDefSGPRFn,
1789 Limit: VALUWriteSGPRVALUReadWaitstates);
1790 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1791 }
1792 }
1793
1794 if (VALU->readsRegister(Reg: AMDGPU::VCC, TRI)) {
1795 UseReg = AMDGPU::VCC;
1796 int WaitStatesNeededForDef =
1797 VALUWriteSGPRVALUReadWaitstates -
1798 getWaitStatesSince(IsHazard: IsVALUDefSGPRFn, Limit: VALUWriteSGPRVALUReadWaitstates);
1799 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1800 }
1801
1802 switch (VALU->getOpcode()) {
1803 case AMDGPU::V_READLANE_B32:
1804 case AMDGPU::V_READFIRSTLANE_B32: {
1805 MachineOperand *Src = TII.getNamedOperand(MI&: *VALU, OperandName: AMDGPU::OpName::src0);
1806 UseReg = Src->getReg();
1807 int WaitStatesNeededForDef =
1808 VALUWriteVGPRReadlaneRead -
1809 getWaitStatesSince(IsHazard: IsVALUDefSGPRFn, Limit: VALUWriteVGPRReadlaneRead);
1810 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1811 }
1812 [[fallthrough]];
1813 case AMDGPU::V_WRITELANE_B32: {
1814 UseReg = AMDGPU::EXEC;
1815 int WaitStatesNeededForDef =
1816 VALUWriteEXECRWLane -
1817 getWaitStatesSince(IsHazard: IsVALUDefSGPRFn, Limit: VALUWriteEXECRWLane);
1818 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1819 break;
1820 }
1821 default:
1822 break;
1823 }
1824 }
1825
1826 // This checks for the hazard where VMEM instructions that store more than
1827 // 8 bytes can have there store data over written by the next instruction.
1828 if (!ST.has12DWordStoreHazard())
1829 return WaitStatesNeeded;
1830
1831 const MachineRegisterInfo &MRI = MF.getRegInfo();
1832
1833 for (const MachineOperand &Def : VALU->defs()) {
1834 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: checkVALUHazardsHelper(Def, MRI));
1835 }
1836
1837 return WaitStatesNeeded;
1838}
1839
1840int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) const {
1841 // This checks for hazards associated with inline asm statements.
1842 // Since inline asms can contain just about anything, we use this
1843 // to call/leverage other check*Hazard routines. Note that
1844 // this function doesn't attempt to address all possible inline asm
1845 // hazards (good luck), but is a collection of what has been
1846 // problematic thus far.
1847
1848 // see checkVALUHazards()
1849 if (!ST.has12DWordStoreHazard() && !ST.hasDstSelForwardingHazard() &&
1850 !ST.hasCvtScaleForwardingHazard())
1851 return 0;
1852
1853 const MachineRegisterInfo &MRI = MF.getRegInfo();
1854 int WaitStatesNeeded = 0;
1855
1856 for (const MachineOperand &Op :
1857 llvm::drop_begin(RangeOrContainer: IA->operands(), N: InlineAsm::MIOp_FirstOperand)) {
1858 if (Op.isReg() && Op.isDef()) {
1859 if (!TRI.isVectorRegister(MRI, Reg: Op.getReg()))
1860 continue;
1861
1862 if (ST.has12DWordStoreHazard()) {
1863 WaitStatesNeeded =
1864 std::max(a: WaitStatesNeeded, b: checkVALUHazardsHelper(Def: Op, MRI));
1865 }
1866 }
1867 }
1868
1869 if (ST.hasDstSelForwardingHazard()) {
1870 const int Shift16DefWaitstates = 1;
1871
1872 auto IsShift16BitDefFn = [this, &IA](const MachineInstr &ProducerMI) {
1873 const MachineOperand *Dst = getDstSelForwardingOperand(MI: ProducerMI, ST);
1874 // Assume inline asm reads the dst
1875 if (Dst)
1876 return IA->modifiesRegister(Reg: Dst->getReg(), TRI: &TRI) ||
1877 IA->readsRegister(Reg: Dst->getReg(), TRI: &TRI);
1878
1879 if (ProducerMI.isInlineAsm()) {
1880 // If MI is inline asm, assume it has dst forwarding hazard
1881 for (auto &Def : ProducerMI.all_defs()) {
1882 if (IA->modifiesRegister(Reg: Def.getReg(), TRI: &TRI) ||
1883 IA->readsRegister(Reg: Def.getReg(), TRI: &TRI)) {
1884 return true;
1885 }
1886 }
1887 }
1888
1889 return false;
1890 };
1891
1892 int WaitStatesNeededForDef =
1893 Shift16DefWaitstates -
1894 getWaitStatesSince(IsHazard: IsShift16BitDefFn, Limit: Shift16DefWaitstates);
1895 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1896 }
1897
1898 return WaitStatesNeeded;
1899}
1900
1901int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) const {
1902 const SIInstrInfo *TII = ST.getInstrInfo();
1903 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1904 const MachineRegisterInfo &MRI = MF.getRegInfo();
1905
1906 const MachineOperand *LaneSelectOp =
1907 TII->getNamedOperand(MI&: *RWLane, OperandName: AMDGPU::OpName::src1);
1908
1909 if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, Reg: LaneSelectOp->getReg()))
1910 return 0;
1911
1912 Register LaneSelectReg = LaneSelectOp->getReg();
1913 auto IsHazardFn = [TII](const MachineInstr &MI) {
1914 return TII->isVALU(MI, /*AllowLDSDMA=*/true);
1915 };
1916
1917 const int RWLaneWaitStates = 4;
1918 int WaitStatesSince = getWaitStatesSinceDef(Reg: LaneSelectReg, IsHazardDef: IsHazardFn,
1919 Limit: RWLaneWaitStates);
1920 return RWLaneWaitStates - WaitStatesSince;
1921}
1922
1923int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) const {
1924 if (!ST.hasRFEHazards())
1925 return 0;
1926
1927 const SIInstrInfo *TII = ST.getInstrInfo();
1928
1929 const int RFEWaitStates = 1;
1930
1931 auto IsHazardFn = [TII](const MachineInstr &MI) {
1932 return getHWReg(TII, RegInstr: MI) == AMDGPU::Hwreg::ID_TRAPSTS;
1933 };
1934 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazard: IsHazardFn, Limit: RFEWaitStates);
1935 return RFEWaitStates - WaitStatesNeeded;
1936}
1937
1938int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) const {
1939 const SIInstrInfo *TII = ST.getInstrInfo();
1940 const int ReadM0WaitStates = 1;
1941 auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isSALU(MI); };
1942 return ReadM0WaitStates -
1943 getWaitStatesSinceDef(Reg: AMDGPU::M0, IsHazardDef: IsHazardFn, Limit: ReadM0WaitStates);
1944}
1945
1946void GCNHazardRecognizer::emitVNops(MachineBasicBlock &MBB,
1947 MachineBasicBlock::iterator InsertPt,
1948 int WaitStatesNeeded, bool IsHoisting) {
1949 const DebugLoc &DL = IsHoisting ? DebugLoc() : InsertPt->getDebugLoc();
1950 for (int I = 0; I < WaitStatesNeeded; ++I)
1951 BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_NOP_e32));
1952}
1953
1954void GCNHazardRecognizer::fixHazards(MachineInstr *MI) {
1955 fixVMEMtoScalarWriteHazards(MI);
1956 fixVcmpxPermlaneHazards(MI);
1957 fixSMEMtoVectorWriteHazards(MI);
1958 fixVcmpxExecWARHazard(MI);
1959 fixLdsBranchVmemWARHazard(MI);
1960 if (ST.hasLdsDirect()) {
1961 fixLdsDirectVALUHazard(MI);
1962 fixLdsDirectVMEMHazard(MI);
1963 }
1964 fixVALUPartialForwardingHazard(MI);
1965 fixVALUTransUseHazard(MI);
1966 fixVALUTransCoexecutionHazards(MI);
1967 fixWMMAHazards(MI); // fall-through if co-execution is enabled.
1968 fixWMMACoexecutionHazards(MI);
1969 fixShift64HighRegBug(MI);
1970 fixPkF32InvalidOpSel(MI);
1971 fixVALUMaskWriteHazard(MI);
1972 fixRequiredExportPriority(MI);
1973 if (ST.hasVPermPk16Hazard())
1974 fixVPermPk16Hazard(MI);
1975 if (ST.requiresWaitIdleBeforeGetReg())
1976 fixGetRegWaitIdle(MI);
1977 if (ST.hasDsAtomicAsyncBarrierArriveB64PipeBug())
1978 fixDsAtomicAsyncBarrierArriveB64(MI);
1979 if (ST.hasScratchBaseForwardingHazard())
1980 fixScratchBaseForwardingHazard(MI);
1981 if (ST.setRegModeNeedsVNOPs())
1982 fixSetRegMode(MI);
1983 if (ST.hasNeedsTDMDrain())
1984 fixTDM(MI);
1985}
1986
1987static bool isVCmpXWritesExec(const SIInstrInfo &TII, const SIRegisterInfo &TRI,
1988 const MachineInstr &MI) {
1989 return (TII.isVOPC(MI) ||
1990 (MI.isCompare() && (TII.isVOP3(MI) || TII.isSDWA(MI)))) &&
1991 MI.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &TRI);
1992}
1993
1994bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) {
1995 if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(MI: *MI))
1996 return false;
1997
1998 const SIInstrInfo *TII = ST.getInstrInfo();
1999 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2000 auto IsHazardFn = [TII, TRI](const MachineInstr &MI) {
2001 return isVCmpXWritesExec(TII: *TII, TRI: *TRI, MI);
2002 };
2003
2004 auto IsExpiredFn = [](const MachineInstr &MI, int) {
2005 unsigned Opc = MI.getOpcode();
2006 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
2007 Opc != AMDGPU::V_NOP_e32 && Opc != AMDGPU::V_NOP_e64 &&
2008 Opc != AMDGPU::V_NOP_sdwa;
2009 };
2010
2011 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2012 std::numeric_limits<int>::max())
2013 return false;
2014
2015 // V_NOP will be discarded by SQ.
2016 // Use V_MOV_B32 v?, v?. Register must be alive so use src0 of V_PERMLANE*
2017 // which is always a VGPR and available.
2018 auto *Src0 = TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src0);
2019 Register Reg = Src0->getReg();
2020 bool IsUndef = Src0->isUndef();
2021 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2022 MCID: TII->get(Opcode: AMDGPU::V_MOV_B32_e32))
2023 .addReg(RegNo: Reg, Flags: RegState::Define | getDeadRegState(B: IsUndef))
2024 .addReg(RegNo: Reg, Flags: IsUndef ? RegState::Undef : RegState::Kill);
2025
2026 return true;
2027}
2028
2029bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) {
2030 if (!ST.hasVMEMtoScalarWriteHazard())
2031 return false;
2032 assert(!ST.hasExtendedWaitCounts());
2033
2034 if (!SIInstrInfo::isSALU(MI: *MI) && !SIInstrInfo::isSMRD(MI: *MI))
2035 return false;
2036
2037 if (MI->getNumDefs() == 0)
2038 return false;
2039
2040 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2041
2042 auto IsHazardFn = [TRI, MI](const MachineInstr &I) {
2043 if (!SIInstrInfo::isVMEM(MI: I) && !SIInstrInfo::isDS(MI: I))
2044 return false;
2045
2046 for (const MachineOperand &Def : MI->defs()) {
2047 const MachineOperand *Op =
2048 I.findRegisterUseOperand(Reg: Def.getReg(), TRI, isKill: false);
2049 if (!Op)
2050 continue;
2051 return true;
2052 }
2053 return false;
2054 };
2055
2056 auto IsExpiredFn = [](const MachineInstr &MI, int) {
2057 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ||
2058 (MI.getOpcode() == AMDGPU::S_WAITCNT &&
2059 !MI.getOperand(i: 0).getImm()) ||
2060 (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2061 AMDGPU::DepCtr::decodeFieldVmVsrc(
2062 Encoded: static_cast<unsigned>(MI.getOperand(i: 0).getImm())) == 0);
2063 };
2064
2065 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2066 std::numeric_limits<int>::max())
2067 return false;
2068
2069 const SIInstrInfo *TII = ST.getInstrInfo();
2070 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2071 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
2072 .addImm(Val: AMDGPU::DepCtr::encodeFieldVmVsrc(VmVsrc: 0, STI: ST));
2073 return true;
2074}
2075
2076bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) {
2077 if (!ST.hasSMEMtoVectorWriteHazard())
2078 return false;
2079 assert(!ST.hasExtendedWaitCounts());
2080
2081 if (!SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true))
2082 return false;
2083
2084 AMDGPU::OpName SDSTName;
2085 switch (MI->getOpcode()) {
2086 case AMDGPU::V_READLANE_B32:
2087 case AMDGPU::V_READFIRSTLANE_B32:
2088 SDSTName = AMDGPU::OpName::vdst;
2089 break;
2090 default:
2091 SDSTName = AMDGPU::OpName::sdst;
2092 break;
2093 }
2094
2095 const SIInstrInfo *TII = ST.getInstrInfo();
2096 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2097 const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(GPU: ST.getCPU());
2098 const MachineOperand *SDST = TII->getNamedOperand(MI&: *MI, OperandName: SDSTName);
2099 if (!SDST) {
2100 for (const auto &MO : MI->implicit_operands()) {
2101 if (MO.isDef() && TRI->isSGPRClass(RC: TRI->getPhysRegBaseClass(Reg: MO.getReg()))) {
2102 SDST = &MO;
2103 break;
2104 }
2105 }
2106 }
2107
2108 if (!SDST)
2109 return false;
2110
2111 const Register SDSTReg = SDST->getReg();
2112 auto IsHazardFn = [SDSTReg, TRI](const MachineInstr &I) {
2113 return SIInstrInfo::isSMRD(MI: I) && I.readsRegister(Reg: SDSTReg, TRI);
2114 };
2115
2116 auto IsExpiredFn = [TII, IV](const MachineInstr &MI, int) {
2117 if (TII->isSALU(MI)) {
2118 switch (MI.getOpcode()) {
2119 case AMDGPU::S_SETVSKIP:
2120 case AMDGPU::S_VERSION:
2121 case AMDGPU::S_WAITCNT_VSCNT:
2122 case AMDGPU::S_WAITCNT_VMCNT:
2123 case AMDGPU::S_WAITCNT_EXPCNT:
2124 // These instructions cannot not mitigate the hazard.
2125 return false;
2126 case AMDGPU::S_WAITCNT_LGKMCNT:
2127 // Reducing lgkmcnt count to 0 always mitigates the hazard.
2128 return (MI.getOperand(i: 1).getImm() == 0) &&
2129 (MI.getOperand(i: 0).getReg() == AMDGPU::SGPR_NULL);
2130 case AMDGPU::S_WAITCNT: {
2131 const int64_t Imm = MI.getOperand(i: 0).getImm();
2132 AMDGPU::Waitcnt Decoded =
2133 AMDGPU::decodeWaitcnt(Version: IV, Encoded: static_cast<unsigned>(Imm));
2134 // DsCnt corresponds to LGKMCnt here.
2135 return Decoded.get(T: AMDGPU::DS_CNT) == 0;
2136 }
2137 default:
2138 assert((!SIInstrInfo::isWaitcnt(MI.getOpcode()) ||
2139 MI.getOpcode() == AMDGPU::S_WAIT_IDLE) &&
2140 "unexpected wait count instruction");
2141 // SOPP instructions cannot mitigate the hazard.
2142 if (TII->isSOPP(MI))
2143 return false;
2144 // At this point the SALU can be assumed to mitigate the hazard
2145 // because either:
2146 // (a) it is independent of the at risk SMEM (breaking chain),
2147 // or
2148 // (b) it is dependent on the SMEM, in which case an appropriate
2149 // s_waitcnt lgkmcnt _must_ exist between it and the at risk
2150 // SMEM instruction.
2151 return true;
2152 }
2153 }
2154 return false;
2155 };
2156
2157 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2158 std::numeric_limits<int>::max())
2159 return false;
2160
2161 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2162 MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::SGPR_NULL)
2163 .addImm(Val: 0);
2164 return true;
2165}
2166
2167bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) {
2168 if (!ST.hasVcmpxExecWARHazard())
2169 return false;
2170 assert(!ST.hasExtendedWaitCounts());
2171
2172 if (!SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true))
2173 return false;
2174
2175 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2176 if (!MI->modifiesRegister(Reg: AMDGPU::EXEC, TRI))
2177 return false;
2178
2179 auto IsHazardFn = [TRI](const MachineInstr &I) {
2180 if (SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true))
2181 return false;
2182 return I.readsRegister(Reg: AMDGPU::EXEC, TRI);
2183 };
2184
2185 const SIInstrInfo *TII = ST.getInstrInfo();
2186 auto IsExpiredFn = [TII, TRI](const MachineInstr &MI, int) {
2187 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true)) {
2188 if (TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst))
2189 return true;
2190 for (auto MO : MI.implicit_operands())
2191 if (MO.isDef() && TRI->isSGPRClass(RC: TRI->getPhysRegBaseClass(Reg: MO.getReg())))
2192 return true;
2193 }
2194 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2195 AMDGPU::DepCtr::decodeFieldSaSdst(
2196 Encoded: static_cast<unsigned>(MI.getOperand(i: 0).getImm())) == 0)
2197 return true;
2198 return false;
2199 };
2200
2201 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2202 std::numeric_limits<int>::max())
2203 return false;
2204
2205 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2206 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
2207 .addImm(Val: AMDGPU::DepCtr::encodeFieldSaSdst(SaSdst: 0, STI: ST));
2208 return true;
2209}
2210
2211static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF,
2212 const GCNSubtarget &ST) {
2213 if (!ST.hasLdsBranchVmemWARHazard())
2214 return false;
2215
2216 // Check if the necessary condition for the hazard is met: both LDS and VMEM
2217 // instructions need to appear in the same function.
2218 bool HasLds = false;
2219 bool HasVmem = false;
2220 for (auto &MBB : MF) {
2221 for (auto &MI : MBB) {
2222 HasLds |= SIInstrInfo::isDS(MI) || SIInstrInfo::isLDSDMA(MI);
2223 HasVmem |= SIInstrInfo::isVMEM(MI);
2224 if (HasLds && HasVmem)
2225 return true;
2226 }
2227 }
2228 return false;
2229}
2230
2231static bool isStoreCountWaitZero(const MachineInstr &I) {
2232 return I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
2233 I.getOperand(i: 0).getReg() == AMDGPU::SGPR_NULL &&
2234 !I.getOperand(i: 1).getImm();
2235}
2236
2237bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) {
2238 if (!RunLdsBranchVmemWARHazardFixup)
2239 return false;
2240
2241 assert(ST.hasLdsBranchVmemWARHazard());
2242 assert(!ST.hasExtendedWaitCounts());
2243
2244 auto IsHazardInst = [](const MachineInstr &MI) {
2245 if (SIInstrInfo::isDS(MI) || SIInstrInfo::isLDSDMA(MI))
2246 return 1;
2247 if (SIInstrInfo::isVMEM(MI))
2248 return 2;
2249 return 0;
2250 };
2251
2252 auto InstType = IsHazardInst(*MI);
2253 if (!InstType)
2254 return false;
2255
2256 auto IsExpiredFn = [&IsHazardInst](const MachineInstr &I, int) {
2257 return IsHazardInst(I) || isStoreCountWaitZero(I);
2258 };
2259
2260 auto IsHazardFn = [InstType, &IsHazardInst](const MachineInstr &I) {
2261 if (!I.isBranch())
2262 return false;
2263
2264 auto IsHazardFn = [InstType, IsHazardInst](const MachineInstr &I) {
2265 auto InstType2 = IsHazardInst(I);
2266 return InstType2 && InstType != InstType2;
2267 };
2268
2269 auto IsExpiredFn = [InstType, &IsHazardInst](const MachineInstr &I, int) {
2270 auto InstType2 = IsHazardInst(I);
2271 if (InstType == InstType2)
2272 return true;
2273
2274 return isStoreCountWaitZero(I);
2275 };
2276
2277 return ::getWaitStatesSince(IsHazard: IsHazardFn, MI: &I, IsExpired: IsExpiredFn) !=
2278 std::numeric_limits<int>::max();
2279 };
2280
2281 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2282 std::numeric_limits<int>::max())
2283 return false;
2284
2285 const SIInstrInfo *TII = ST.getInstrInfo();
2286 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2287 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_VSCNT))
2288 .addReg(RegNo: AMDGPU::SGPR_NULL, Flags: RegState::Undef)
2289 .addImm(Val: 0);
2290
2291 return true;
2292}
2293
2294bool GCNHazardRecognizer::fixLdsDirectVALUHazard(MachineInstr *MI) {
2295 if (!SIInstrInfo::isLDSDIR(MI: *MI))
2296 return false;
2297
2298 const int NoHazardWaitStates = 15;
2299 const MachineOperand *VDST = TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::vdst);
2300 const Register VDSTReg = VDST->getReg();
2301
2302 bool VisitedTrans = false;
2303 auto IsHazardFn = [this, VDSTReg, &VisitedTrans](const MachineInstr &I) {
2304 if (!SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true))
2305 return false;
2306 VisitedTrans = VisitedTrans || SIInstrInfo::isTRANS(MI: I);
2307 // Cover both WAR and WAW
2308 return I.readsRegister(Reg: VDSTReg, TRI: &TRI) || I.modifiesRegister(Reg: VDSTReg, TRI: &TRI);
2309 };
2310 auto IsExpiredFn = [&](const MachineInstr &I, int WaitStates) {
2311 if (WaitStates >= NoHazardWaitStates)
2312 return true;
2313 // Instructions which cause va_vdst==0 expire hazard
2314 return SIInstrInfo::isVMEM(MI: I) || SIInstrInfo::isDS(MI: I) ||
2315 SIInstrInfo::isEXP(MI: I);
2316 };
2317 auto GetWaitStatesFn = [](const MachineInstr &MI) {
2318 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ? 1 : 0;
2319 };
2320
2321 DenseSet<const MachineBasicBlock *> Visited;
2322 auto Count = ::getWaitStatesSince(IsHazard: IsHazardFn, MBB: MI->getParent(),
2323 I: std::next(x: MI->getReverseIterator()), WaitStates: 0,
2324 IsExpired: IsExpiredFn, Visited, GetNumWaitStates: GetWaitStatesFn);
2325
2326 // Transcendentals can execute in parallel to other VALUs.
2327 // This makes va_vdst count unusable with a mixture of VALU and TRANS.
2328 if (VisitedTrans)
2329 Count = 0;
2330
2331 MachineOperand *WaitVdstOp =
2332 TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::waitvdst);
2333 WaitVdstOp->setImm(std::min(a: Count, b: NoHazardWaitStates));
2334
2335 return true;
2336}
2337
2338bool GCNHazardRecognizer::fixLdsDirectVMEMHazard(MachineInstr *MI) {
2339 if (!SIInstrInfo::isLDSDIR(MI: *MI))
2340 return false;
2341
2342 const MachineOperand *VDST = TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::vdst);
2343 const Register VDSTReg = VDST->getReg();
2344
2345 auto IsHazardFn = [this, VDSTReg](const MachineInstr &I) {
2346 if (!SIInstrInfo::isVMEM(MI: I) && !SIInstrInfo::isDS(MI: I))
2347 return false;
2348 return I.readsRegister(Reg: VDSTReg, TRI: &TRI) || I.modifiesRegister(Reg: VDSTReg, TRI: &TRI);
2349 };
2350 bool LdsdirCanWait = ST.hasLdsWaitVMSRC();
2351 // TODO: On GFX12 the hazard should expire on S_WAIT_LOADCNT/SAMPLECNT/BVHCNT
2352 // according to the type of VMEM instruction.
2353 auto IsExpiredFn = [this, LdsdirCanWait](const MachineInstr &I, int) {
2354 return SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true) ||
2355 SIInstrInfo::isEXP(MI: I) ||
2356 (I.getOpcode() == AMDGPU::S_WAITCNT && !I.getOperand(i: 0).getImm()) ||
2357 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2358 AMDGPU::DepCtr::decodeFieldVmVsrc(
2359 Encoded: static_cast<unsigned>(I.getOperand(i: 0).getImm())) == 0) ||
2360 (LdsdirCanWait && SIInstrInfo::isLDSDIR(MI: I) &&
2361 !TII.getNamedOperand(MI: I, OperandName: AMDGPU::OpName::waitvsrc)->getImm());
2362 };
2363
2364 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2365 std::numeric_limits<int>::max())
2366 return false;
2367
2368 if (LdsdirCanWait) {
2369 TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::waitvsrc)->setImm(0);
2370 } else {
2371 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2372 MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
2373 .addImm(Val: AMDGPU::DepCtr::encodeFieldVmVsrc(VmVsrc: 0, STI: ST));
2374 }
2375
2376 return true;
2377}
2378
2379bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(MachineInstr *MI) {
2380 if (!ST.hasVALUPartialForwardingHazard())
2381 return false;
2382 assert(!ST.hasExtendedWaitCounts());
2383
2384 if (!ST.isWave64() || !SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true))
2385 return false;
2386
2387 SmallSetVector<Register, 4> SrcVGPRs;
2388
2389 for (const MachineOperand &Use : MI->explicit_uses()) {
2390 if (Use.isReg() && TRI.isVGPR(MRI: MF.getRegInfo(), Reg: Use.getReg()))
2391 SrcVGPRs.insert(X: Use.getReg());
2392 }
2393
2394 // Only applies with >= 2 unique VGPR sources
2395 if (SrcVGPRs.size() <= 1)
2396 return false;
2397
2398 // Look for the following pattern:
2399 // Va <- VALU [PreExecPos]
2400 // intv1
2401 // Exec <- SALU [ExecPos]
2402 // intv2
2403 // Vb <- VALU [PostExecPos]
2404 // intv3
2405 // MI Va, Vb (WaitState = 0)
2406 //
2407 // Where:
2408 // intv1 + intv2 <= 2 VALUs
2409 // intv3 <= 4 VALUs
2410 //
2411 // If found, insert an appropriate S_WAITCNT_DEPCTR before MI.
2412
2413 const int Intv1plus2MaxVALUs = 2;
2414 const int Intv3MaxVALUs = 4;
2415 const int IntvMaxVALUs = 6;
2416 const int NoHazardVALUWaitStates = IntvMaxVALUs + 2;
2417
2418 struct StateType {
2419 SmallDenseMap<Register, int, 4> DefPos;
2420 int ExecPos = std::numeric_limits<int>::max();
2421 int VALUs = 0;
2422
2423 static unsigned getHashValue(const StateType &State) {
2424 hash_code H = hash_combine(args: State.ExecPos, args: State.VALUs);
2425 for (const auto &[Reg, Pos] : State.DefPos)
2426 H = hash_combine(args: H, args: Reg, args: Pos);
2427 return static_cast<unsigned>(H);
2428 }
2429 static bool isEqual(const StateType &LHS, const StateType &RHS) {
2430 return LHS.DefPos == RHS.DefPos && LHS.ExecPos == RHS.ExecPos &&
2431 LHS.VALUs == RHS.VALUs;
2432 }
2433 };
2434
2435 StateType State;
2436
2437 // This overloads expiry testing with all the hazard detection
2438 auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) {
2439 // Too many VALU states have passed
2440 if (State.VALUs > NoHazardVALUWaitStates)
2441 return HazardExpired;
2442
2443 // Instructions which cause va_vdst==0 expire hazard
2444 if (SIInstrInfo::isVMEM(MI: I) || SIInstrInfo::isDS(MI: I) ||
2445 SIInstrInfo::isEXP(MI: I) ||
2446 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2447 AMDGPU::DepCtr::decodeFieldVaVdst(
2448 Encoded: static_cast<unsigned>(I.getOperand(i: 0).getImm())) == 0))
2449 return HazardExpired;
2450
2451 // Track registers writes
2452 bool Changed = false;
2453 if (SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true)) {
2454 for (Register Src : SrcVGPRs) {
2455 if (!State.DefPos.count(Val: Src) && I.modifiesRegister(Reg: Src, TRI: &TRI)) {
2456 State.DefPos[Src] = State.VALUs;
2457 Changed = true;
2458 }
2459 }
2460 } else if (SIInstrInfo::isSALU(MI: I)) {
2461 if (State.ExecPos == std::numeric_limits<int>::max()) {
2462 if (!State.DefPos.empty() && I.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &TRI)) {
2463 State.ExecPos = State.VALUs;
2464 Changed = true;
2465 }
2466 }
2467 }
2468
2469 // Early expiration: too many VALUs in intv3
2470 if (State.VALUs > Intv3MaxVALUs && State.DefPos.empty())
2471 return HazardExpired;
2472
2473 // Only evaluate state if something changed
2474 if (!Changed)
2475 return NoHazardFound;
2476
2477 // Determine positions of VALUs pre/post exec change
2478 if (State.ExecPos == std::numeric_limits<int>::max())
2479 return NoHazardFound;
2480
2481 int PreExecPos = std::numeric_limits<int>::max();
2482 int PostExecPos = std::numeric_limits<int>::max();
2483
2484 for (auto Entry : State.DefPos) {
2485 int DefVALUs = Entry.second;
2486 if (DefVALUs != std::numeric_limits<int>::max()) {
2487 if (DefVALUs >= State.ExecPos)
2488 PreExecPos = std::min(a: PreExecPos, b: DefVALUs);
2489 else
2490 PostExecPos = std::min(a: PostExecPos, b: DefVALUs);
2491 }
2492 }
2493
2494 // Need a VALUs post exec change
2495 if (PostExecPos == std::numeric_limits<int>::max())
2496 return NoHazardFound;
2497
2498 // Too many VALUs in intv3?
2499 int Intv3VALUs = PostExecPos;
2500 if (Intv3VALUs > Intv3MaxVALUs)
2501 return HazardExpired;
2502
2503 // Too many VALUs in intv2?
2504 int Intv2VALUs = (State.ExecPos - PostExecPos) - 1;
2505 if (Intv2VALUs > Intv1plus2MaxVALUs)
2506 return HazardExpired;
2507
2508 // Need a VALUs pre exec change
2509 if (PreExecPos == std::numeric_limits<int>::max())
2510 return NoHazardFound;
2511
2512 // Too many VALUs in intv1?
2513 int Intv1VALUs = PreExecPos - State.ExecPos;
2514 if (Intv1VALUs > Intv1plus2MaxVALUs)
2515 return HazardExpired;
2516
2517 // Too many VALUs in intv1 + intv2
2518 if (Intv1VALUs + Intv2VALUs > Intv1plus2MaxVALUs)
2519 return HazardExpired;
2520
2521 return HazardFound;
2522 };
2523 auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) {
2524 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
2525 State.VALUs += 1;
2526 };
2527
2528 if (!hasHazard<StateType>(InitialState: State, IsHazard: IsHazardFn, UpdateState: UpdateStateFn, InitialMBB: MI->getParent(),
2529 InitialI: std::next(x: MI->getReverseIterator())))
2530 return false;
2531
2532 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2533 MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
2534 .addImm(Val: AMDGPU::DepCtr::encodeFieldVaVdst(VaVdst: 0, STI: ST));
2535
2536 return true;
2537}
2538
2539bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) {
2540 if (!ST.hasVALUTransUseHazard())
2541 return false;
2542 assert(!ST.hasExtendedWaitCounts());
2543
2544 if (!SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true))
2545 return false;
2546
2547 SmallSet<Register, 4> SrcVGPRs;
2548
2549 for (const MachineOperand &Use : MI->explicit_uses()) {
2550 if (Use.isReg() && TRI.isVGPR(MRI: MF.getRegInfo(), Reg: Use.getReg()))
2551 SrcVGPRs.insert(V: Use.getReg());
2552 }
2553
2554 // Look for the following pattern:
2555 // Va <- TRANS VALU
2556 // intv
2557 // MI Va (WaitState = 0)
2558 //
2559 // Where:
2560 // intv <= 5 VALUs / 1 TRANS
2561 //
2562 // If found, insert an appropriate S_WAITCNT_DEPCTR before MI.
2563
2564 const int IntvMaxVALUs = 5;
2565 const int IntvMaxTRANS = 1;
2566
2567 struct StateType {
2568 int VALUs = 0;
2569 int TRANS = 0;
2570
2571 static unsigned getHashValue(const StateType &State) {
2572 return static_cast<unsigned>(hash_combine(args: State.VALUs, args: State.TRANS));
2573 }
2574 static bool isEqual(const StateType &LHS, const StateType &RHS) {
2575 return LHS.VALUs == RHS.VALUs && LHS.TRANS == RHS.TRANS;
2576 }
2577 };
2578
2579 StateType State;
2580
2581 // This overloads expiry testing with all the hazard detection
2582 auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) {
2583 // Too many VALU states have passed
2584 if (State.VALUs > IntvMaxVALUs || State.TRANS > IntvMaxTRANS)
2585 return HazardExpired;
2586
2587 // Instructions which cause va_vdst==0 expire hazard
2588 if (SIInstrInfo::isVMEM(MI: I) || SIInstrInfo::isDS(MI: I) ||
2589 SIInstrInfo::isEXP(MI: I) ||
2590 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2591 AMDGPU::DepCtr::decodeFieldVaVdst(
2592 Encoded: static_cast<unsigned>(I.getOperand(i: 0).getImm())) == 0))
2593 return HazardExpired;
2594
2595 // Track registers writes
2596 if (SIInstrInfo::isTRANS(MI: I)) {
2597 for (Register Src : SrcVGPRs) {
2598 if (I.modifiesRegister(Reg: Src, TRI: &TRI)) {
2599 return HazardFound;
2600 }
2601 }
2602 }
2603
2604 return NoHazardFound;
2605 };
2606 auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) {
2607 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
2608 State.VALUs += 1;
2609 if (SIInstrInfo::isTRANS(MI))
2610 State.TRANS += 1;
2611 };
2612
2613 if (!hasHazard<StateType>(InitialState: State, IsHazard: IsHazardFn, UpdateState: UpdateStateFn, InitialMBB: MI->getParent(),
2614 InitialI: std::next(x: MI->getReverseIterator())))
2615 return false;
2616
2617 // Hazard is observed - insert a wait on va_dst counter to ensure hazard is
2618 // avoided.
2619 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2620 MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
2621 .addImm(Val: AMDGPU::DepCtr::encodeFieldVaVdst(VaVdst: 0, STI: ST));
2622
2623 return true;
2624}
2625
2626bool GCNHazardRecognizer::fixVALUTransCoexecutionHazards(MachineInstr *MI) {
2627 if (!ST.hasTransCoexecutionHazard() || // Coexecution disabled.
2628 !SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true) ||
2629 SIInstrInfo::isTRANS(MI: *MI))
2630 return false;
2631
2632 const SIInstrInfo *TII = ST.getInstrInfo();
2633 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2634
2635 auto IsTransHazardFn = [MI, TII, TRI](const MachineInstr &I) {
2636 if (!SIInstrInfo::isTRANS(MI: I))
2637 return false;
2638
2639 // RAW: Trans(I) writes, VALU(MI) reads.
2640 Register TransDef = TII->getNamedOperand(MI: I, OperandName: AMDGPU::OpName::vdst)->getReg();
2641 for (const MachineOperand &ValuUse : MI->explicit_uses()) {
2642 if (ValuUse.isReg() && TRI->regsOverlap(RegA: TransDef, RegB: ValuUse.getReg()))
2643 return true;
2644 }
2645
2646 // WAR: Trans(I) reads, VALU(MI) writes. Use modifiesRegister for
2647 // multi-dest VALUs.
2648 for (const MachineOperand &TransUse : I.explicit_uses()) {
2649 if (TransUse.isReg() && MI->modifiesRegister(Reg: TransUse.getReg(), TRI))
2650 return true;
2651 }
2652
2653 return false;
2654 };
2655
2656 auto IsExpiredFn = [](const MachineInstr &I, int) {
2657 return SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true);
2658 };
2659
2660 const int HasVALU = std::numeric_limits<int>::max();
2661 if (::getWaitStatesSince(IsHazard: IsTransHazardFn, MI, IsExpired: IsExpiredFn) == HasVALU)
2662 return false;
2663
2664 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::V_NOP_e32));
2665 return true;
2666}
2667
2668bool GCNHazardRecognizer::fixWMMAHazards(MachineInstr *MI) {
2669 if (!SIInstrInfo::isWMMA(MI: *MI) && !SIInstrInfo::isSWMMAC(MI: *MI))
2670 return false;
2671
2672 const SIInstrInfo *TII = ST.getInstrInfo();
2673 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2674
2675 auto IsHazardFn = [MI, TII, TRI, this](const MachineInstr &I) {
2676 if (!SIInstrInfo::isWMMA(MI: I) && !SIInstrInfo::isSWMMAC(MI: I))
2677 return false;
2678
2679 // Src0(matrix A) or Src1(matrix B) of the current wmma instruction overlaps
2680 // with the dest(matrix D) of the previous wmma.
2681 const Register CurSrc0Reg =
2682 TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src0)->getReg();
2683 const Register CurSrc1Reg =
2684 TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src1)->getReg();
2685
2686 const Register PrevDstReg =
2687 TII->getNamedOperand(MI: I, OperandName: AMDGPU::OpName::vdst)->getReg();
2688
2689 if (TRI->regsOverlap(RegA: PrevDstReg, RegB: CurSrc0Reg) ||
2690 TRI->regsOverlap(RegA: PrevDstReg, RegB: CurSrc1Reg)) {
2691 return true;
2692 }
2693
2694 // GFX12+ allows overlap of matrix C with PrevDstReg (hardware will stall)
2695 // but Index can't overlap with PrevDstReg.
2696 if (AMDGPU::isGFX12Plus(STI: ST)) {
2697 if (SIInstrInfo::isSWMMAC(MI: *MI)) {
2698 const Register CurIndex =
2699 TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src2)->getReg();
2700 if (TRI->regsOverlap(RegA: PrevDstReg, RegB: CurIndex))
2701 return true;
2702 }
2703 return false;
2704 }
2705
2706 return false;
2707 };
2708
2709 auto IsExpiredFn = [](const MachineInstr &I, int) {
2710 return SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true);
2711 };
2712
2713 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2714 std::numeric_limits<int>::max())
2715 return false;
2716
2717 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::V_NOP_e32));
2718
2719 return true;
2720}
2721
2722static bool isCoexecutableVALUInst(const MachineInstr &MI) {
2723 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false) &&
2724 !SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI);
2725}
2726
2727// Classify XDL WMMA instructions into co-execution hazard categories
2728// (Refer to SPG 4.6.12.1), mainly based on instruction latency.
2729//
2730// Category 0: WMMA with Latency 8
2731// WMMA_*F16, WMMA_*BF16
2732// WMMA_*_16X16X128_{FP8,BF8}
2733// WMMA_*F8F6F4 if SRCA & SRCB are not both F4
2734//
2735// Category 1: WMMA Latency 16
2736// WMMA_IU8
2737//
2738// Category 2: SWMMAC with Latency 8
2739// SWMMAC_*F16, SWMMAC_*BF16,
2740// SWMMAC_*FP8FP8
2741// SWMMAC_*BF8FP8
2742// SWMMAC_*FP8BF8
2743// SWMMAC_*BF8BF8
2744//
2745// Category 3: SWMMAC with Latency 16
2746// SWMMAC_IU8
2747//
2748// Category 4: 16 Pass GFX1251 WMMA with latency 16
2749// V_WMMA_*_16X16X32_{F16,BF16}
2750// V_WMMA_{F32,F16}_16X16X64_{FP8,BF8}*
2751// V_WMMA_F32_16x16x128_F8F6F4 (F4 only)
2752// V_SWMMAC_*_16X16X64_{F16,BF16}
2753// V_SWMMAC_{F32,F16}_16X16X128_{FP8,BF8}*
2754//
2755// Category 5: 32 Pass GFX1251 WMMA with latency 32
2756// V_WMMA_F32_16x16x128_F8F6F4 (not all F4)
2757// V_WMMA_{F32,F16}_16X16X128_{FP8,BF8}*
2758// V_WMMA_F32_32X16X128_F4
2759// V_WMMA_I32_16X16X64_IU8
2760// V_WMMA_I32_16X16X64_IU8
2761//
2762// Category 6: gfx1250 WMMA with Latency 4 (one co-execution slot)
2763// WMMA_*_16X16X64_{FP8,BF8}
2764// WMMA_*F8F6F4 if SRCA & SRCB are both F4
2765static unsigned getWMMAHazardInstInCategory(const MachineInstr &MI,
2766 const SIInstrInfo *TII,
2767 const TargetSchedModel &SchedModel,
2768 const GCNSubtarget &ST) {
2769 assert(TII->isXDLWMMA(MI) && "must be xdl wmma");
2770 bool IsSWMMAC = SIInstrInfo::isSWMMAC(MI);
2771 bool IsLowestRateWMMA = ST.hasGFX125xLowestRateWMMA();
2772 unsigned Category = 0;
2773
2774 unsigned Latency = SchedModel.computeInstrLatency(MI: &MI);
2775 switch (Latency) {
2776 case 4:
2777 // Dense 4-cycle WMMA (gfx1250 16x16x64 FP8/BF8 and f8f6f4 with both
2778 // inputs F4). One co-execution slot; there is no 4-cycle SWMMAC.
2779 assert(!IsSWMMAC && "no 4-cycle SWMMAC expected");
2780 Category = 6;
2781 break;
2782 case 8:
2783 Category = IsSWMMAC ? 2 : 0;
2784 break;
2785 case 16:
2786 Category = IsLowestRateWMMA ? 4 : (IsSWMMAC ? 3 : 1);
2787 break;
2788 case 32:
2789 assert(IsLowestRateWMMA && "latency 32 is not expected");
2790 Category = 5;
2791 break;
2792 default:
2793 llvm_unreachable("unexpected xdl wmma latency");
2794 } // end switch.
2795
2796 return Category;
2797}
2798
2799int GCNHazardRecognizer::checkWMMACoexecutionHazards(MachineInstr *MI) const {
2800 if (!ST.hasWMMACoexecutionHazards())
2801 return 0;
2802
2803 const SIInstrInfo *TII = ST.getInstrInfo();
2804 if (!TII->isXDLWMMA(MI: *MI) && !isCoexecutableVALUInst(MI: *MI))
2805 return 0;
2806
2807 // WaitStates here is the number of V_NOPs or unrelated VALU instructions must
2808 // be in between the first WMMA and the second instruction to cover the hazard
2809 // (WMMAWaitStates if the second is also a WMMA, VALUWaitStates if the second
2810 // is a VALU). Refer to SPG 4.6.12.1. "Requirements for WMMA data hazards" for
2811 // numbers, which depends on the category of the first WMMA.
2812 const int WMMAWaitStates[] = {5, 9, 3, 5, 9, 17, 2};
2813 const int VALUWaitStates[] = {4, 8, 2, 4, 8, 16, 1};
2814 const bool IsWMMA = TII->isXDLWMMA(MI: *MI);
2815 const bool IsLowestRateWMMA = ST.hasGFX125xLowestRateWMMA();
2816 ArrayRef<int> RequiredWaitStates =
2817 IsWMMA ? ArrayRef<int>(WMMAWaitStates) : ArrayRef<int>(VALUWaitStates);
2818 const int MaxWaitStates =
2819 IsWMMA ? (IsLowestRateWMMA ? 17 : 9) : (IsLowestRateWMMA ? 16 : 8);
2820
2821 auto WindowFor = [=](const MachineInstr &I) -> std::optional<int> {
2822 if (!TII->isXDLWMMA(MI: I))
2823 return std::nullopt;
2824 if (IsWMMA ? !hasWMMAToWMMARegOverlap(WMMA: I, MI: *MI)
2825 : !hasWMMAToVALURegOverlap(WMMA: I, MI: *MI))
2826 return std::nullopt;
2827 return RequiredWaitStates[getWMMAHazardInstInCategory(MI: I, TII, SchedModel: TSchedModel,
2828 ST)];
2829 };
2830
2831 return getMaxVALUWindowDeficit(MaxWindow: MaxWaitStates, WindowFor);
2832}
2833
2834bool GCNHazardRecognizer::hasWMMAToWMMARegOverlap(
2835 const MachineInstr &WMMA, const MachineInstr &MI) const {
2836 Register D0 = TII.getNamedOperand(MI: WMMA, OperandName: AMDGPU::OpName::vdst)->getReg();
2837 Register A1 = TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src0)->getReg();
2838 Register B1 = TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src1)->getReg();
2839
2840 // WMMA0 writes (D0), WMMA1 reads (A1/B1/Idx1).
2841 if (TRI.regsOverlap(RegA: D0, RegB: A1) || TRI.regsOverlap(RegA: D0, RegB: B1))
2842 return true;
2843
2844 if (SIInstrInfo::isSWMMAC(MI)) {
2845 Register Idx1 = TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src2)->getReg();
2846 if (TRI.regsOverlap(RegA: D0, RegB: Idx1))
2847 return true;
2848 }
2849 return false;
2850}
2851
2852bool GCNHazardRecognizer::hasWMMAToVALURegOverlap(
2853 const MachineInstr &WMMA, const MachineInstr &MI) const {
2854 // WMMA writes, VALU reads.
2855 Register D0 = TII.getNamedOperand(MI: WMMA, OperandName: AMDGPU::OpName::vdst)->getReg();
2856 for (const MachineOperand &ValuUse : MI.explicit_uses()) {
2857 if (ValuUse.isReg() && TRI.regsOverlap(RegA: D0, RegB: ValuUse.getReg()))
2858 return true;
2859 }
2860
2861 // WMMA reads or writes, VALU writes.
2862 Register A0 = TII.getNamedOperand(MI: WMMA, OperandName: AMDGPU::OpName::src0)->getReg();
2863 Register B0 = TII.getNamedOperand(MI: WMMA, OperandName: AMDGPU::OpName::src1)->getReg();
2864 SmallVector<Register, 4> WMMARegs({D0, A0, B0});
2865
2866 const MachineOperand *Src2 = TII.getNamedOperand(MI: WMMA, OperandName: AMDGPU::OpName::src2);
2867 if (Src2->isReg())
2868 WMMARegs.push_back(Elt: Src2->getReg());
2869
2870 for (const MachineOperand &ValuDef : MI.defs()) {
2871 Register VDstReg = ValuDef.getReg();
2872 for (Register WMMAReg : WMMARegs) {
2873 if (TRI.regsOverlap(RegA: VDstReg, RegB: WMMAReg))
2874 return true;
2875 }
2876 }
2877 return false;
2878}
2879
2880bool GCNHazardRecognizer::isCoexecutionHazardFor(const MachineInstr &I,
2881 const MachineInstr &MI) const {
2882 // I is the potential WMMA hazard source, MI is the instruction being checked
2883 // for hazard.
2884 if (!TII.isXDLWMMA(MI: I))
2885 return false;
2886
2887 // Dispatch based on MI type
2888 if (TII.isXDLWMMA(MI))
2889 return hasWMMAToWMMARegOverlap(WMMA: I, MI);
2890 if (isCoexecutableVALUInst(MI))
2891 return hasWMMAToVALURegOverlap(WMMA: I, MI);
2892
2893 return false;
2894}
2895
2896bool GCNHazardRecognizer::hasWMMAHazardInLoop(MachineLoop *L, MachineInstr *MI,
2897 bool IncludeSubloops) {
2898 // Scan loop for any WMMA that hazards MI.
2899 // TODO: Avoid full loop scan when WMMA is beyond VALU distance.
2900 for (MachineBasicBlock *MBB : L->getBlocks()) {
2901 if (!IncludeSubloops && MLI->getLoopFor(BB: MBB) != L)
2902 continue;
2903 for (MachineInstr &I : *MBB) {
2904 if (&I == MI)
2905 continue;
2906 if (isCoexecutionHazardFor(I, MI: *MI))
2907 return true;
2908 }
2909 }
2910 return false;
2911}
2912
2913bool GCNHazardRecognizer::tryHoistWMMAVnopsFromLoop(MachineInstr *MI,
2914 int WaitStatesNeeded) {
2915 if (!MLI)
2916 return false;
2917
2918 MachineLoop *L = MLI->getLoopFor(BB: MI->getParent());
2919 if (!L) {
2920 ++NumWMMAHoistingBailed;
2921 return false;
2922 }
2923
2924 // If innermost loop has WMMA hazard, we can't hoist at all
2925 if (hasWMMAHazardInLoop(L, MI)) {
2926 ++NumWMMAHoistingBailed;
2927 return false;
2928 }
2929
2930 // Find outermost loop with no internal hazard
2931 MachineLoop *TargetLoop = L;
2932 while (MachineLoop *Parent = TargetLoop->getParentLoop()) {
2933 if (hasWMMAHazardInLoop(L: Parent, MI, IncludeSubloops: false))
2934 break; // Parent has hazard in its own blocks, stop here
2935 TargetLoop = Parent; // Safe to hoist further out
2936 }
2937
2938 // Need valid preheader to insert V_NOPs
2939 MachineBasicBlock *Preheader = TargetLoop->getLoopPreheader();
2940 if (!Preheader) {
2941 ++NumWMMAHoistingBailed;
2942 return false;
2943 }
2944
2945 LLVM_DEBUG(dbgs() << "WMMA V_NOP Hoisting: Moving " << WaitStatesNeeded
2946 << " V_NOPs from loop to " << printMBBReference(*Preheader)
2947 << "\n");
2948
2949 emitVNops(MBB&: *Preheader, InsertPt: Preheader->getFirstTerminator(), WaitStatesNeeded,
2950 /*IsHoisting=*/true);
2951 NumWMMANopsHoisted += WaitStatesNeeded;
2952 return true;
2953}
2954
2955bool GCNHazardRecognizer::fixWMMACoexecutionHazards(MachineInstr *MI) {
2956 int WaitStatesNeeded = checkWMMACoexecutionHazards(MI);
2957 if (WaitStatesNeeded <= 0)
2958 return false;
2959
2960 if (EnableWMMAVnopHoisting && tryHoistWMMAVnopsFromLoop(MI, WaitStatesNeeded))
2961 return true;
2962
2963 emitVNops(MBB&: *MI->getParent(), InsertPt: MI->getIterator(), WaitStatesNeeded);
2964 return true;
2965}
2966
2967bool GCNHazardRecognizer::fixShift64HighRegBug(MachineInstr *MI) {
2968 if (!ST.hasShift64HighRegBug())
2969 return false;
2970 assert(!ST.hasExtendedWaitCounts());
2971
2972 switch (MI->getOpcode()) {
2973 default:
2974 return false;
2975 case AMDGPU::V_LSHLREV_B64_e64:
2976 case AMDGPU::V_LSHRREV_B64_e64:
2977 case AMDGPU::V_ASHRREV_I64_e64:
2978 break;
2979 }
2980
2981 MachineOperand *Amt = TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src0);
2982 if (!Amt->isReg())
2983 return false;
2984
2985 Register AmtReg = Amt->getReg();
2986 const MachineRegisterInfo &MRI = MF.getRegInfo();
2987 // Check if this is a last VGPR in the allocation block.
2988 if (!TRI.isVGPR(MRI, Reg: AmtReg) || ((AmtReg - AMDGPU::VGPR0) & 7) != 7)
2989 return false;
2990
2991 if (AmtReg != AMDGPU::VGPR255 && MRI.isPhysRegUsed(PhysReg: AmtReg + 1))
2992 return false;
2993
2994 assert(ST.needsAlignedVGPRs());
2995 static_assert(AMDGPU::VGPR0 + 1 == AMDGPU::VGPR1);
2996
2997 const DebugLoc &DL = MI->getDebugLoc();
2998 MachineBasicBlock *MBB = MI->getParent();
2999 MachineOperand *Src1 = TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src1);
3000
3001 // In:
3002 //
3003 // Dst = shiftrev64 Amt, Src1
3004 //
3005 // if Dst!=Src1 then avoid the bug with:
3006 //
3007 // Dst.sub0 = Amt
3008 // Dst = shift64 Dst.sub0, Src1
3009
3010 Register DstReg = MI->getOperand(i: 0).getReg();
3011 if (!Src1->isReg() || Src1->getReg() != DstReg) {
3012 Register DstLo = TRI.getSubReg(Reg: DstReg, Idx: AMDGPU::sub0);
3013 runOnInstruction(
3014 MI: BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstLo).add(MO: *Amt));
3015 Amt->setReg(DstLo);
3016 Amt->setIsKill(true);
3017 return true;
3018 }
3019
3020 bool Overlapped = MI->modifiesRegister(Reg: AmtReg, TRI: &TRI);
3021 Register NewReg;
3022 for (MCRegister Reg : Overlapped ? AMDGPU::VReg_64_Align2RegClass
3023 : AMDGPU::VGPR_32RegClass) {
3024 if (!MI->modifiesRegister(Reg, TRI: &TRI) && !MI->readsRegister(Reg, TRI: &TRI)) {
3025 NewReg = Reg;
3026 break;
3027 }
3028 }
3029
3030 Register NewAmt = Overlapped ? (Register)TRI.getSubReg(Reg: NewReg, Idx: AMDGPU::sub1)
3031 : NewReg;
3032 Register NewAmtLo;
3033
3034 if (Overlapped)
3035 NewAmtLo = TRI.getSubReg(Reg: NewReg, Idx: AMDGPU::sub0);
3036
3037 // Insert a full wait count because found register might be pending a wait.
3038 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT))
3039 .addImm(Val: 0);
3040
3041 // Insert V_SWAP_B32 instruction(s) and run hazard recognizer on them.
3042 if (Overlapped)
3043 runOnInstruction(
3044 MI: BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SWAP_B32), DestReg: NewAmtLo)
3045 .addDef(RegNo: AmtReg - 1)
3046 .addReg(RegNo: AmtReg - 1, Flags: RegState::Undef)
3047 .addReg(RegNo: NewAmtLo, Flags: RegState::Undef));
3048 runOnInstruction(MI: BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SWAP_B32), DestReg: NewAmt)
3049 .addDef(RegNo: AmtReg)
3050 .addReg(RegNo: AmtReg, Flags: RegState::Undef)
3051 .addReg(RegNo: NewAmt, Flags: RegState::Undef));
3052
3053 // Instructions emitted after the current instruction will be processed by the
3054 // parent loop of the hazard recognizer in a natural way.
3055 BuildMI(BB&: *MBB, I: std::next(x: MI->getIterator()), MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SWAP_B32),
3056 DestReg: AmtReg)
3057 .addDef(RegNo: NewAmt)
3058 .addReg(RegNo: NewAmt)
3059 .addReg(RegNo: AmtReg);
3060 if (Overlapped)
3061 BuildMI(BB&: *MBB, I: std::next(x: MI->getIterator()), MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SWAP_B32),
3062 DestReg: AmtReg - 1)
3063 .addDef(RegNo: NewAmtLo)
3064 .addReg(RegNo: NewAmtLo)
3065 .addReg(RegNo: AmtReg - 1);
3066
3067 // Re-running hazard recognizer on the modified instruction is not necessary,
3068 // inserted V_SWAP_B32 has already both read and write new registers so
3069 // hazards related to these register has already been handled.
3070 Amt->setReg(NewAmt);
3071 Amt->setIsKill(false);
3072 // We do not update liveness, so verifier may see it as undef.
3073 Amt->setIsUndef();
3074 if (Overlapped) {
3075 MI->getOperand(i: 0).setReg(NewReg);
3076 Src1->setReg(NewReg);
3077 Src1->setIsKill(false);
3078 Src1->setIsUndef();
3079 }
3080
3081 return true;
3082}
3083
3084// Handle GFX950 v_pk_{add,mul,fma}_f32 invalid op_sels
3085// 1. Fix: opsel[1:0]==2’b10 : Packed ADD_F32/MUL_F32
3086// 2. Fail: opsel[2:1]==2'b01/2'b10/2'b11 : Packed FMA_F32
3087bool GCNHazardRecognizer::fixPkF32InvalidOpSel(MachineInstr *MI) {
3088 if (!ST.hasPkF32InvalidOpSel())
3089 return false;
3090
3091 switch (MI->getOpcode()) {
3092 default:
3093 return false;
3094 case AMDGPU::V_PK_ADD_F32:
3095 case AMDGPU::V_PK_MUL_F32:
3096 case AMDGPU::V_PK_FMA_F32:
3097 break;
3098 }
3099
3100 const MachineOperand *Src0Mods =
3101 TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src0_modifiers);
3102 const MachineOperand *Src1Mods =
3103 TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src1_modifiers);
3104 if (!Src0Mods || !Src1Mods)
3105 return false;
3106
3107 if (MI->getOpcode() != AMDGPU::V_PK_FMA_F32) {
3108 if (!(Src0Mods->getImm() & SISrcMods::OP_SEL_0) &&
3109 (Src1Mods->getImm() & SISrcMods::OP_SEL_0)) {
3110 if (!TII.commuteInstruction(MI&: *MI)) {
3111 reportFatalInternalError(
3112 reason: "cannot swap the GFX950 v_pk_add/mul_f32 src0 and src1: " +
3113 Twine(TII.getName(Opcode: MI->getOpcode())));
3114 }
3115 return true;
3116 }
3117 return false;
3118 }
3119
3120 // Safety net: v_pk_fma_f32 invalid opsels are expected to already
3121 // be prevented earlier, so just fail loudly if it is ever seen here
3122 // instead of silently miscompiling, as a safeguard against future changes.
3123 if (!TII.isLegalPkFMAF32OpSel(MI: *MI)) {
3124 reportFatalInternalError(reason: "GFX950 v_pk_fma_f32 has an invalid op_sel that "
3125 "should have been legalized earlier in the "
3126 "pipeline");
3127 }
3128
3129 return false;
3130}
3131
3132int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) const {
3133 int NSAtoVMEMWaitStates = 1;
3134
3135 if (!ST.hasNSAtoVMEMBug())
3136 return 0;
3137
3138 if (!SIInstrInfo::isMUBUF(MI: *MI) && !SIInstrInfo::isMTBUF(MI: *MI))
3139 return 0;
3140
3141 const SIInstrInfo *TII = ST.getInstrInfo();
3142 const auto *Offset = TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::offset);
3143 if (!Offset || (Offset->getImm() & 6) == 0)
3144 return 0;
3145
3146 auto IsHazardFn = [TII](const MachineInstr &I) {
3147 if (!SIInstrInfo::isMIMG(MI: I))
3148 return false;
3149 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc: I.getOpcode());
3150 return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA &&
3151 TII->getInstSizeInBytes(MI: I) >= 16;
3152 };
3153
3154 return NSAtoVMEMWaitStates - getWaitStatesSince(IsHazard: IsHazardFn, Limit: 1);
3155}
3156
3157int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(
3158 MachineInstr *MI) const {
3159 int FPAtomicToDenormModeWaitStates = 3;
3160
3161 if (!ST.hasFPAtomicToDenormModeHazard())
3162 return 0;
3163 assert(!ST.hasExtendedWaitCounts());
3164
3165 if (MI->getOpcode() != AMDGPU::S_DENORM_MODE)
3166 return 0;
3167
3168 auto IsHazardFn = [](const MachineInstr &I) {
3169 if (!SIInstrInfo::isVMEM(MI: I))
3170 return false;
3171 return SIInstrInfo::isFPAtomic(MI: I);
3172 };
3173
3174 auto IsExpiredFn = [](const MachineInstr &MI, int WaitStates) {
3175 if (WaitStates >= 3 || SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
3176 return true;
3177
3178 return SIInstrInfo::isWaitcnt(Opcode: MI.getOpcode());
3179 };
3180
3181 return FPAtomicToDenormModeWaitStates -
3182 ::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn);
3183}
3184
3185int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) const {
3186 assert(SIInstrInfo::isMAI(*MI));
3187
3188 return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI);
3189}
3190
3191int GCNHazardRecognizer::checkMFMAPadding(MachineInstr *MI) const {
3192 // Early exit if no padding is requested.
3193 if (MFMAPaddingRatio == 0)
3194 return 0;
3195
3196 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
3197 if (!SIInstrInfo::isMFMA(MI: *MI) || MFI->getOccupancy() < 2)
3198 return 0;
3199
3200 int NeighborMFMALatency = 0;
3201 auto IsNeighboringMFMA = [&NeighborMFMALatency,
3202 this](const MachineInstr &MI) {
3203 if (!SIInstrInfo::isMFMA(MI))
3204 return false;
3205
3206 NeighborMFMALatency = this->getMFMAPipelineWaitStates(MI);
3207 return true;
3208 };
3209
3210 const int MaxMFMAPipelineWaitStates = 16;
3211 int WaitStatesSinceNeighborMFMA =
3212 getWaitStatesSince(IsHazard: IsNeighboringMFMA, Limit: MaxMFMAPipelineWaitStates);
3213
3214 int NeighborMFMAPaddingNeeded =
3215 (NeighborMFMALatency * MFMAPaddingRatio / 100) -
3216 WaitStatesSinceNeighborMFMA;
3217
3218 return std::max(a: 0, b: NeighborMFMAPaddingNeeded);
3219}
3220
3221int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) const {
3222 int WaitStatesNeeded = 0;
3223 unsigned Opc = MI->getOpcode();
3224
3225 auto IsVALUFn = [](const MachineInstr &MI) {
3226 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) || MI.isInlineAsm();
3227 };
3228
3229 if (Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) { // MFMA or v_accvgpr_write
3230 const int LegacyVALUWritesVGPRWaitStates = 2;
3231 const int VALUWritesExecWaitStates = 4;
3232 const int MaxWaitStates = 4;
3233
3234 int WaitStatesNeededForUse = VALUWritesExecWaitStates -
3235 getWaitStatesSinceDef(Reg: AMDGPU::EXEC, IsHazardDef: IsVALUFn, Limit: MaxWaitStates);
3236 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3237
3238 if (WaitStatesNeeded < MaxWaitStates) {
3239 for (const MachineOperand &Use : MI->explicit_uses()) {
3240 const int MaxWaitStates = 2;
3241
3242 if (!Use.isReg() || !TRI.isVGPR(MRI: MF.getRegInfo(), Reg: Use.getReg()))
3243 continue;
3244
3245 int WaitStatesNeededForUse = LegacyVALUWritesVGPRWaitStates -
3246 getWaitStatesSinceDef(Reg: Use.getReg(), IsHazardDef: IsVALUFn, Limit: MaxWaitStates);
3247 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3248
3249 if (WaitStatesNeeded == MaxWaitStates)
3250 break;
3251 }
3252 }
3253 }
3254
3255 for (const MachineOperand &Op : MI->explicit_operands()) {
3256 if (!Op.isReg() || !TRI.isAGPR(MRI: MF.getRegInfo(), Reg: Op.getReg()))
3257 continue;
3258
3259 if (Op.isDef() && Opc != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
3260 continue;
3261
3262 const int MFMAWritesAGPROverlappedSrcABWaitStates = 4;
3263 const int MFMAWritesAGPROverlappedSrcCWaitStates = 2;
3264 const int MFMA4x4WritesAGPRAccVgprReadWaitStates = 4;
3265 const int MFMA16x16WritesAGPRAccVgprReadWaitStates = 10;
3266 const int MFMA32x32WritesAGPRAccVgprReadWaitStates = 18;
3267 const int MFMA4x4WritesAGPRAccVgprWriteWaitStates = 1;
3268 const int MFMA16x16WritesAGPRAccVgprWriteWaitStates = 7;
3269 const int MFMA32x32WritesAGPRAccVgprWriteWaitStates = 15;
3270 const int MaxWaitStates = 18;
3271 Register Reg = Op.getReg();
3272 unsigned HazardDefLatency = 0;
3273
3274 auto IsOverlappedMFMAFn = [Reg, &HazardDefLatency,
3275 this](const MachineInstr &MI) {
3276 if (!SIInstrInfo::isMFMA(MI))
3277 return false;
3278 Register DstReg = MI.getOperand(i: 0).getReg();
3279 if (DstReg == Reg)
3280 return false;
3281 HazardDefLatency =
3282 std::max(a: HazardDefLatency, b: TSchedModel.computeInstrLatency(MI: &MI));
3283 return TRI.regsOverlap(RegA: DstReg, RegB: Reg);
3284 };
3285
3286 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsHazardDef: IsOverlappedMFMAFn,
3287 Limit: MaxWaitStates);
3288 int NeedWaitStates = MFMAWritesAGPROverlappedSrcABWaitStates;
3289 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2);
3290 int OpNo = Op.getOperandNo();
3291 if (OpNo == SrcCIdx) {
3292 NeedWaitStates = MFMAWritesAGPROverlappedSrcCWaitStates;
3293 } else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) {
3294 switch (HazardDefLatency) {
3295 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprReadWaitStates;
3296 break;
3297 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprReadWaitStates;
3298 break;
3299 case 16: [[fallthrough]];
3300 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprReadWaitStates;
3301 break;
3302 }
3303 } else if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
3304 switch (HazardDefLatency) {
3305 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprWriteWaitStates;
3306 break;
3307 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprWriteWaitStates;
3308 break;
3309 case 16: [[fallthrough]];
3310 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprWriteWaitStates;
3311 break;
3312 }
3313 }
3314
3315 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
3316 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3317
3318 if (WaitStatesNeeded == MaxWaitStates)
3319 return WaitStatesNeeded; // Early exit.
3320
3321 auto IsAccVgprWriteFn = [Reg, this](const MachineInstr &MI) {
3322 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
3323 return false;
3324 Register DstReg = MI.getOperand(i: 0).getReg();
3325 return TRI.regsOverlap(RegA: Reg, RegB: DstReg);
3326 };
3327
3328 const int AccVGPRWriteMFMAReadSrcCWaitStates = 1;
3329 const int AccVGPRWriteMFMAReadSrcABWaitStates = 3;
3330 const int AccVGPRWriteAccVgprReadWaitStates = 3;
3331 NeedWaitStates = AccVGPRWriteMFMAReadSrcABWaitStates;
3332 if (OpNo == SrcCIdx)
3333 NeedWaitStates = AccVGPRWriteMFMAReadSrcCWaitStates;
3334 else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64)
3335 NeedWaitStates = AccVGPRWriteAccVgprReadWaitStates;
3336
3337 WaitStatesNeededForUse = NeedWaitStates -
3338 getWaitStatesSinceDef(Reg, IsHazardDef: IsAccVgprWriteFn, Limit: MaxWaitStates);
3339 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3340
3341 if (WaitStatesNeeded == MaxWaitStates)
3342 return WaitStatesNeeded; // Early exit.
3343 }
3344
3345 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
3346 const int MFMA4x4ReadSrcCAccVgprWriteWaitStates = 0;
3347 const int MFMA16x16ReadSrcCAccVgprWriteWaitStates = 5;
3348 const int MFMA32x32ReadSrcCAccVgprWriteWaitStates = 13;
3349 const int MaxWaitStates = 13;
3350 Register DstReg = MI->getOperand(i: 0).getReg();
3351 unsigned HazardDefLatency = 0;
3352
3353 auto IsSrcCMFMAFn = [DstReg, &HazardDefLatency,
3354 this](const MachineInstr &MI) {
3355 if (!SIInstrInfo::isMFMA(MI))
3356 return false;
3357 Register Reg = TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src2)->getReg();
3358 HazardDefLatency =
3359 std::max(a: HazardDefLatency, b: TSchedModel.computeInstrLatency(MI: &MI));
3360 return TRI.regsOverlap(RegA: Reg, RegB: DstReg);
3361 };
3362
3363 int WaitStatesSince = getWaitStatesSince(IsHazard: IsSrcCMFMAFn, Limit: MaxWaitStates);
3364 int NeedWaitStates;
3365 switch (HazardDefLatency) {
3366 case 2: NeedWaitStates = MFMA4x4ReadSrcCAccVgprWriteWaitStates;
3367 break;
3368 case 8: NeedWaitStates = MFMA16x16ReadSrcCAccVgprWriteWaitStates;
3369 break;
3370 case 16: [[fallthrough]];
3371 default: NeedWaitStates = MFMA32x32ReadSrcCAccVgprWriteWaitStates;
3372 break;
3373 }
3374
3375 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSince;
3376 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3377 }
3378
3379 // Pad neighboring MFMA with noops for better inter-wave performance.
3380 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: checkMFMAPadding(MI));
3381
3382 return WaitStatesNeeded;
3383}
3384
3385/// One MFMA can be written with up to four opcodes that differ only in how vdst
3386/// and src2 are encoded: both are either AGPRs or VGPRs, and the mac form ties
3387/// vdst to src2 instead of taking them as separate operands. \returns the AGPR,
3388/// non-mac opcode, so that every form of the same MFMA maps to one value.
3389static unsigned getMFMANonMacAGPRFormOp(unsigned Opc) {
3390 if (int NonMacOp = AMDGPU::getMFMAEarlyClobberOp(Opcode: Opc); NonMacOp != -1)
3391 Opc = NonMacOp;
3392 if (int AGPROp = AMDGPU::getAGPRFormOp(Opcode: Opc); AGPROp != -1)
3393 Opc = AGPROp;
3394 return Opc;
3395}
3396
3397/// \returns true if \p Opc0 and \p Opc1 are the same MFMA, ignoring the mac
3398/// form and whether vdst/src2 are AGPRs or VGPRs.
3399static bool isSameMFMA(unsigned Opc0, unsigned Opc1) {
3400 return getMFMANonMacAGPRFormOp(Opc: Opc0) == getMFMANonMacAGPRFormOp(Opc: Opc1);
3401}
3402
3403static int
3404GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates(int NumPasses,
3405 bool IsGFX950) {
3406 // xdl def cycles | gfx940 | gfx950
3407 // 2 pass | 3 4
3408 // 4 pass | 5 6
3409 // 8 pass | 9 10
3410 // 16 pass | 17 18
3411 return NumPasses + 1 + IsGFX950;
3412}
3413
3414static int
3415GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates(int NumPasses,
3416 bool IsGFX950) {
3417 // xdl def cycles | gfx940 | gfx950
3418 // 2 pass | 3 3
3419 // 4 pass | 5 6
3420 // 8 pass | 9 10
3421 // 16 pass | 17 18
3422 return NumPasses + 1 + (NumPasses != 2 && IsGFX950);
3423}
3424
3425static int
3426GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates(int NumPasses) {
3427 // 2 pass -> 2
3428 // 4 pass -> 4
3429 // 8 pass -> 8
3430 // 16 pass -> 16
3431 return NumPasses;
3432}
3433
3434static int
3435GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses) {
3436 // 2 pass -> 4
3437 // 4 pass -> 6
3438 // 8 pass -> 10
3439 // 16 pass -> 18
3440 return NumPasses + 2;
3441}
3442
3443static int GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses,
3444 bool IsGFX950) {
3445 // xdl def cycles | gfx942 | gfx950
3446 // 2 pass | 5 5
3447 // 4 pass | 7 8
3448 // 8 pass | 11 12
3449 // 16 pass | 19 20
3450 return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
3451}
3452
3453int GCNHazardRecognizer::getMFMAOverlappedSrcCWaitStates(
3454 const MachineInstr *Reader, const MachineInstr *Writer) const {
3455 constexpr int SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates = 2;
3456 constexpr int SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates = 8;
3457 constexpr int SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates = 16;
3458 constexpr int SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates = 3;
3459 constexpr int SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates = 9;
3460 constexpr int SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates = 17;
3461 constexpr int DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 9;
3462 constexpr int GFX950_DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 17;
3463 constexpr int DMFMA4x4WritesVGPROverlappedSrcCWaitStates = 4;
3464
3465 // An XDL read of a non-XDL result needs no wait states. DGEMM is never XDL,
3466 // so this also covers the f64 writers handled below.
3467 if (TII.isXDL(MI: *Reader) && !TII.isXDL(MI: *Writer))
3468 return 0;
3469
3470 switch (Writer->getOpcode()) {
3471 case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
3472 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
3473 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
3474 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
3475 return ST.hasGFX950Insts()
3476 ? GFX950_DMFMA16x16WritesVGPROverlappedSrcCWaitStates
3477 : DMFMA16x16WritesVGPROverlappedSrcCWaitStates;
3478 case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
3479 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
3480 return DMFMA4x4WritesVGPROverlappedSrcCWaitStates;
3481 default:
3482 break;
3483 }
3484
3485 int NumPasses = TSchedModel.computeInstrLatency(MI: Writer);
3486 if (ST.hasGFX940Insts()) {
3487 if (!TII.isXDL(MI: *Writer))
3488 return GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates(
3489 NumPasses);
3490 return TII.isXDL(MI: *Reader)
3491 ? GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates(
3492 NumPasses, IsGFX950: ST.hasGFX950Insts())
3493 : GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates(
3494 NumPasses, IsGFX950: ST.hasGFX950Insts());
3495 }
3496
3497 bool IsDGEMM = SIInstrInfo::isDGEMM(Opcode: Reader->getOpcode());
3498 switch (NumPasses) {
3499 case 2:
3500 return IsDGEMM ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates
3501 : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates;
3502 case 8:
3503 return IsDGEMM ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates
3504 : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates;
3505 case 16:
3506 return IsDGEMM ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates
3507 : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates;
3508 default:
3509 llvm_unreachable("unexpected number of passes");
3510 }
3511}
3512
3513int GCNHazardRecognizer::getMFMAReadWaitStates(const MachineInstr &Consumer,
3514 const MachineInstr &Producer,
3515 Register Reg,
3516 bool IsSrcC) const {
3517 unsigned Opc = Consumer.getOpcode();
3518 const int SMFMA4x4WritesVGPROverlappedSrcABWaitStates = 5;
3519 const int SMFMA16x16WritesVGPROverlappedSrcABWaitStates = 11;
3520 const int SMFMA32x32WritesVGPROverlappedSrcABWaitStates = 19;
3521 const int DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates = 6;
3522 const int DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 11;
3523 const int GFX950_DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 19;
3524 const int DMFMA4x4WritesVGPRFullSrcCWaitStates = 4;
3525 const int GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates = 2;
3526
3527 bool FullReg = Producer.getOperand(i: 0).getReg() == Reg;
3528 unsigned Opc1 = Producer.getOpcode();
3529 int NeedWaitStates = 0;
3530 if (IsSrcC) {
3531 if (!SIInstrInfo::isDGEMM(Opcode: Opc) &&
3532 (!ST.hasGFX940Insts() && SIInstrInfo::isDGEMM(Opcode: Opc1))) {
3533 NeedWaitStates = 0;
3534 } else if (FullReg) {
3535 if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
3536 Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) &&
3537 (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
3538 Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64))
3539 NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates;
3540 else if (ST.hasGFX940Insts() &&
3541 TSchedModel.computeInstrLatency(MI: &Producer) == 2)
3542 NeedWaitStates = GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates;
3543
3544 // The accumulator forwarding path that allows zero wait states is only
3545 // available while the chain stays on a single MFMA. Two different MFMAs
3546 // sharing an accumulator need the wait states of a partial overlap.
3547 if (ST.hasGFX940Insts() && !isSameMFMA(Opc0: Opc, Opc1)) {
3548 NeedWaitStates =
3549 std::max(a: NeedWaitStates,
3550 b: getMFMAOverlappedSrcCWaitStates(Reader: &Consumer, Writer: &Producer));
3551 }
3552 } else {
3553 NeedWaitStates = getMFMAOverlappedSrcCWaitStates(Reader: &Consumer, Writer: &Producer);
3554 }
3555 } else {
3556 switch (Opc1) {
3557 case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
3558 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
3559 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
3560 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
3561 NeedWaitStates =
3562 ST.hasGFX950Insts()
3563 ? GFX950_DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates
3564 : DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates;
3565 break;
3566 case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
3567 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
3568 NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates;
3569 break;
3570 default:
3571 int NumPasses = TSchedModel.computeInstrLatency(MI: &Producer);
3572
3573 if (ST.hasGFX940Insts()) {
3574 NeedWaitStates =
3575 TII.isXDL(MI: Producer)
3576 ? GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(
3577 NumPasses, IsGFX950: ST.hasGFX950Insts())
3578 : GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates(
3579 NumPasses);
3580 break;
3581 }
3582
3583 switch (NumPasses) {
3584 case 2:
3585 NeedWaitStates = SMFMA4x4WritesVGPROverlappedSrcABWaitStates;
3586 break;
3587 case 4:
3588 llvm_unreachable("unexpected number of passes for mfma");
3589 case 8:
3590 NeedWaitStates = SMFMA16x16WritesVGPROverlappedSrcABWaitStates;
3591 break;
3592 case 16:
3593 default:
3594 NeedWaitStates = SMFMA32x32WritesVGPROverlappedSrcABWaitStates;
3595 }
3596 }
3597 }
3598 return NeedWaitStates;
3599}
3600
3601int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) const {
3602 int WaitStatesNeeded = 0;
3603 unsigned Opc = MI->getOpcode();
3604
3605 auto IsLegacyVALUFn = [](const MachineInstr &MI) {
3606 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
3607 !SIInstrInfo::isMFMA(MI);
3608 };
3609
3610 auto IsLegacyVALUNotDotFn = [](const MachineInstr &MI) {
3611 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
3612 !SIInstrInfo::isMFMA(MI) && !SIInstrInfo::isDOT(MI);
3613 };
3614
3615 if (!SIInstrInfo::isMFMA(MI: *MI))
3616 return WaitStatesNeeded;
3617
3618 const int VALUWritesExecWaitStates = 4;
3619 int WaitStatesNeededForUse = VALUWritesExecWaitStates -
3620 getWaitStatesSinceDef(Reg: AMDGPU::EXEC, IsHazardDef: IsLegacyVALUFn,
3621 Limit: VALUWritesExecWaitStates);
3622 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3623
3624 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2);
3625
3626 // Loop for both DGEMM and S/HGEMM 2nd instruction.
3627 for (const MachineOperand &Use : MI->explicit_uses()) {
3628 const int LegacyVALUNotDotWritesVGPRWaitStates = 2;
3629 const int MaxWaitStates =
3630 GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(
3631 NumPasses: 16, IsGFX950: ST.hasGFX950Insts());
3632
3633 if (!Use.isReg())
3634 continue;
3635 Register Reg = Use.getReg();
3636
3637 WaitStatesNeededForUse = LegacyVALUNotDotWritesVGPRWaitStates -
3638 getWaitStatesSinceDef(Reg, IsHazardDef: IsLegacyVALUNotDotFn, Limit: MaxWaitStates);
3639 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3640
3641 int OpNo = Use.getOperandNo();
3642 WaitStatesNeeded = std::max(
3643 a: WaitStatesNeeded,
3644 b: getMaxWindowDeficit(
3645 MaxWindow: MaxWaitStates, WindowFor: [&](const MachineInstr &P) -> std::optional<int> {
3646 if (!SIInstrInfo::isMFMA(MI: P) ||
3647 !TRI.regsOverlap(RegA: P.getOperand(i: 0).getReg(), RegB: Reg))
3648 return std::nullopt;
3649 return getMFMAReadWaitStates(Consumer: *MI, Producer: P, Reg, IsSrcC: OpNo == SrcCIdx);
3650 }));
3651
3652 if (WaitStatesNeeded == MaxWaitStates)
3653 break;
3654 }
3655
3656 // Pad neighboring MFMA with noops for better inter-wave performance.
3657 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: checkMFMAPadding(MI));
3658
3659 return WaitStatesNeeded;
3660}
3661
3662int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) const {
3663 // On gfx90a+ relevant hazards are checked in checkMAIVALUHazards()
3664 if (!ST.hasMAIInsts() || ST.hasGFX90AInsts())
3665 return 0;
3666
3667 int WaitStatesNeeded = 0;
3668
3669 auto IsAccVgprReadFn = [](const MachineInstr &MI) {
3670 return MI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64;
3671 };
3672
3673 for (const MachineOperand &Op : MI->explicit_uses()) {
3674 if (!Op.isReg() || !TRI.isVGPR(MRI: MF.getRegInfo(), Reg: Op.getReg()))
3675 continue;
3676
3677 Register Reg = Op.getReg();
3678
3679 const int AccVgprReadLdStWaitStates = 2;
3680 const int VALUWriteAccVgprRdWrLdStDepVALUWaitStates = 1;
3681 const int MaxWaitStates = 2;
3682
3683 int WaitStatesNeededForUse = AccVgprReadLdStWaitStates -
3684 getWaitStatesSinceDef(Reg, IsHazardDef: IsAccVgprReadFn, Limit: MaxWaitStates);
3685 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3686
3687 if (WaitStatesNeeded == MaxWaitStates)
3688 return WaitStatesNeeded; // Early exit.
3689
3690 auto IsVALUAccVgprRdWrCheckFn = [Reg, this](const MachineInstr &MI) {
3691 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 &&
3692 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
3693 return false;
3694 auto IsVALUFn = [](const MachineInstr &MI) {
3695 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
3696 !SIInstrInfo::isMAI(MI);
3697 };
3698 return getWaitStatesSinceDef(Reg, IsHazardDef: IsVALUFn, Limit: 2 /*MaxWaitStates*/) <
3699 std::numeric_limits<int>::max();
3700 };
3701
3702 WaitStatesNeededForUse = VALUWriteAccVgprRdWrLdStDepVALUWaitStates -
3703 getWaitStatesSince(IsHazard: IsVALUAccVgprRdWrCheckFn, Limit: MaxWaitStates);
3704 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3705 }
3706
3707 return WaitStatesNeeded;
3708}
3709
3710int GCNHazardRecognizer::checkPermlaneHazards(MachineInstr *MI) const {
3711 assert(!ST.hasVcmpxPermlaneHazard() &&
3712 "this is a different vcmpx+permlane hazard");
3713 const SIRegisterInfo *TRI = ST.getRegisterInfo();
3714 const SIInstrInfo *TII = ST.getInstrInfo();
3715
3716 auto IsVCmpXWritesExecFn = [TII, TRI](const MachineInstr &MI) {
3717 return isVCmpXWritesExec(TII: *TII, TRI: *TRI, MI);
3718 };
3719
3720 auto IsVALUFn = [](const MachineInstr &MI) {
3721 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true);
3722 };
3723
3724 const int VCmpXWritesExecWaitStates = 4;
3725 const int VALUWritesVDstWaitStates = 2;
3726 int WaitStatesNeeded = 0;
3727
3728 for (const MachineOperand &Op : MI->explicit_uses()) {
3729 if (!Op.isReg() || !TRI->isVGPR(MRI: MF.getRegInfo(), Reg: Op.getReg()))
3730 continue;
3731 Register Reg = Op.getReg();
3732
3733 int WaitStatesSinceDef =
3734 VALUWritesVDstWaitStates -
3735 getWaitStatesSinceDef(Reg, IsHazardDef: IsVALUFn,
3736 /*MaxWaitStates=*/Limit: VALUWritesVDstWaitStates);
3737 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesSinceDef);
3738 if (WaitStatesNeeded >= VALUWritesVDstWaitStates)
3739 break;
3740 }
3741
3742 int VCmpXHazardWaits =
3743 VCmpXWritesExecWaitStates -
3744 getWaitStatesSince(IsHazard: IsVCmpXWritesExecFn, Limit: VCmpXWritesExecWaitStates);
3745
3746 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: VCmpXHazardWaits);
3747 return WaitStatesNeeded;
3748}
3749
3750static int GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(int NumPasses) {
3751 // 2 pass -> 4
3752 // 4 pass -> 6
3753 // 8 pass -> 10
3754 // 16 pass -> 18
3755 return NumPasses + 2;
3756}
3757
3758static int GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(int NumPasses,
3759 bool IsGFX950) {
3760 // xdl def cycles | gfx942 | gfx950
3761 // 2 pass | 5 5
3762 // 4 pass | 7 8
3763 // 8 pass | 11 12
3764 // 16 pass | 19 20
3765 return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
3766}
3767
3768static int GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses,
3769 bool IsGFX950) {
3770 // xdl def cycles | gfx942 | gfx950
3771 // 2 pass | 5 5
3772 // 4 pass | 7 8
3773 // 8 pass | 11 12
3774 // 16 pass | 19 20
3775 return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
3776}
3777
3778static int GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses) {
3779 // 2 pass -> 4
3780 // 4 pass -> 6
3781 // 8 pass -> 10
3782 // 16 pass -> 18
3783 return NumPasses + 2;
3784}
3785
3786int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const {
3787 if (!ST.hasGFX90AInsts())
3788 return 0;
3789
3790 auto IsDGEMMFn = [](const MachineInstr &MI) -> bool {
3791 return SIInstrInfo::isDGEMM(Opcode: MI.getOpcode());
3792 };
3793
3794 // This is checked in checkMAIHazards90A()
3795 if (SIInstrInfo::isMFMA(MI: *MI))
3796 return 0;
3797
3798 const MachineRegisterInfo &MRI = MF.getRegInfo();
3799
3800 int WaitStatesNeeded = 0;
3801
3802 bool IsMem = SIInstrInfo::isVMEM(MI: *MI) || SIInstrInfo::isDS(MI: *MI);
3803 bool IsMemOrExport = IsMem || SIInstrInfo::isEXP(MI: *MI);
3804 bool IsVALU = SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true);
3805
3806 unsigned Reg;
3807 auto IsMFMAWriteFn = [&Reg, this](const MachineInstr &MI) {
3808 if (!SIInstrInfo::isMFMA(MI) ||
3809 !TRI.regsOverlap(RegA: MI.getOperand(i: 0).getReg(), RegB: Reg))
3810 return false;
3811 return true;
3812 };
3813
3814 const MachineInstr *DOT = nullptr;
3815 auto IsDotWriteFn = [&Reg, &DOT, this](const MachineInstr &MI) {
3816 if (!SIInstrInfo::isDOT(MI) ||
3817 !TRI.regsOverlap(RegA: MI.getOperand(i: 0).getReg(), RegB: Reg))
3818 return false;
3819 DOT = &MI;
3820 return true;
3821 };
3822
3823 bool DGEMMAfterVALUWrite = false;
3824 auto IsDGEMMHazard = [&DGEMMAfterVALUWrite, this](const MachineInstr &MI) {
3825 // Found DGEMM on reverse traversal to def.
3826 if (SIInstrInfo::isDGEMM(Opcode: MI.getOpcode()))
3827 DGEMMAfterVALUWrite = true;
3828
3829 // Only hazard if register is defined by a VALU and a DGEMM is found after
3830 // after the def.
3831 if (!TII.isVALU(MI, /*AllowLDSDMA=*/true) || !DGEMMAfterVALUWrite)
3832 return false;
3833
3834 return true;
3835 };
3836
3837 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opcode: MI->getOpcode(),
3838 Name: AMDGPU::OpName::src2);
3839
3840 if (IsMemOrExport || IsVALU) {
3841 const int SMFMA4x4WriteVgprVALUMemExpReadWaitStates = 5;
3842 const int SMFMA16x16WriteVgprVALUMemExpReadWaitStates = 11;
3843 const int SMFMA32x32WriteVgprVALUMemExpReadWaitStates = 19;
3844 const int DMFMA4x4WriteVgprMemExpReadWaitStates = 9;
3845 const int DMFMA16x16WriteVgprMemExpReadWaitStates = 18;
3846 const int DMFMA4x4WriteVgprVALUReadWaitStates = 6;
3847 const int DMFMA16x16WriteVgprVALUReadWaitStates = 11;
3848 const int GFX950_DMFMA16x16WriteVgprVALUReadWaitStates = 19;
3849 const int DotWriteSameDotReadSrcAB = 3;
3850 const int DotWriteDifferentVALURead = 3;
3851 const int DMFMABetweenVALUWriteVMEMRead = 2;
3852 const int MaxWaitStates =
3853 GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(NumPasses: 16,
3854 IsGFX950: ST.hasGFX950Insts());
3855
3856 for (const MachineOperand &Use : MI->explicit_uses()) {
3857 if (!Use.isReg())
3858 continue;
3859 Reg = Use.getReg();
3860
3861 DOT = nullptr;
3862 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsHazardDef: IsDotWriteFn,
3863 Limit: MaxWaitStates);
3864 if (DOT) {
3865 int NeedWaitStates = 0;
3866 if (DOT->getOpcode() == MI->getOpcode()) {
3867 if (&Use - &MI->getOperand(i: 0) != SrcCIdx)
3868 NeedWaitStates = DotWriteSameDotReadSrcAB;
3869 } else {
3870 NeedWaitStates = DotWriteDifferentVALURead;
3871 }
3872
3873 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
3874 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3875 }
3876
3877 // Workaround for HW data hazard bug observed only in GFX90A. When there
3878 // is a DGEMM instruction in-between a VALU and a VMEM instruction it
3879 // causes the SQ to incorrectly not insert two wait states between the two
3880 // instructions needed to avoid data hazard.
3881 if (IsMem && ST.hasGFX90AInsts() && !ST.hasGFX940Insts()) {
3882 DGEMMAfterVALUWrite = false;
3883 if (TRI.isVectorRegister(MRI, Reg)) {
3884 int WaitStatesNeededForUse =
3885 DMFMABetweenVALUWriteVMEMRead -
3886 getWaitStatesSinceDef(Reg, IsHazardDef: IsDGEMMHazard,
3887 Limit: DMFMABetweenVALUWriteVMEMRead);
3888
3889 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3890 }
3891 }
3892
3893 auto MFMAWindow = [&](const MachineInstr &Producer) {
3894 int NumPasses = TSchedModel.computeInstrLatency(MI: &Producer);
3895 int NeedWaitStates = MaxWaitStates;
3896
3897 if (SIInstrInfo::isDGEMM(Opcode: Producer.getOpcode())) {
3898 switch (NumPasses) {
3899 case 4:
3900 NeedWaitStates = IsMemOrExport
3901 ? DMFMA4x4WriteVgprMemExpReadWaitStates
3902 : DMFMA4x4WriteVgprVALUReadWaitStates;
3903 break;
3904 case 8:
3905 case 16:
3906 NeedWaitStates =
3907 IsMemOrExport
3908 ? DMFMA16x16WriteVgprMemExpReadWaitStates
3909 : (ST.hasGFX950Insts()
3910 ? GFX950_DMFMA16x16WriteVgprVALUReadWaitStates
3911 : DMFMA16x16WriteVgprVALUReadWaitStates);
3912 break;
3913 default:
3914 llvm_unreachable("unexpected dgemm");
3915 }
3916 } else if (ST.hasGFX940Insts()) {
3917 NeedWaitStates =
3918 TII.isXDL(MI: Producer)
3919 ? GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(
3920 NumPasses, IsGFX950: ST.hasGFX950Insts())
3921 : GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates(
3922 NumPasses);
3923 } else {
3924 switch (NumPasses) {
3925 case 2:
3926 NeedWaitStates = SMFMA4x4WriteVgprVALUMemExpReadWaitStates;
3927 break;
3928 case 8:
3929 NeedWaitStates = SMFMA16x16WriteVgprVALUMemExpReadWaitStates;
3930 break;
3931 case 16:
3932 NeedWaitStates = SMFMA32x32WriteVgprVALUMemExpReadWaitStates;
3933 break;
3934 default:
3935 llvm_unreachable("unexpected number of passes for mfma");
3936 }
3937 }
3938
3939 assert(NeedWaitStates <= MaxWaitStates &&
3940 "hazard requirement exceeds the scan window");
3941 return NeedWaitStates;
3942 };
3943
3944 WaitStatesNeeded = std::max(
3945 a: WaitStatesNeeded,
3946 b: getMaxWindowDeficit(MaxWindow: MaxWaitStates,
3947 WindowFor: [&](const MachineInstr &P) -> std::optional<int> {
3948 if (!IsMFMAWriteFn(P))
3949 return std::nullopt;
3950 return MFMAWindow(P);
3951 }));
3952
3953 if (WaitStatesNeeded >= MaxWaitStates)
3954 break;
3955 }
3956 }
3957
3958 unsigned Opc = MI->getOpcode();
3959 const int DMFMAToFMA64WaitStates = 2;
3960 if ((Opc == AMDGPU::V_FMA_F64_e64 ||
3961 Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64 ||
3962 Opc == AMDGPU::V_FMAC_F64_dpp) &&
3963 WaitStatesNeeded < DMFMAToFMA64WaitStates) {
3964 int WaitStatesNeededForUse = DMFMAToFMA64WaitStates -
3965 getWaitStatesSince(IsHazard: IsDGEMMFn, Limit: DMFMAToFMA64WaitStates);
3966 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3967 }
3968
3969 if (!IsVALU && !IsMemOrExport)
3970 return WaitStatesNeeded;
3971
3972 for (const MachineOperand &Def : MI->defs()) {
3973 const int SMFMA4x4WriteVgprVALUWawWaitStates = 5;
3974 const int SMFMA16x16WriteVgprVALUWawWaitStates = 11;
3975 const int SMFMA32x32WriteVgprVALUWawWaitStates = 19;
3976 const int SMFMA4x4ReadVgprVALUWarWaitStates = 1;
3977 const int GFX940_XDL4PassReadVgprVALUWarWaitStates = 3;
3978 const int SMFMA16x16ReadVgprVALUWarWaitStates = 7;
3979 const int SMFMA32x32ReadVgprVALUWarWaitStates = 15;
3980 const int DMFMA4x4WriteVgprVALUWriteWaitStates = 6;
3981 const int DMFMA16x16WriteVgprVALUWriteWaitStates = 11;
3982 const int DotWriteDifferentVALUWrite = 3;
3983 const int MaxWaitStates =
3984 GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(NumPasses: 16, IsGFX950: ST.hasGFX950Insts());
3985 const int MaxWarWaitStates = 15;
3986
3987 Reg = Def.getReg();
3988
3989 DOT = nullptr;
3990 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsHazardDef: IsDotWriteFn,
3991 Limit: MaxWaitStates);
3992 if (DOT && DOT->getOpcode() != MI->getOpcode())
3993 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: DotWriteDifferentVALUWrite -
3994 WaitStatesSinceDef);
3995
3996 auto MFMAWawWindow = [&](const MachineInstr &Producer) {
3997 int NeedWaitStates = MaxWaitStates;
3998 int NumPasses = TSchedModel.computeInstrLatency(MI: &Producer);
3999
4000 if (SIInstrInfo::isDGEMM(Opcode: Producer.getOpcode())) {
4001 switch (NumPasses) {
4002 case 4:
4003 NeedWaitStates = DMFMA4x4WriteVgprVALUWriteWaitStates;
4004 break;
4005 case 8:
4006 case 16:
4007 NeedWaitStates = DMFMA16x16WriteVgprVALUWriteWaitStates;
4008 break;
4009 default:
4010 llvm_unreachable("unexpected number of cycles for dgemm");
4011 }
4012 } else if (ST.hasGFX940Insts()) {
4013 NeedWaitStates =
4014 TII.isXDL(MI: Producer)
4015 ? GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(
4016 NumPasses, IsGFX950: ST.hasGFX950Insts())
4017 : GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(NumPasses);
4018 } else {
4019 switch (NumPasses) {
4020 case 2:
4021 NeedWaitStates = SMFMA4x4WriteVgprVALUWawWaitStates;
4022 break;
4023 case 8:
4024 NeedWaitStates = SMFMA16x16WriteVgprVALUWawWaitStates;
4025 break;
4026 case 16:
4027 NeedWaitStates = SMFMA32x32WriteVgprVALUWawWaitStates;
4028 break;
4029 default:
4030 llvm_unreachable("Unexpected number of passes for mfma");
4031 }
4032 }
4033
4034 assert(NeedWaitStates <= MaxWaitStates &&
4035 "hazard requirement exceeds the scan window");
4036 return NeedWaitStates;
4037 };
4038
4039 WaitStatesNeeded = std::max(
4040 a: WaitStatesNeeded,
4041 b: getMaxWindowDeficit(MaxWindow: MaxWaitStates,
4042 WindowFor: [&](const MachineInstr &P) -> std::optional<int> {
4043 if (!IsMFMAWriteFn(P))
4044 return std::nullopt;
4045 return MFMAWawWindow(P);
4046 }));
4047
4048 if (WaitStatesNeeded >= MaxWaitStates)
4049 break;
4050
4051 // Wait states this write needs after an MFMA that read Reg as srcC.
4052 auto SMFMAReadAsCWindow =
4053 [&](const MachineInstr &Reader) -> std::optional<int> {
4054 if (!SIInstrInfo::isMFMA(MI: Reader) ||
4055 SIInstrInfo::isDGEMM(Opcode: Reader.getOpcode()) ||
4056 !Reader.readsRegister(Reg, TRI: &TRI))
4057 return std::nullopt;
4058
4059 if (ST.hasGFX940Insts() && !TII.isXDL(MI: Reader))
4060 return std::nullopt;
4061
4062 const MachineOperand *SrcC =
4063 TII.getNamedOperand(MI: Reader, OperandName: AMDGPU::OpName::src2);
4064 assert(SrcC && "an MFMA must have a src2 operand");
4065 if (!SrcC->isReg() || !TRI.regsOverlap(RegA: SrcC->getReg(), RegB: Reg))
4066 return std::nullopt;
4067
4068 switch (TSchedModel.computeInstrLatency(MI: &Reader)) {
4069 case 2:
4070 return SMFMA4x4ReadVgprVALUWarWaitStates;
4071 case 4:
4072 assert(ST.hasGFX940Insts() && "4-pass XDL is gfx940 or later");
4073 return GFX940_XDL4PassReadVgprVALUWarWaitStates;
4074 case 8:
4075 return SMFMA16x16ReadVgprVALUWarWaitStates;
4076 case 16:
4077 [[fallthrough]];
4078 default:
4079 return SMFMA32x32ReadVgprVALUWarWaitStates;
4080 }
4081 };
4082
4083 WaitStatesNeeded =
4084 std::max(a: WaitStatesNeeded,
4085 b: getMaxWindowDeficit(MaxWindow: MaxWarWaitStates, WindowFor: SMFMAReadAsCWindow));
4086 }
4087
4088 return WaitStatesNeeded;
4089}
4090
4091bool GCNHazardRecognizer::ShouldPreferAnother(SUnit *SU) const {
4092 if (!SU->isInstr())
4093 return false;
4094
4095 const MachineInstr *MAI = nullptr;
4096
4097 auto IsMFMAFn = [&MAI](const MachineInstr &MI) {
4098 MAI = nullptr;
4099 if (SIInstrInfo::isMFMA(MI))
4100 MAI = &MI;
4101 return MAI != nullptr;
4102 };
4103
4104 MachineInstr *MI = SU->getInstr();
4105 if (IsMFMAFn(*MI)) {
4106 int W = getWaitStatesSince(IsHazard: IsMFMAFn, Limit: 16);
4107 if (MAI)
4108 return W < (int)TSchedModel.computeInstrLatency(MI: MAI);
4109 }
4110
4111 return false;
4112}
4113
4114// Adjust global offsets for instructions bundled with S_GETPC_B64 after
4115// insertion of a new instruction.
4116static void updateGetPCBundle(MachineInstr *NewMI) {
4117 if (!NewMI->isBundled())
4118 return;
4119
4120 // Find start of bundle.
4121 auto I = NewMI->getIterator();
4122 while (I->isBundledWithPred())
4123 I--;
4124 if (I->isBundle())
4125 I++;
4126
4127 // Bail if this is not an S_GETPC bundle.
4128 if (I->getOpcode() != AMDGPU::S_GETPC_B64)
4129 return;
4130
4131 // Update offsets of any references in the bundle.
4132 const unsigned NewBytes = 4;
4133 assert(NewMI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
4134 "Unexpected instruction insertion in bundle");
4135 auto NextMI = std::next(x: NewMI->getIterator());
4136 auto End = NewMI->getParent()->end();
4137 while (NextMI != End && NextMI->isBundledWithPred()) {
4138 for (auto &Operand : NextMI->operands()) {
4139 if (Operand.isGlobal())
4140 Operand.setOffset(Operand.getOffset() + NewBytes);
4141 }
4142 NextMI++;
4143 }
4144}
4145
4146bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) {
4147 if (!ST.hasVALUMaskWriteHazard())
4148 return false;
4149 assert(!ST.hasExtendedWaitCounts());
4150
4151 if (!ST.isWave64())
4152 return false;
4153
4154 const bool IsSALU = SIInstrInfo::isSALU(MI: *MI);
4155 const bool IsVALU = SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true);
4156 if (!IsSALU && !IsVALU)
4157 return false;
4158
4159 // The hazard sequence is three instructions:
4160 // 1. VALU reads SGPR as mask
4161 // 2. VALU/SALU writes SGPR
4162 // 3. VALU/SALU reads SGPR
4163 // The hazard can expire if the distance between 2 and 3 is sufficient,
4164 // or (2) is VALU and (3) is SALU.
4165 // In practice this happens <10% of the time, hence always assume the hazard
4166 // exists if (1) and (2) are present to avoid searching all SGPR reads.
4167
4168 const SIRegisterInfo *TRI = ST.getRegisterInfo();
4169 const MachineRegisterInfo &MRI = MF.getRegInfo();
4170
4171 auto IgnoreableSGPR = [](const Register Reg) {
4172 switch (Reg) {
4173 case AMDGPU::EXEC:
4174 case AMDGPU::EXEC_LO:
4175 case AMDGPU::EXEC_HI:
4176 case AMDGPU::M0:
4177 case AMDGPU::SGPR_NULL:
4178 case AMDGPU::SGPR_NULL64:
4179 case AMDGPU::SCC:
4180 return true;
4181 default:
4182 return false;
4183 }
4184 };
4185 auto IsVCC = [](const Register Reg) {
4186 return Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::VCC_HI;
4187 };
4188
4189 struct StateType {
4190 SmallSet<Register, 2> HazardSGPRs;
4191
4192 static unsigned getHashValue(const StateType &State) {
4193 return static_cast<unsigned>(hash_combine_range(R: State.HazardSGPRs));
4194 }
4195 static bool isEqual(const StateType &LHS, const StateType &RHS) {
4196 return LHS.HazardSGPRs == RHS.HazardSGPRs;
4197 }
4198 };
4199
4200 SmallVector<const MachineInstr *> WaitInstrs;
4201 StateType InitialState;
4202
4203 // Look for SGPR write.
4204 MachineOperand *HazardDef = nullptr;
4205 for (MachineOperand &Op : MI->all_defs()) {
4206 Register Reg = Op.getReg();
4207 if (IgnoreableSGPR(Reg))
4208 continue;
4209 if (!IsVCC(Reg)) {
4210 if (Op.isImplicit())
4211 continue;
4212 if (!TRI->isSGPRReg(MRI, Reg))
4213 continue;
4214 }
4215
4216 HazardDef = &Op;
4217 break;
4218 }
4219
4220 if (!HazardDef)
4221 return false;
4222
4223 // Setup to track writes to individual SGPRs
4224 const Register HazardReg = HazardDef->getReg();
4225 if (AMDGPU::SReg_32RegClass.contains(Reg: HazardReg)) {
4226 InitialState.HazardSGPRs.insert(V: HazardReg);
4227 } else {
4228 assert(AMDGPU::SReg_64RegClass.contains(HazardReg));
4229 InitialState.HazardSGPRs.insert(V: TRI->getSubReg(Reg: HazardReg, Idx: AMDGPU::sub0));
4230 InitialState.HazardSGPRs.insert(V: TRI->getSubReg(Reg: HazardReg, Idx: AMDGPU::sub1));
4231 }
4232
4233 auto IsHazardFn = [&](StateType &State, const MachineInstr &I) {
4234 if (State.HazardSGPRs.empty())
4235 return HazardExpired;
4236
4237 switch (I.getOpcode()) {
4238 case AMDGPU::V_ADDC_U32_e32:
4239 case AMDGPU::V_ADDC_U32_dpp:
4240 case AMDGPU::V_CNDMASK_B16_t16_e32:
4241 case AMDGPU::V_CNDMASK_B16_fake16_e32:
4242 case AMDGPU::V_CNDMASK_B16_t16_dpp:
4243 case AMDGPU::V_CNDMASK_B16_fake16_dpp:
4244 case AMDGPU::V_CNDMASK_B32_e32:
4245 case AMDGPU::V_CNDMASK_B32_dpp:
4246 case AMDGPU::V_DIV_FMAS_F32_e64:
4247 case AMDGPU::V_DIV_FMAS_F64_e64:
4248 case AMDGPU::V_SUBB_U32_e32:
4249 case AMDGPU::V_SUBB_U32_dpp:
4250 case AMDGPU::V_SUBBREV_U32_e32:
4251 case AMDGPU::V_SUBBREV_U32_dpp: {
4252 // These implicitly read VCC as mask source.
4253 return IsVCC(HazardReg) ? HazardFound : NoHazardFound;
4254 }
4255 case AMDGPU::V_ADDC_U32_e64:
4256 case AMDGPU::V_ADDC_U32_e64_dpp:
4257 case AMDGPU::V_CNDMASK_B16_t16_e64:
4258 case AMDGPU::V_CNDMASK_B16_fake16_e64:
4259 case AMDGPU::V_CNDMASK_B16_t16_e64_dpp:
4260 case AMDGPU::V_CNDMASK_B16_fake16_e64_dpp:
4261 case AMDGPU::V_CNDMASK_B32_e64:
4262 case AMDGPU::V_CNDMASK_B32_e64_dpp:
4263 case AMDGPU::V_SUBB_U32_e64:
4264 case AMDGPU::V_SUBB_U32_e64_dpp:
4265 case AMDGPU::V_SUBBREV_U32_e64:
4266 case AMDGPU::V_SUBBREV_U32_e64_dpp: {
4267 // Only check mask register overlaps.
4268 const MachineOperand *SSRCOp = TII.getNamedOperand(MI: I, OperandName: AMDGPU::OpName::src2);
4269 assert(SSRCOp);
4270 bool Result = TRI->regsOverlap(RegA: SSRCOp->getReg(), RegB: HazardReg);
4271 return Result ? HazardFound : NoHazardFound;
4272 }
4273 default:
4274 return NoHazardFound;
4275 }
4276 };
4277
4278 auto UpdateStateFn = [&](StateType &State, const MachineInstr &I) {
4279 // Update tracking of SGPR writes.
4280 for (auto &Op : I.all_defs()) {
4281 Register Reg = Op.getReg();
4282 if (IgnoreableSGPR(Reg))
4283 continue;
4284 if (!IsVCC(Reg)) {
4285 if (Op.isImplicit())
4286 continue;
4287 if (!TRI->isSGPRReg(MRI, Reg))
4288 continue;
4289 }
4290
4291 // Stop tracking any SGPRs with writes on the basis that they will
4292 // already have an appropriate wait inserted afterwards.
4293 SmallVector<Register, 2> Found;
4294 for (Register SGPR : State.HazardSGPRs) {
4295 if (Reg == SGPR || TRI->regsOverlap(RegA: Reg, RegB: SGPR))
4296 Found.push_back(Elt: SGPR);
4297 }
4298 for (Register SGPR : Found)
4299 State.HazardSGPRs.erase(V: SGPR);
4300 }
4301 };
4302
4303 // Check for hazard
4304 if (!hasHazard<StateType>(InitialState, IsHazard: IsHazardFn, UpdateState: UpdateStateFn,
4305 InitialMBB: MI->getParent(),
4306 InitialI: std::next(x: MI->getReverseIterator())))
4307 return false;
4308
4309 // Compute counter mask
4310 unsigned DepCtr =
4311 IsVALU ? (IsVCC(HazardReg) ? AMDGPU::DepCtr::encodeFieldVaVcc(VaVcc: 0, STI: ST)
4312 : AMDGPU::DepCtr::encodeFieldVaSdst(VaSdst: 0, STI: ST))
4313 : AMDGPU::DepCtr::encodeFieldSaSdst(SaSdst: 0, STI: ST);
4314
4315 // Add s_waitcnt_depctr after SGPR write.
4316 auto NextMI = std::next(x: MI->getIterator());
4317 auto NewMI = BuildMI(BB&: *MI->getParent(), I: NextMI, MIMD: MI->getDebugLoc(),
4318 MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4319 .addImm(Val: DepCtr);
4320
4321 // SALU write may be s_getpc in a bundle.
4322 updateGetPCBundle(NewMI);
4323
4324 return true;
4325}
4326
4327static bool ensureEntrySetPrio(MachineFunction *MF, int Priority,
4328 const SIInstrInfo &TII) {
4329 MachineBasicBlock &EntryMBB = MF->front();
4330 if (EntryMBB.begin() != EntryMBB.end()) {
4331 auto &EntryMI = *EntryMBB.begin();
4332 if (EntryMI.getOpcode() == AMDGPU::S_SETPRIO &&
4333 EntryMI.getOperand(i: 0).getImm() >= Priority)
4334 return false;
4335 }
4336
4337 BuildMI(BB&: EntryMBB, I: EntryMBB.begin(), MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_SETPRIO))
4338 .addImm(Val: Priority);
4339 return true;
4340}
4341
4342bool GCNHazardRecognizer::fixRequiredExportPriority(MachineInstr *MI) {
4343 if (!ST.hasRequiredExportPriority())
4344 return false;
4345
4346 // Assume the following shader types will never have exports,
4347 // and avoid adding or adjusting S_SETPRIO.
4348 MachineBasicBlock *MBB = MI->getParent();
4349 MachineFunction *MF = MBB->getParent();
4350 auto CC = MF->getFunction().getCallingConv();
4351 switch (CC) {
4352 case CallingConv::AMDGPU_CS:
4353 case CallingConv::AMDGPU_CS_Chain:
4354 case CallingConv::AMDGPU_CS_ChainPreserve:
4355 case CallingConv::AMDGPU_KERNEL:
4356 return false;
4357 default:
4358 break;
4359 }
4360
4361 const int MaxPriority = 3;
4362 const int NormalPriority = 2;
4363 const int PostExportPriority = 0;
4364
4365 auto It = MI->getIterator();
4366 switch (MI->getOpcode()) {
4367 case AMDGPU::S_ENDPGM:
4368 case AMDGPU::S_ENDPGM_SAVED:
4369 case AMDGPU::S_ENDPGM_ORDERED_PS_DONE:
4370 case AMDGPU::SI_RETURN_TO_EPILOG:
4371 // Ensure shader with calls raises priority at entry.
4372 // This ensures correct priority if exports exist in callee.
4373 if (MF->getFrameInfo().hasCalls())
4374 return ensureEntrySetPrio(MF, Priority: NormalPriority, TII);
4375 return false;
4376 case AMDGPU::S_SETPRIO: {
4377 // Raise minimum priority unless in workaround.
4378 auto &PrioOp = MI->getOperand(i: 0);
4379 int Prio = static_cast<int>(PrioOp.getImm());
4380 bool InWA = (Prio == PostExportPriority) &&
4381 (It != MBB->begin() && TII.isEXP(MI: *std::prev(x: It)));
4382 if (InWA || Prio >= NormalPriority)
4383 return false;
4384 PrioOp.setImm(std::min(a: Prio + NormalPriority, b: MaxPriority));
4385 return true;
4386 }
4387 default:
4388 if (!TII.isEXP(MI: *MI))
4389 return false;
4390 break;
4391 }
4392
4393 // Check entry priority at each export (as there will only be a few).
4394 // Note: amdgpu_gfx can only be a callee, so defer to caller setprio.
4395 bool Changed = false;
4396 if (CC != CallingConv::AMDGPU_Gfx && CC != CallingConv::AMDGPU_Gfx_WholeWave)
4397 Changed = ensureEntrySetPrio(MF, Priority: NormalPriority, TII);
4398
4399 auto NextMI = std::next(x: It);
4400 bool EndOfShader = false;
4401 if (NextMI != MBB->end()) {
4402 // Only need WA at end of sequence of exports.
4403 if (TII.isEXP(MI: *NextMI))
4404 return Changed;
4405 // Assume appropriate S_SETPRIO after export means WA already applied.
4406 if (NextMI->getOpcode() == AMDGPU::S_SETPRIO &&
4407 NextMI->getOperand(i: 0).getImm() == PostExportPriority)
4408 return Changed;
4409 EndOfShader = NextMI->getOpcode() == AMDGPU::S_ENDPGM;
4410 }
4411
4412 const DebugLoc &DL = MI->getDebugLoc();
4413
4414 // Lower priority.
4415 BuildMI(BB&: *MBB, I: NextMI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_SETPRIO))
4416 .addImm(Val: PostExportPriority);
4417
4418 if (!EndOfShader) {
4419 // Wait for exports to complete.
4420 BuildMI(BB&: *MBB, I: NextMI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_EXPCNT))
4421 .addReg(RegNo: AMDGPU::SGPR_NULL)
4422 .addImm(Val: 0);
4423 }
4424
4425 BuildMI(BB&: *MBB, I: NextMI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_NOP)).addImm(Val: 0);
4426 BuildMI(BB&: *MBB, I: NextMI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_NOP)).addImm(Val: 0);
4427
4428 if (!EndOfShader) {
4429 // Return to normal (higher) priority.
4430 BuildMI(BB&: *MBB, I: NextMI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_SETPRIO))
4431 .addImm(Val: NormalPriority);
4432 }
4433
4434 return true;
4435}
4436
4437// Advance past meta instructions (debug values, labels, CFI, KILL, etc.) to the
4438// next instruction that actually issues. Unlike skipDebugInstructionsForward /
4439// next_nodbg, this skips the full isMetaInstruction() set.
4440static MachineBasicBlock::iterator
4441skipMetaInstructionsForward(MachineBasicBlock::iterator I,
4442 MachineBasicBlock::iterator End) {
4443 while (I != End && I->isMetaInstruction())
4444 ++I;
4445 return I;
4446}
4447
4448bool GCNHazardRecognizer::fixVPermPk16Hazard(MachineInstr *MI) {
4449 // Requirement #1 of 2:
4450 // The cross-wave entry-block mitigation is delegated to the mandatory
4451 // unclaused-VMEM entry prologue (GLOBAL_PREFETCH_B8 + V_NOP).
4452 assert(ST.hasRequiresInitialUnclausedVmem() &&
4453 "V_PERM_PK16-hazard subtarget must provide the unclaused-VMEM entry "
4454 "prologue to satisfy the cross-wave entry mitigation");
4455
4456 if (!SIInstrInfo::isVPermPk16(Opcode: MI->getOpcode()))
4457 return false;
4458
4459 MachineBasicBlock *MBB = MI->getParent();
4460
4461 // Requirement #2 of 2:
4462 // V_PERM_PK16 must be immediately followed by a safe instruction.
4463 MachineBasicBlock::iterator NextI =
4464 skipMetaInstructionsForward(I: std::next(x: MI->getIterator()), End: MBB->end());
4465 if (NextI != MBB->end() && TII.isVPermPk16SafeInstr(MI: *NextI))
4466 return false;
4467
4468 // EXEC is guaranteed non-zero here: V_PERM_PK16 reports unwanted effects
4469 // when EXEC is empty, so s_cbranch_execz over this region is retained.
4470 // A plain V_NOP is therefore a real VALU nop and clears the hazard.
4471 BuildMI(BB&: *MBB, I: NextI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_NOP_e32));
4472 return true;
4473}
4474
4475bool GCNHazardRecognizer::fixGetRegWaitIdle(MachineInstr *MI) {
4476 if (!isSGetReg(Opcode: MI->getOpcode()))
4477 return false;
4478
4479 const SIInstrInfo *TII = ST.getInstrInfo();
4480 switch (getHWReg(TII, RegInstr: *MI)) {
4481 default:
4482 return false;
4483 case AMDGPU::Hwreg::ID_STATUS:
4484 case AMDGPU::Hwreg::ID_STATE_PRIV:
4485 case AMDGPU::Hwreg::ID_EXCP_FLAG_PRIV:
4486 case AMDGPU::Hwreg::ID_EXCP_FLAG_USER:
4487 break;
4488 }
4489
4490 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
4491 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4492 .addImm(Val: 0);
4493 return true;
4494}
4495
4496bool GCNHazardRecognizer::fixDsAtomicAsyncBarrierArriveB64(MachineInstr *MI) {
4497 if (MI->getOpcode() != AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
4498 return false;
4499
4500 const SIInstrInfo *TII = ST.getInstrInfo();
4501 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
4502 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4503 .addImm(Val: AMDGPU::DepCtr::encodeFieldVmVsrc(VmVsrc: 0, STI: ST));
4504 BuildMI(BB&: *MI->getParent(), I: std::next(x: MI->getIterator()), MIMD: MI->getDebugLoc(),
4505 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4506 .addImm(Val: AMDGPU::DepCtr::encodeFieldVmVsrc(VmVsrc: 0, STI: ST));
4507
4508 return true;
4509}
4510
4511bool GCNHazardRecognizer::fixScratchBaseForwardingHazard(MachineInstr *MI) {
4512 // No reason to check this in pre-RA scheduling, SGPRs have to be allocated
4513 // for hazard to trigger.
4514 if (!isHazardRecognizerMode())
4515 return false;
4516
4517 const SIRegisterInfo *TRI = ST.getRegisterInfo();
4518 const SIInstrInfo *TII = ST.getInstrInfo();
4519 // Hazard expires after 10 SGPR writes by SALU or 8 SGPR writes by VALU.
4520 const int FlatScrBaseWaitStates = 10;
4521
4522 bool ReadsFlatScrLo =
4523 MI->readsRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_LO, TRI);
4524 bool ReadsFlatScrHi =
4525 MI->readsRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_HI, TRI);
4526 if (isSGetReg(Opcode: MI->getOpcode())) {
4527 switch (getHWReg(TII, RegInstr: *MI)) {
4528 default:
4529 break;
4530 case AMDGPU::Hwreg::ID_FLAT_SCR_LO:
4531 ReadsFlatScrLo = true;
4532 break;
4533 case AMDGPU::Hwreg::ID_FLAT_SCR_HI:
4534 ReadsFlatScrHi = true;
4535 break;
4536 }
4537 }
4538
4539 const MachineRegisterInfo &MRI = MF.getRegInfo();
4540
4541 auto IsRegDefHazard = [&](Register Reg) -> bool {
4542 DenseSet<const MachineBasicBlock *> Visited;
4543 auto IsHazardFn = [TRI, Reg](const MachineInstr &MI) {
4544 return MI.modifiesRegister(Reg, TRI);
4545 };
4546
4547 // This literally abuses the idea of waitstates. Instead of waitstates it
4548 // returns 1 for SGPR written and 0 otherwise.
4549 auto IsSGPRDef = [TII, TRI, &MRI](const MachineInstr &MI) -> unsigned {
4550 if (!TII->isSALU(MI) && !TII->isVALU(MI, /*AllowLDSDMA=*/true))
4551 return 0;
4552 for (const MachineOperand &MO : MI.all_defs()) {
4553 if (TRI->isSGPRReg(MRI, Reg: MO.getReg()))
4554 return 1;
4555 }
4556 return 0;
4557 };
4558
4559 auto IsExpiredFn = [=](const MachineInstr &MI, int SgprWrites) {
4560 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR) {
4561 unsigned Wait = static_cast<unsigned>(MI.getOperand(i: 0).getImm());
4562 if (AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: Wait) == 0 &&
4563 AMDGPU::DepCtr::decodeFieldVaSdst(Encoded: Wait) == 0)
4564 return true;
4565 }
4566 return SgprWrites >= FlatScrBaseWaitStates;
4567 };
4568
4569 return ::getWaitStatesSince(
4570 IsHazard: IsHazardFn, MBB: MI->getParent(), I: std::next(x: MI->getReverseIterator()),
4571 WaitStates: 0, IsExpired: IsExpiredFn, Visited, GetNumWaitStates: IsSGPRDef) < FlatScrBaseWaitStates;
4572 };
4573
4574 if ((!ReadsFlatScrLo || MRI.isConstantPhysReg(PhysReg: AMDGPU::SGPR102) ||
4575 !IsRegDefHazard(AMDGPU::SGPR102)) &&
4576 (!ReadsFlatScrHi || MRI.isConstantPhysReg(PhysReg: AMDGPU::SGPR103) ||
4577 !IsRegDefHazard(AMDGPU::SGPR103)))
4578 return false;
4579
4580 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
4581 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4582 .addImm(Val: AMDGPU::DepCtr::encodeFieldVaSdst(
4583 Encoded: AMDGPU::DepCtr::encodeFieldSaSdst(SaSdst: 0, STI: ST), VaSdst: 0));
4584 return true;
4585}
4586
4587bool GCNHazardRecognizer::fixSetRegMode(MachineInstr *MI) {
4588 if (!isSSetReg(Opcode: MI->getOpcode()) ||
4589 MI->getOperand(i: 1).getImm() != AMDGPU::Hwreg::ID_MODE)
4590 return false;
4591
4592 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_NOP_e32));
4593 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_NOP_e32));
4594 return true;
4595}
4596
4597bool GCNHazardRecognizer::fixTDM(MachineInstr *MI) {
4598 auto IsTDM = [&](const MachineInstr &MI) -> bool {
4599 return SIInstrInfo::usesTENSOR_CNT(MI) &&
4600 MI.getOpcode() != AMDGPU::S_WAIT_TENSORCNT;
4601 };
4602
4603 if (!IsTDM(*MI))
4604 return false;
4605
4606 auto IsExpiredFn = [](const MachineInstr &MI, int) {
4607 if (MI.getOpcode() != AMDGPU::S_WAIT_TENSORCNT)
4608 return false;
4609 return MI.getOperand(i: 0).getImm() <= 10;
4610 };
4611
4612 if (::getWaitStatesSince(IsHazard: IsTDM, MI, IsExpired: IsExpiredFn) ==
4613 std::numeric_limits<int>::max())
4614 return false;
4615
4616 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
4617 MCID: TII.get(Opcode: AMDGPU::S_WAIT_TENSORCNT))
4618 .addImm(Val: 10);
4619 return true;
4620}
4621