1//===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements hazard recognizers for scheduling on GCN processors.
10//
11//===----------------------------------------------------------------------===//
12
13#include "GCNHazardRecognizer.h"
14#include "AMDGPUTargetMachine.h"
15#include "AMDGPUWaitcntUtils.h"
16#include "GCNSubtarget.h"
17#include "SIMachineFunctionInfo.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/CodeGen/MachineFrameInfo.h"
20#include "llvm/CodeGen/MachineFunction.h"
21#include "llvm/CodeGen/MachineInstrBuilder.h"
22#include "llvm/CodeGen/ScheduleDAG.h"
23#include "llvm/Support/Debug.h"
24#include "llvm/TargetParser/AMDGPUTargetParser.h"
25
26using namespace llvm;
27
28#define DEBUG_TYPE "gcn-hazard-recognizer"
29// Opt-in debug type for the per-candidate co-execution slot traces, which are
30// far too noisy for the normal debug output. Pass both types to get everything.
31#define DEBUG_TYPE_VERBOSE "gcn-hazard-recognizer-verbose"
32
33STATISTIC(NumWMMANopsHoisted,
34 "Number of WMMA hazard V_NOPs hoisted from loops");
35STATISTIC(NumWMMAHoistingBailed,
36 "Number of WMMA hazards where V_NOP hoisting was not possible");
37
38namespace {
39
40struct MFMAPaddingRatioParser : public cl::parser<unsigned> {
41 MFMAPaddingRatioParser(cl::Option &O) : cl::parser<unsigned>(O) {}
42
43 bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) {
44 if (Arg.getAsInteger(Radix: 0, Result&: Value))
45 return O.error(Message: "'" + Arg + "' value invalid for uint argument!");
46
47 if (Value > 100)
48 return O.error(Message: "'" + Arg + "' value must be in the range [0, 100]!");
49
50 return false;
51 }
52};
53
54} // end anonymous namespace
55
56static cl::opt<unsigned, false, MFMAPaddingRatioParser>
57 MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(Val: 0), cl::Hidden,
58 cl::desc("Fill a percentage of the latency between "
59 "neighboring MFMA with s_nops."));
60
61// This is intended for debugging purposes only.
62static cl::opt<unsigned>
63 NopPadding("amdgpu-snop-padding", cl::init(Val: 0), cl::Hidden,
64 cl::desc("Insert a s_nop x before every instruction"));
65
66static cl::opt<bool> EnableWMMAVnopHoisting(
67 "amdgpu-wmma-vnop-hoisting", cl::init(Val: true), cl::Hidden,
68 cl::desc("Hoist WMMA hazard V_NOPs from loops to preheaders"));
69
70//===----------------------------------------------------------------------===//
71// Hazard Recognizer Implementation
72//===----------------------------------------------------------------------===//
73
74static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF,
75 const GCNSubtarget &ST);
76
77GCNHazardRecognizer::GCNHazardRecognizer(
78 const MachineFunction &MF, GCNHazardRecognizer::OperatingMode Mode,
79 MachineLoopInfo *MLI)
80 : Mode(Mode), CurrCycleInstr(nullptr), MF(MF),
81 ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()),
82 TRI(TII.getRegisterInfo()), TSchedModel(TII.getSchedModel()), MLI(MLI),
83 ClauseUses(TRI.getNumRegUnits()), ClauseDefs(TRI.getNumRegUnits()) {
84 MaxLookAhead = MF.getRegInfo().isPhysRegUsed(PhysReg: AMDGPU::AGPR0) ? 19 : 5;
85 RunLdsBranchVmemWARHazardFixup = shouldRunLdsBranchVmemWARHazardFixup(MF, ST);
86 LLVM_DEBUG({
87 if (isPreRA())
88 dbgs() << " PreRA hazard recognizer: " << MF.getName() << "\n";
89 });
90}
91
92GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF,
93 MachineLoopInfo *MLI)
94 : GCNHazardRecognizer(MF, OperatingMode::PostRA, MLI) {}
95
96GCNHazardRecognizer::~GCNHazardRecognizer() {
97 // Dump any active co-execution window that did not complete naturally
98 // (e.g. region ended before the window expired).
99 LLVM_DEBUG({
100 if (CurrentCoExecStage.has_value()) {
101 unsigned Stage = *CurrentCoExecStage;
102 if (Stage < AMDGPU::MaxCoExecStages)
103 CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
104 dbgs() << " CoExec window ended at stage " << Stage << ":\n";
105 dumpCoExecWindow();
106 }
107 });
108}
109
110void GCNHazardRecognizer::Reset() {
111 EmittedInstrs.clear();
112 EmittedVALUInstrs.clear();
113 HasPendingWMMACoexecHazard = false;
114 if (isSchedulerMode())
115 schedulerReset();
116}
117
118void GCNHazardRecognizer::schedulerReset() {
119 LLVM_DEBUG({
120 if (CurrentCoExecStage.has_value() || CyclesUntilTRANS > 0 ||
121 CyclesUntilVALU > 0)
122 dbgs() << " Scheduler Reset: clearing co-exec window, TRANS="
123 << CyclesUntilTRANS << ", VALU=" << CyclesUntilVALU << "\n";
124 });
125 CurrentCoExecStage = std::nullopt;
126 CoExecWindowStartCycle = 0;
127 CyclesUntilTRANS = 0;
128 CyclesUntilVALU = 0;
129 ActiveCoExecInfo = AMDGPU::CoExecInfo();
130 CoExecWindowLog.fill(u: '.');
131}
132
133void GCNHazardRecognizer::dumpCoExecWindow() const {
134 unsigned W = ActiveCoExecInfo.TotalWindow;
135 if (W == 0)
136 return;
137
138 // Print the stage numbers row.
139 dbgs() << " Stages: ";
140 for (unsigned I = 0; I < W; ++I)
141 dbgs() << I % 10 << ' ';
142 dbgs() << '\n';
143
144 // Print the pattern row.
145 dbgs() << " Slots: ";
146 for (unsigned I = 0; I < W; ++I)
147 dbgs() << ActiveCoExecInfo.Pattern[I] << ' ';
148 dbgs() << '\n';
149
150 // Print the scheduled row.
151 dbgs() << " Scheduled: ";
152 for (unsigned I = 0; I < W; ++I)
153 dbgs() << CoExecWindowLog[I] << ' ';
154 dbgs() << '\n';
155}
156
157void GCNHazardRecognizer::schedulerAdvanceCycle() {
158 // Record what happened at the current stage of the co-exec window.
159 if (CurrentCoExecStage.has_value()) {
160 unsigned Stage = *CurrentCoExecStage;
161 if (Stage < AMDGPU::MaxCoExecStages) {
162 if (CurrCycleInstr)
163 CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
164 else
165 CoExecWindowLog[Stage] = '-';
166 }
167 }
168
169 LLVM_DEBUG({
170 bool HasState = CurrentCoExecStage.has_value() || CyclesUntilTRANS > 0 ||
171 CyclesUntilVALU > 0;
172 if (HasState) {
173 dbgs() << " Scheduler AdvanceCycle:";
174 if (CurrentCoExecStage.has_value()) {
175 unsigned Stage = *CurrentCoExecStage;
176 unsigned Next = Stage + 1;
177 if (Next >= ActiveCoExecInfo.TotalWindow)
178 dbgs() << " stage " << Stage << "->expired";
179 else
180 dbgs() << " stage " << Stage << "->" << Next;
181 }
182 if (CyclesUntilTRANS > 0)
183 dbgs() << " TRANS=" << CyclesUntilTRANS << "->"
184 << (CyclesUntilTRANS - 1);
185 if (CyclesUntilVALU > 0)
186 dbgs() << " VALU=" << CyclesUntilVALU << "->" << (CyclesUntilVALU - 1);
187 dbgs() << "\n";
188 }
189 });
190
191 // Decrement hazard counters.
192 if (CyclesUntilTRANS > 0)
193 --CyclesUntilTRANS;
194 if (CyclesUntilVALU > 0)
195 --CyclesUntilVALU;
196
197 // Advance WMMA co-execution window.
198 if (CurrentCoExecStage.has_value()) {
199 unsigned Stage = *CurrentCoExecStage + 1;
200 if (Stage >= ActiveCoExecInfo.TotalWindow) {
201 // Window expired.
202 LLVM_DEBUG({
203 dbgs() << " CoExec window complete:\n";
204 dumpCoExecWindow();
205 });
206 CurrentCoExecStage = std::nullopt;
207 } else {
208 CurrentCoExecStage = Stage;
209 }
210 }
211}
212
213bool GCNHazardRecognizer::hasCoExecWindowModel() const {
214 // The co-execution slot patterns returned by getCoExecInfo() are derived from
215 // gfx1250 timings, so the window model is restricted to gfx1250 for now.
216 // gfx1251 and gfx12.5-generic report the same co-execution hazard features
217 // but have different WMMA latencies, so they need their own slot patterns
218 // before they can be modeled here.
219 if (ST.hasWMMACoexecutionHazards() && ST.hasTransCoexecutionHazard() &&
220 AMDGPU::isGFX1250(STI: ST))
221 return true;
222
223 if (ST.hasGFX950Insts() &&
224 AMDGPU::getSchedStrategy(F: MF.getFunction()) == "coexec")
225 return true;
226
227 return false;
228}
229
230void GCNHazardRecognizer::updateWMMAWindowState(const MachineInstr &MI) {
231 if (!hasCoExecWindowModel())
232 return;
233
234 if (!SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI) &&
235 !SIInstrInfo::isMFMA(MI))
236 return;
237
238 // If a previous window was still active, dump it before starting a new one.
239 // Record the current stage (filled by this new WMMA) before dumping.
240 LLVM_DEBUG({
241 if (CurrentCoExecStage.has_value()) {
242 unsigned Stage = *CurrentCoExecStage;
243 if (Stage < AMDGPU::MaxCoExecStages)
244 CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
245 dbgs() << " CoExec window interrupted at stage " << Stage << ":\n";
246 dumpCoExecWindow();
247 }
248 });
249
250 // Start a new co-execution window.
251 ActiveCoExecInfo = AMDGPU::getCoExecInfo(MI, TII);
252 CurrentCoExecStage = 0;
253 CoExecWindowLog.fill(u: '.');
254
255 LLVM_DEBUG(dbgs() << " WMMA window started: " << ActiveCoExecInfo.Pattern
256 << " (window=" << ActiveCoExecInfo.TotalWindow << ")\n"
257 << " " << MI);
258}
259
260void GCNHazardRecognizer::updateTRANSState(const MachineInstr &MI) {
261 if (!hasCoExecWindowModel())
262 return;
263 if (!SIInstrInfo::isTRANS(MI))
264 return;
265
266 // Back-to-back TRANS instructions have a 1-cycle hazard.
267 // This is checked via checkTRANSHazard() and does not create a co-exec
268 // window. The TRANS shadow slot allows anything except TRANS and
269 // multi-cycle VALU.
270 // Set to 2: bumpCycle advances to the next pick's cycle (decrementing
271 // by 1 via AdvanceCycle) before the next instruction's hazard check, so
272 // the counter is observed at 1 there. That 1-cycle stall lets the
273 // strategy pick a non-TRANS, non-multi-cycle-VALU candidate to fill the
274 // shadow slot.
275 CyclesUntilTRANS = 2;
276 LLVM_DEBUG(dbgs() << " TRANS hazard set: CyclesUntilTRANS=2\n");
277}
278
279void GCNHazardRecognizer::updateMultiCycleVALUState(const MachineInstr &MI) {
280 if (!hasCoExecWindowModel())
281 return;
282 // Multi-cycle VALU (CVT, etc.) blocks subsequent VALU for repeat rate cycles.
283 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
284 return;
285
286 // Skip WMMA, MFMA, and TRANS - they have their own tracking.
287 if (SIInstrInfo::isWMMA(MI) || SIInstrInfo::isSWMMAC(MI) ||
288 SIInstrInfo::isMFMA(MI) || SIInstrInfo::isTRANS(MI))
289 return;
290
291 unsigned RepeatRate = TII.getRepeatRate(MI);
292 if (RepeatRate > 1) {
293 // bumpCycle's AdvanceCycle decrements once before the next pick's
294 // hazard check (same convention as CyclesUntilTRANS), so to expose
295 // RepeatRate-1 cycles of shadow we must seed with RepeatRate.
296 CyclesUntilVALU = RepeatRate;
297 LLVM_DEBUG(dbgs() << " Multi-cycle VALU: repeat=" << RepeatRate
298 << ", CyclesUntilVALU=" << CyclesUntilVALU << "\n");
299 }
300}
301
302AMDGPU::CoExecMaskT
303GCNHazardRecognizer::getCoExecMaskForMI(const MachineInstr &MI,
304 const SIInstrInfo &TII) {
305 return AMDGPU::getCoExecMask(F: AMDGPU::classifyFlavor(MI, SII: TII));
306}
307
308unsigned GCNHazardRecognizer::checkTRANSHazard(const MachineInstr &MI) const {
309 if (!CyclesUntilTRANS)
310 return 0;
311
312 // Only TRANS and multi-cycle VALU are blocked by the TRANS shadow.
313 if (SIInstrInfo::isTRANS(MI))
314 return CyclesUntilTRANS;
315
316 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
317 !SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI) &&
318 TII.getRepeatRate(MI) > 1)
319 return CyclesUntilTRANS;
320
321 return 0;
322}
323
324unsigned
325GCNHazardRecognizer::checkMultiCycleVALUHazard(const MachineInstr &MI) const {
326 if (!CyclesUntilVALU)
327 return 0;
328
329 // Multi-cycle VALU blocks anything on the VALU pipe - VALU, WMMA, SWMMAC,
330 // and TRANS - for RepeatRate-1 cycles. Only off-pipe instructions (MEM,
331 // SALU, control) can fill the shadow.
332 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
333 !SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI) &&
334 !SIInstrInfo::isTRANS(MI))
335 return 0;
336
337 return CyclesUntilVALU;
338}
339
340unsigned
341GCNHazardRecognizer::checkWMMACoexecSlot(const MachineInstr &MI) const {
342 // No hazard if not in a WMMA window.
343 if (!CurrentCoExecStage.has_value())
344 return 0;
345
346 unsigned Stage = *CurrentCoExecStage;
347 AMDGPU::CoExecMaskT InstMask = getCoExecMaskForMI(MI, TII);
348 unsigned StallCycles = ActiveCoExecInfo.getStallCycles(InstMask, Stage);
349
350 // No stall required if the instruction can co-execute at the current stage.
351 if (StallCycles == 0)
352 return 0;
353
354 // Stall for the required number of cycles until the next allowed stage.
355 unsigned NextStage = Stage + StallCycles;
356 if (NextStage < ActiveCoExecInfo.TotalWindow) {
357 DEBUG_WITH_TYPE(
358 DEBUG_TYPE_VERBOSE,
359 dbgs() << " CoExec stall: stage=" << Stage << "("
360 << AMDGPU::getStageTypeName(ActiveCoExecInfo.getType(Stage))
361 << ") mask=" << AMDGPU::getCoExecMaskName(InstMask)
362 << " -> stall " << StallCycles << " (next allowed=" << NextStage
363 << ")\n"
364 << " " << MI);
365 return StallCycles;
366 }
367
368 // No compatible slot in window - stall until window ends.
369 DEBUG_WITH_TYPE(
370 DEBUG_TYPE_VERBOSE,
371 dbgs() << " CoExec stall: stage=" << Stage << "("
372 << AMDGPU::getStageTypeName(ActiveCoExecInfo.getType(Stage))
373 << ") mask=" << AMDGPU::getCoExecMaskName(InstMask) << " -> stall "
374 << StallCycles << " (window ends)\n"
375 << " " << MI);
376 return StallCycles;
377}
378
379unsigned
380GCNHazardRecognizer::checkMultiShadowHazard(const MachineInstr &MI) const {
381 // This models a VALU caught in both a WMMA and a TRANS shadow.
382 if (!hasCoExecWindowModel())
383 return 0;
384
385 // No hazard if not in a WMMA window.
386 if (!CurrentCoExecStage.has_value())
387 return 0;
388
389 if (!CyclesUntilTRANS)
390 return 0;
391
392 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ||
393 SIInstrInfo::isLDSDMA(MI))
394 return 0;
395
396 // We have a VALU instruction that is under both a TRANS and WMMA shadow.
397 // We need to wait for at least one to clear.
398
399 unsigned LookAheadStage = *CurrentCoExecStage + CyclesUntilTRANS;
400 AMDGPU::CoExecMaskT InstMask = getCoExecMaskForMI(MI, TII);
401 return CyclesUntilTRANS +
402 ActiveCoExecInfo.getStallCycles(InstMask, Stage: LookAheadStage);
403}
404
405void GCNHazardRecognizer::schedulerEmitInstruction(MachineInstr *MI) {
406 LLVM_DEBUG({
407 bool InWindow = CurrentCoExecStage.has_value();
408 bool HasActiveState =
409 InWindow || CyclesUntilTRANS > 0 || CyclesUntilVALU > 0;
410 if (HasActiveState) {
411 if (InWindow) {
412 unsigned Stage = *CurrentCoExecStage;
413 dbgs() << " Stage " << Stage << "("
414 << AMDGPU::getStageTypeName(ActiveCoExecInfo.getType(Stage))
415 << ") Emit ["
416 << AMDGPU::getCoExecMaskName(getCoExecMaskForMI(*MI, TII))
417 << "]: " << *MI;
418 } else {
419 dbgs() << " Emit ["
420 << AMDGPU::getCoExecMaskName(getCoExecMaskForMI(*MI, TII))
421 << "]: " << *MI;
422 }
423 }
424 });
425 DEBUG_WITH_TYPE(DEBUG_TYPE_VERBOSE, {
426 bool HasActiveState = CurrentCoExecStage.has_value() ||
427 CyclesUntilTRANS > 0 || CyclesUntilVALU > 0;
428 if (!HasActiveState)
429 dbgs() << " Emit ["
430 << AMDGPU::getCoExecMaskName(getCoExecMaskForMI(*MI, TII))
431 << "]: " << *MI;
432 });
433 updateWMMAWindowState(MI: *MI);
434 updateTRANSState(MI: *MI);
435 updateMultiCycleVALUState(MI: *MI);
436}
437
438void GCNHazardRecognizer::EmitInstruction(SUnit *SU) {
439 EmitInstruction(MI: SU->getInstr());
440}
441
442void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) {
443 CurrCycleInstr = MI;
444 if (isSchedulerMode())
445 schedulerEmitInstruction(MI);
446}
447
448static bool isDivFMas(unsigned Opcode) {
449 return Opcode == AMDGPU::V_DIV_FMAS_F32_e64 || Opcode == AMDGPU::V_DIV_FMAS_F64_e64;
450}
451
452static bool isSGetReg(unsigned Opcode) {
453 return Opcode == AMDGPU::S_GETREG_B32 || Opcode == AMDGPU::S_GETREG_B32_const;
454}
455
456static bool isSSetReg(unsigned Opcode) {
457 switch (Opcode) {
458 case AMDGPU::S_SETREG_B32:
459 case AMDGPU::S_SETREG_B32_mode:
460 case AMDGPU::S_SETREG_IMM32_B32:
461 case AMDGPU::S_SETREG_IMM32_B32_mode:
462 return true;
463 }
464 return false;
465}
466
467static bool isRWLane(unsigned Opcode) {
468 return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32;
469}
470
471static bool isRFE(unsigned Opcode) {
472 return Opcode == AMDGPU::S_RFE_B64;
473}
474
475static bool isSMovRel(unsigned Opcode) {
476 switch (Opcode) {
477 case AMDGPU::S_MOVRELS_B32:
478 case AMDGPU::S_MOVRELS_B64:
479 case AMDGPU::S_MOVRELD_B32:
480 case AMDGPU::S_MOVRELD_B64:
481 return true;
482 default:
483 return false;
484 }
485}
486
487static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII,
488 const MachineInstr &MI) {
489 if (TII.isAlwaysGDS(Opcode: MI.getOpcode()))
490 return true;
491
492 switch (MI.getOpcode()) {
493 case AMDGPU::S_SENDMSG:
494 case AMDGPU::S_SENDMSGHALT:
495 case AMDGPU::S_TTRACEDATA:
496 return true;
497 // These DS opcodes don't support GDS.
498 case AMDGPU::DS_NOP:
499 case AMDGPU::DS_PERMUTE_B32:
500 case AMDGPU::DS_BPERMUTE_B32:
501 return false;
502 default:
503 if (TII.isDS(Opcode: MI.getOpcode())) {
504 int GDS = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
505 Name: AMDGPU::OpName::gds);
506 if (MI.getOperand(i: GDS).getImm())
507 return true;
508 }
509 return false;
510 }
511}
512
513static bool isPermlane(const MachineInstr &MI) {
514 unsigned Opcode = MI.getOpcode();
515 return Opcode == AMDGPU::V_PERMLANE16_B32_e64 ||
516 Opcode == AMDGPU::V_PERMLANE64_B32 ||
517 Opcode == AMDGPU::V_PERMLANEX16_B32_e64 ||
518 Opcode == AMDGPU::V_PERMLANE16_VAR_B32_e64 ||
519 Opcode == AMDGPU::V_PERMLANEX16_VAR_B32_e64 ||
520 Opcode == AMDGPU::V_PERMLANE16_SWAP_B32_e32 ||
521 Opcode == AMDGPU::V_PERMLANE16_SWAP_B32_e64 ||
522 Opcode == AMDGPU::V_PERMLANE32_SWAP_B32_e32 ||
523 Opcode == AMDGPU::V_PERMLANE32_SWAP_B32_e64 ||
524 Opcode == AMDGPU::V_PERMLANE_BCAST_B32_e64 ||
525 Opcode == AMDGPU::V_PERMLANE_UP_B32_e64 ||
526 Opcode == AMDGPU::V_PERMLANE_DOWN_B32_e64 ||
527 Opcode == AMDGPU::V_PERMLANE_XOR_B32_e64 ||
528 Opcode == AMDGPU::V_PERMLANE_IDX_GEN_B32_e64;
529}
530
531static bool isLdsDma(const MachineInstr &MI) {
532 return SIInstrInfo::isLDSDMA(MI);
533}
534
535static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) {
536 const MachineOperand *RegOp = TII->getNamedOperand(MI: RegInstr,
537 OperandName: AMDGPU::OpName::simm16);
538 return std::get<0>(t: AMDGPU::Hwreg::HwregEncoding::decode(Encoded: RegOp->getImm()));
539}
540
541ScheduleHazardRecognizer::HazardType
542GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
543 MachineInstr *MI = SU->getInstr();
544 // If we are not in "HazardRecognizerMode" and therefore not being run from
545 // the scheduler, track possible stalls from hazards but don't insert noops.
546 auto HazardType = isHazardRecognizerMode() ? NoopHazard : Hazard;
547
548 if (MI->isBundle())
549 return NoHazard;
550
551 // Check co-execution slot hazards and pipeline stalls in scheduler modes.
552 if (isSchedulerMode()) {
553 if (checkMultiShadowHazard(MI: *MI) > 0)
554 return Hazard;
555 if (checkWMMACoexecSlot(MI: *MI) > 0)
556 return Hazard;
557 if (checkTRANSHazard(MI: *MI) > 0)
558 return Hazard;
559 if (checkMultiCycleVALUHazard(MI: *MI) > 0)
560 return Hazard;
561 // The remaining checks are all defined by register dependences.
562 if (!hasPhysRegs())
563 return NoHazard;
564 }
565
566 if (SIInstrInfo::isSMRD(MI: *MI) && checkSMRDHazards(SMRD: MI) > 0)
567 return HazardType;
568
569 if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0)
570 return HazardType;
571
572 if (checkFPAtomicToDenormModeHazard(MI) > 0)
573 return HazardType;
574
575 // Hazards which cannot be mitigated with S_NOPs.
576 if (!isHazardRecognizerMode()) {
577 if (checkWMMACoexecutionHazards(MI) > 0) {
578 HasPendingWMMACoexecHazard = true;
579 return Hazard;
580 }
581 }
582
583 if (ST.hasNoDataDepHazard())
584 return NoHazard;
585
586 if (SIInstrInfo::isVMEM(MI: *MI) && checkVMEMHazards(VMEM: MI) > 0)
587 return HazardType;
588
589 if (SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true) &&
590 checkVALUHazards(VALU: MI) > 0)
591 return HazardType;
592
593 if (SIInstrInfo::isDPP(MI: *MI) && checkDPPHazards(DPP: MI) > 0)
594 return HazardType;
595
596 if (isDivFMas(Opcode: MI->getOpcode()) && checkDivFMasHazards(DivFMas: MI) > 0)
597 return HazardType;
598
599 if (isRWLane(Opcode: MI->getOpcode()) && checkRWLaneHazards(RWLane: MI) > 0)
600 return HazardType;
601
602 if ((SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true) ||
603 SIInstrInfo::isVMEM(MI: *MI) || SIInstrInfo::isDS(MI: *MI) ||
604 SIInstrInfo::isEXP(MI: *MI)) &&
605 checkMAIVALUHazards(MI) > 0)
606 return HazardType;
607
608 if (isSGetReg(Opcode: MI->getOpcode()) && checkGetRegHazards(GetRegInstr: MI) > 0)
609 return HazardType;
610
611 if (isSSetReg(Opcode: MI->getOpcode()) && checkSetRegHazards(SetRegInstr: MI) > 0)
612 return HazardType;
613
614 if (isRFE(Opcode: MI->getOpcode()) && checkRFEHazards(RFE: MI) > 0)
615 return HazardType;
616
617 if (((ST.hasReadM0MovRelInterpHazard() &&
618 (TII.isVINTRP(MI: *MI) || isSMovRel(Opcode: MI->getOpcode()) ||
619 MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 ||
620 MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) ||
621 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, MI: *MI)) ||
622 (ST.hasReadM0LdsDmaHazard() && isLdsDma(MI: *MI)) ||
623 (ST.hasReadM0LdsDirectHazard() &&
624 MI->readsRegister(Reg: AMDGPU::LDS_DIRECT, /*TRI=*/nullptr))) &&
625 checkReadM0Hazards(SMovRel: MI) > 0)
626 return HazardType;
627
628 if (SIInstrInfo::isMAI(MI: *MI) && checkMAIHazards(MI) > 0)
629 return HazardType;
630
631 if ((SIInstrInfo::isVMEM(MI: *MI) || SIInstrInfo::isDS(MI: *MI)) &&
632 checkMAILdStHazards(MI) > 0)
633 return HazardType;
634
635 if (MI->isInlineAsm() && checkInlineAsmHazards(IA: MI) > 0)
636 return HazardType;
637
638 return NoHazard;
639}
640
641static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII,
642 unsigned Quantity) {
643 while (Quantity > 0) {
644 unsigned Arg = std::min(a: Quantity, b: 8u);
645 Quantity -= Arg;
646 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_NOP))
647 .addImm(Val: Arg - 1);
648 }
649}
650
651unsigned
652GCNHazardRecognizer::getMFMAPipelineWaitStates(const MachineInstr &MI) const {
653 const MCSchedClassDesc *SC = TSchedModel.resolveSchedClass(MI: &MI);
654 assert(TSchedModel.getWriteProcResBegin(SC) !=
655 TSchedModel.getWriteProcResEnd(SC));
656 return TSchedModel.getWriteProcResBegin(SC)->ReleaseAtCycle;
657}
658
659void GCNHazardRecognizer::processBundle() {
660 MachineBasicBlock::instr_iterator MI = std::next(x: CurrCycleInstr->getIterator());
661 MachineBasicBlock::instr_iterator E = CurrCycleInstr->getParent()->instr_end();
662 // Check bundled MachineInstr's for hazards.
663 for (; MI != E && MI->isInsideBundle(); ++MI) {
664 CurrCycleInstr = &*MI;
665 unsigned WaitStates = PreEmitNoopsCommon(CurrCycleInstr);
666
667 if (isHazardRecognizerMode()) {
668 fixHazards(MI: CurrCycleInstr);
669
670 insertNoopsInBundle(MI: CurrCycleInstr, TII, Quantity: WaitStates);
671 }
672
673 // It’s unnecessary to track more than MaxLookAhead instructions. Since we
674 // include the bundled MI directly after, only add a maximum of
675 // (MaxLookAhead - 1) noops to EmittedInstrs.
676 for (unsigned i = 0, e = std::min(a: WaitStates, b: MaxLookAhead - 1); i < e; ++i)
677 EmittedInstrs.push_front(x: nullptr);
678
679 EmittedInstrs.push_front(x: CurrCycleInstr);
680 EmittedInstrs.resize(new_size: MaxLookAhead);
681 }
682 CurrCycleInstr = nullptr;
683}
684
685void GCNHazardRecognizer::runOnInstruction(MachineInstr *MI) {
686 assert(isHazardRecognizerMode());
687
688 unsigned NumPreNoops = PreEmitNoops(MI);
689 EmitNoops(Quantity: NumPreNoops);
690 if (MI->isInsideBundle())
691 insertNoopsInBundle(MI, TII, Quantity: NumPreNoops);
692 else
693 TII.insertNoops(MBB&: *MI->getParent(), MI: MachineBasicBlock::iterator(MI),
694 Quantity: NumPreNoops);
695 EmitInstruction(MI);
696 AdvanceCycle();
697}
698
699unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
700 assert(isHazardRecognizerMode());
701 CurrCycleInstr = MI;
702 unsigned W = PreEmitNoopsCommon(MI);
703 fixHazards(MI);
704 CurrCycleInstr = nullptr;
705 return std::max(a: W, b: NopPadding.getValue());
706}
707
708unsigned GCNHazardRecognizer::getHazardWaitStates(MachineInstr *MI) const {
709 unsigned W = 0;
710
711 // Check co-execution slot hazards and pipeline stalls in scheduler modes.
712 if (isSchedulerMode()) {
713 W = checkWMMACoexecSlot(MI: *MI);
714 W = std::max(a: W, b: checkTRANSHazard(MI: *MI));
715 W = std::max(a: W, b: checkMultiCycleVALUHazard(MI: *MI));
716 W = std::max(a: W, b: checkMultiShadowHazard(MI: *MI));
717 // The remaining checks are all defined by register dependences.
718 if (!hasPhysRegs())
719 return W;
720 }
721
722 return std::max(a: W, b: PreEmitNoopsCommon(MI));
723}
724
725unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) const {
726 if (MI->isBundle())
727 return 0;
728
729 int WaitStates = 0;
730
731 if (SIInstrInfo::isSMRD(MI: *MI))
732 return std::max(a: WaitStates, b: checkSMRDHazards(SMRD: MI));
733
734 if (ST.hasNSAtoVMEMBug())
735 WaitStates = std::max(a: WaitStates, b: checkNSAtoVMEMHazard(MI));
736
737 WaitStates = std::max(a: WaitStates, b: checkFPAtomicToDenormModeHazard(MI));
738
739 if (ST.hasNoDataDepHazard())
740 return WaitStates;
741
742 if (SIInstrInfo::isVMEM(MI: *MI))
743 WaitStates = std::max(a: WaitStates, b: checkVMEMHazards(VMEM: MI));
744
745 if (SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true))
746 WaitStates = std::max(a: WaitStates, b: checkVALUHazards(VALU: MI));
747
748 if (SIInstrInfo::isDPP(MI: *MI))
749 WaitStates = std::max(a: WaitStates, b: checkDPPHazards(DPP: MI));
750
751 if (isDivFMas(Opcode: MI->getOpcode()))
752 WaitStates = std::max(a: WaitStates, b: checkDivFMasHazards(DivFMas: MI));
753
754 if (isRWLane(Opcode: MI->getOpcode()))
755 WaitStates = std::max(a: WaitStates, b: checkRWLaneHazards(RWLane: MI));
756
757 if ((SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true) ||
758 SIInstrInfo::isVMEM(MI: *MI) || SIInstrInfo::isDS(MI: *MI) ||
759 SIInstrInfo::isEXP(MI: *MI)) &&
760 checkMAIVALUHazards(MI) > 0)
761 WaitStates = std::max(a: WaitStates, b: checkMAIVALUHazards(MI));
762
763 if (MI->isInlineAsm())
764 return std::max(a: WaitStates, b: checkInlineAsmHazards(IA: MI));
765
766 if (isSGetReg(Opcode: MI->getOpcode()))
767 return std::max(a: WaitStates, b: checkGetRegHazards(GetRegInstr: MI));
768
769 if (isSSetReg(Opcode: MI->getOpcode()))
770 return std::max(a: WaitStates, b: checkSetRegHazards(SetRegInstr: MI));
771
772 if (isRFE(Opcode: MI->getOpcode()))
773 return std::max(a: WaitStates, b: checkRFEHazards(RFE: MI));
774
775 if ((ST.hasReadM0MovRelInterpHazard() &&
776 (TII.isVINTRP(MI: *MI) || isSMovRel(Opcode: MI->getOpcode()) ||
777 MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 ||
778 MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) ||
779 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, MI: *MI)) ||
780 (ST.hasReadM0LdsDmaHazard() && isLdsDma(MI: *MI)) ||
781 (ST.hasReadM0LdsDirectHazard() &&
782 MI->readsRegister(Reg: AMDGPU::LDS_DIRECT, /*TRI=*/nullptr)))
783 return std::max(a: WaitStates, b: checkReadM0Hazards(SMovRel: MI));
784
785 if (SIInstrInfo::isMAI(MI: *MI))
786 return std::max(a: WaitStates, b: checkMAIHazards(MI));
787
788 if (SIInstrInfo::isVMEM(MI: *MI) || SIInstrInfo::isDS(MI: *MI))
789 return std::max(a: WaitStates, b: checkMAILdStHazards(MI));
790
791 if (ST.hasGFX950Insts() && isPermlane(MI: *MI))
792 return std::max(a: WaitStates, b: checkPermlaneHazards(MI));
793
794 return WaitStates;
795}
796
797void GCNHazardRecognizer::EmitNoop() {
798 EmittedInstrs.push_front(x: nullptr);
799}
800
801void GCNHazardRecognizer::AdvanceCycle() {
802 if (isSchedulerMode())
803 schedulerAdvanceCycle();
804
805 // When the scheduler detects a stall, it will call AdvanceCycle() without
806 // emitting any instructions.
807 if (!CurrCycleInstr) {
808 EmittedInstrs.push_front(x: nullptr);
809
810 if (HasPendingWMMACoexecHazard)
811 EmittedVALUInstrs.push_front(x: nullptr);
812 return;
813 }
814
815 HasPendingWMMACoexecHazard = false;
816
817 if (CurrCycleInstr->isBundle()) {
818 processBundle();
819 return;
820 }
821
822 unsigned NumWaitStates = TII.getNumWaitStates(MI: *CurrCycleInstr);
823 if (!NumWaitStates) {
824 CurrCycleInstr = nullptr;
825 return;
826 }
827
828 // Keep track of emitted instructions
829 EmittedInstrs.push_front(x: CurrCycleInstr);
830
831 bool IsVALUOrWMMA =
832 SIInstrInfo::isVALU(MI: *CurrCycleInstr, /*AllowLDSDMA=*/true) ||
833 SIInstrInfo::isWMMA(MI: *CurrCycleInstr) ||
834 SIInstrInfo::isSWMMAC(MI: *CurrCycleInstr);
835 if (IsVALUOrWMMA) {
836 EmittedVALUInstrs.push_front(x: CurrCycleInstr);
837 } else {
838 // A pending WMMA co-execution hazard optimistically records stall cycles as
839 // future V_NOPs. If the scheduler instead stalls for a different
840 // (S_NOP-resolvable) hazard and schedules a non-VALU into those cycles,
841 // they will not resolve the VALU-pipe hazard, so drop them here.
842 while (!EmittedVALUInstrs.empty() && EmittedVALUInstrs.front() == nullptr)
843 EmittedVALUInstrs.pop_front();
844 }
845
846 // Add a nullptr for each additional wait state after the first. Make sure
847 // not to add more than getMaxLookAhead() items to the list, since we
848 // truncate the list to that size right after this loop.
849 for (unsigned i = 1, e = std::min(a: NumWaitStates, b: getMaxLookAhead());
850 i < e; ++i) {
851 EmittedInstrs.push_front(x: nullptr);
852 }
853
854 // getMaxLookahead() is the largest number of wait states we will ever need
855 // to insert, so there is no point in keeping track of more than that many
856 // wait states.
857 EmittedInstrs.resize(new_size: getMaxLookAhead());
858 if (EmittedVALUInstrs.size() > MaxVALULookAhead)
859 EmittedVALUInstrs.resize(new_size: MaxVALULookAhead);
860
861 CurrCycleInstr = nullptr;
862}
863
864void GCNHazardRecognizer::RecedeCycle() {
865 assert(!isHazardRecognizerMode() &&
866 "Bottom-up scheduling shouldn't run in hazard recognizer mode");
867}
868
869//===----------------------------------------------------------------------===//
870// Helper Functions
871//===----------------------------------------------------------------------===//
872
873enum HazardFnResult { HazardFound, HazardExpired, NoHazardFound };
874
875// Search for a hazard in a block and its predecessors.
876template <typename StateT>
877static bool
878hasHazard(StateT InitialState,
879 function_ref<HazardFnResult(StateT &, const MachineInstr &)> IsHazard,
880 function_ref<void(StateT &, const MachineInstr &)> UpdateState,
881 const MachineBasicBlock *InitialMBB,
882 MachineBasicBlock::const_reverse_instr_iterator InitialI) {
883 struct StateMapKey {
884 SmallVectorImpl<StateT> *States;
885 unsigned Idx;
886 static bool isEqual(const StateMapKey &LHS, const StateMapKey &RHS) {
887 return LHS.States == RHS.States && LHS.Idx == RHS.Idx;
888 }
889 };
890 struct StateMapKeyTraits : DenseMapInfo<StateMapKey> {
891 static unsigned getHashValue(const StateMapKey &Key) {
892 return StateT::getHashValue((*Key.States)[Key.Idx]);
893 }
894 static unsigned getHashValue(const StateT &State) {
895 return StateT::getHashValue(State);
896 }
897 static bool isEqual(const StateMapKey &LHS, const StateMapKey &RHS) {
898 return StateT::isEqual((*LHS.States)[LHS.Idx], (*RHS.States)[RHS.Idx]);
899 }
900 static bool isEqual(const StateT &LHS, const StateMapKey &RHS) {
901 return StateT::isEqual(LHS, (*RHS.States)[RHS.Idx]);
902 }
903 };
904
905 SmallDenseMap<StateMapKey, unsigned, 8, StateMapKeyTraits> StateMap;
906 SmallVector<StateT, 8> States;
907
908 MachineBasicBlock::const_reverse_instr_iterator I = InitialI;
909 const MachineBasicBlock *MBB = InitialMBB;
910 StateT State = InitialState;
911
912 SmallSetVector<std::pair<const MachineBasicBlock *, unsigned>, 16> Worklist;
913 unsigned WorkIdx = 0;
914 for (;;) {
915 bool Expired = false;
916 for (auto E = MBB->instr_rend(); I != E; ++I) {
917 // No need to look at parent BUNDLE instructions.
918 if (I->isBundle())
919 continue;
920
921 auto Result = IsHazard(State, *I);
922 if (Result == HazardFound)
923 return true;
924 if (Result == HazardExpired) {
925 Expired = true;
926 break;
927 }
928
929 if (I->isInlineAsm() || I->isMetaInstruction())
930 continue;
931
932 UpdateState(State, *I);
933 }
934
935 if (!Expired) {
936 unsigned StateIdx = States.size();
937 StateMapKey Key = {&States, StateIdx};
938 auto Insertion = StateMap.insert_as(std::pair(Key, StateIdx), State);
939 if (Insertion.second) {
940 States.emplace_back(State);
941 } else {
942 StateIdx = Insertion.first->second;
943 }
944 for (MachineBasicBlock *Pred : MBB->predecessors())
945 Worklist.insert(X: std::pair(Pred, StateIdx));
946 }
947
948 if (WorkIdx == Worklist.size())
949 break;
950
951 unsigned StateIdx;
952 std::tie(args&: MBB, args&: StateIdx) = Worklist[WorkIdx++];
953 State = States[StateIdx];
954 I = MBB->instr_rbegin();
955 }
956
957 return false;
958}
959
960// Returns a minimum wait states since \p I walking all predecessors.
961// Only scans until \p IsExpired does not return true.
962// Can only be run in a hazard recognizer mode.
963static int
964getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
965 const MachineBasicBlock *MBB,
966 MachineBasicBlock::const_reverse_instr_iterator I,
967 int WaitStates, GCNHazardRecognizer::IsExpiredFn IsExpired,
968 DenseSet<const MachineBasicBlock *> &Visited,
969 GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates =
970 SIInstrInfo::getNumWaitStates) {
971 for (auto E = MBB->instr_rend(); I != E; ++I) {
972 // Don't add WaitStates for parent BUNDLE instructions.
973 if (I->isBundle())
974 continue;
975
976 if (IsHazard(*I))
977 return WaitStates;
978
979 if (I->isInlineAsm())
980 continue;
981
982 WaitStates += GetNumWaitStates(*I);
983
984 if (IsExpired(*I, WaitStates))
985 return std::numeric_limits<int>::max();
986 }
987
988 int MinWaitStates = std::numeric_limits<int>::max();
989 for (MachineBasicBlock *Pred : MBB->predecessors()) {
990 if (!Visited.insert(V: Pred).second)
991 continue;
992
993 int W = getWaitStatesSince(IsHazard, MBB: Pred, I: Pred->instr_rbegin(), WaitStates,
994 IsExpired, Visited, GetNumWaitStates);
995
996 MinWaitStates = std::min(a: MinWaitStates, b: W);
997 }
998
999 return MinWaitStates;
1000}
1001
1002static int
1003getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
1004 const MachineInstr *MI,
1005 GCNHazardRecognizer::IsExpiredFn IsExpired,
1006 GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates =
1007 SIInstrInfo::getNumWaitStates) {
1008 DenseSet<const MachineBasicBlock *> Visited;
1009 return getWaitStatesSince(IsHazard, MBB: MI->getParent(),
1010 I: std::next(x: MI->getReverseIterator()), WaitStates: 0, IsExpired,
1011 Visited, GetNumWaitStates);
1012}
1013
1014int GCNHazardRecognizer::getWaitStatesSince(
1015 IsHazardFn IsHazard, int Limit, GetNumWaitStatesFn GetNumWaitStates) const {
1016 if (isHazardRecognizerMode()) {
1017 auto IsExpiredFn = [Limit](const MachineInstr &, int WaitStates) {
1018 return WaitStates >= Limit;
1019 };
1020 return ::getWaitStatesSince(IsHazard, MI: CurrCycleInstr, IsExpired: IsExpiredFn,
1021 GetNumWaitStates);
1022 }
1023
1024 int WaitStates = 0;
1025 for (MachineInstr *MI : EmittedInstrs) {
1026 if (MI) {
1027 if (IsHazard(*MI))
1028 return WaitStates;
1029
1030 if (MI->isInlineAsm())
1031 continue;
1032 }
1033 WaitStates += MI ? GetNumWaitStates(*MI) : 1;
1034
1035 if (WaitStates >= Limit)
1036 break;
1037 }
1038 return std::numeric_limits<int>::max();
1039}
1040
1041int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard,
1042 int Limit) const {
1043 return getWaitStatesSince(IsHazard, Limit, GetNumWaitStates: SIInstrInfo::getNumWaitStates);
1044}
1045
1046int GCNHazardRecognizer::getWaitStatesSinceVALU(IsHazardFn IsHazard,
1047 int Limit) const {
1048 if (isHazardRecognizerMode()) {
1049 auto GetVALUWaitStates = [](const MachineInstr &MI) -> unsigned {
1050 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ? 1 : 0;
1051 };
1052 return getWaitStatesSince(IsHazard, Limit, GetNumWaitStates: GetVALUWaitStates);
1053 }
1054
1055 // EmittedVALUInstrs is capped at MaxVALULookAhead, so a Limit beyond that
1056 // window could miss a hazard. Keep the cap in sync with the wait-state
1057 // tables.
1058 assert(Limit <= (int)MaxVALULookAhead &&
1059 "Limit exceeds the EmittedVALUInstrs lookahead window");
1060 int WaitStates = 0;
1061 for (MachineInstr *MI : EmittedVALUInstrs) {
1062 if (MI) {
1063 if (IsHazard(*MI))
1064 return WaitStates;
1065 }
1066
1067 ++WaitStates;
1068
1069 if (WaitStates >= Limit)
1070 break;
1071 }
1072 return std::numeric_limits<int>::max();
1073}
1074
1075int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg,
1076 IsHazardFn IsHazardDef,
1077 int Limit) const {
1078 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1079
1080 auto IsHazardFn = [IsHazardDef, TRI, Reg](const MachineInstr &MI) {
1081 return IsHazardDef(MI) && MI.modifiesRegister(Reg, TRI);
1082 };
1083
1084 return getWaitStatesSince(IsHazard: IsHazardFn, Limit);
1085}
1086
1087int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard,
1088 int Limit) const {
1089 auto IsHazardFn = [IsHazard](const MachineInstr &MI) {
1090 return isSSetReg(Opcode: MI.getOpcode()) && IsHazard(MI);
1091 };
1092
1093 return getWaitStatesSince(IsHazard: IsHazardFn, Limit);
1094}
1095
1096//===----------------------------------------------------------------------===//
1097// No-op Hazard Detection
1098//===----------------------------------------------------------------------===//
1099
1100static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV,
1101 MCRegister Reg) {
1102 for (MCRegUnit Unit : TRI.regunits(Reg))
1103 BV.set(static_cast<unsigned>(Unit));
1104}
1105
1106static void addRegsToSet(const SIRegisterInfo &TRI,
1107 iterator_range<MachineInstr::const_mop_iterator> Ops,
1108 BitVector &DefSet, BitVector &UseSet) {
1109 for (const MachineOperand &Op : Ops) {
1110 if (Op.isReg())
1111 addRegUnits(TRI, BV&: Op.isDef() ? DefSet : UseSet, Reg: Op.getReg().asMCReg());
1112 }
1113}
1114
1115void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) const {
1116 addRegsToSet(TRI, Ops: MI.operands(), DefSet&: ClauseDefs, UseSet&: ClauseUses);
1117}
1118
1119static bool breaksSMEMSoftClause(MachineInstr *MI) {
1120 return !SIInstrInfo::isSMRD(MI: *MI);
1121}
1122
1123static bool breaksVMEMSoftClause(MachineInstr *MI) {
1124 return !SIInstrInfo::isVMEM(MI: *MI);
1125}
1126
1127int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) const {
1128 // SMEM soft clause are only present on VI+, and only matter if xnack is
1129 // enabled.
1130 if (!ST.isXNACKEnabled())
1131 return 0;
1132
1133 bool IsSMRD = TII.isSMRD(MI: *MEM);
1134
1135 resetClause();
1136
1137 // A soft-clause is any group of consecutive SMEM instructions. The
1138 // instructions in this group may return out of order and/or may be
1139 // replayed (i.e. the same instruction issued more than once).
1140 //
1141 // In order to handle these situations correctly we need to make sure that
1142 // when a clause has more than one instruction, no instruction in the clause
1143 // writes to a register that is read by another instruction in the clause
1144 // (including itself). If we encounter this situation, we need to break the
1145 // clause by inserting a non SMEM instruction.
1146
1147 for (MachineInstr *MI : EmittedInstrs) {
1148 // When we hit a non-SMEM instruction then we have passed the start of the
1149 // clause and we can stop.
1150 if (!MI)
1151 break;
1152
1153 if (IsSMRD ? breaksSMEMSoftClause(MI) : breaksVMEMSoftClause(MI))
1154 break;
1155
1156 addClauseInst(MI: *MI);
1157 }
1158
1159 if (ClauseDefs.none())
1160 return 0;
1161
1162 // We need to make sure not to put loads and stores in the same clause if they
1163 // use the same address. For now, just start a new clause whenever we see a
1164 // store.
1165 if (MEM->mayStore())
1166 return 1;
1167
1168 addClauseInst(MI: *MEM);
1169
1170 // If the set of defs and uses intersect then we cannot add this instruction
1171 // to the clause, so we have a hazard.
1172 return ClauseDefs.anyCommon(RHS: ClauseUses) ? 1 : 0;
1173}
1174
1175int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) const {
1176 int WaitStatesNeeded = 0;
1177
1178 WaitStatesNeeded = checkSoftClauseHazards(MEM: SMRD);
1179
1180 // This SMRD hazard only affects SI.
1181 if (!ST.hasSMRDReadVALUDefHazard())
1182 return WaitStatesNeeded;
1183
1184 // A read of an SGPR by SMRD instruction requires 4 wait states when the
1185 // SGPR was written by a VALU instruction.
1186 int SmrdSgprWaitStates = 4;
1187 auto IsHazardDefFn = [this](const MachineInstr &MI) {
1188 return TII.isVALU(MI, /*AllowLDSDMA=*/true);
1189 };
1190 auto IsBufferHazardDefFn = [this](const MachineInstr &MI) {
1191 return TII.isSALU(MI);
1192 };
1193
1194 bool IsBufferSMRD = TII.isBufferSMRD(MI: *SMRD);
1195
1196 for (const MachineOperand &Use : SMRD->uses()) {
1197 if (!Use.isReg())
1198 continue;
1199 int WaitStatesNeededForUse =
1200 SmrdSgprWaitStates - getWaitStatesSinceDef(Reg: Use.getReg(), IsHazardDef: IsHazardDefFn,
1201 Limit: SmrdSgprWaitStates);
1202 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
1203
1204 // This fixes what appears to be undocumented hardware behavior in SI where
1205 // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor
1206 // needs some number of nops in between. We don't know how many we need, but
1207 // let's use 4. This wasn't discovered before probably because the only
1208 // case when this happens is when we expand a 64-bit pointer into a full
1209 // descriptor and use s_buffer_load_dword instead of s_load_dword, which was
1210 // probably never encountered in the closed-source land.
1211 if (IsBufferSMRD) {
1212 int WaitStatesNeededForUse =
1213 SmrdSgprWaitStates - getWaitStatesSinceDef(Reg: Use.getReg(),
1214 IsHazardDef: IsBufferHazardDefFn,
1215 Limit: SmrdSgprWaitStates);
1216 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
1217 }
1218 }
1219
1220 return WaitStatesNeeded;
1221}
1222
1223int GCNHazardRecognizer::checkVMEMHazards(MachineInstr *VMEM) const {
1224 if (!ST.hasVMEMReadSGPRVALUDefHazard())
1225 return 0;
1226
1227 int WaitStatesNeeded = checkSoftClauseHazards(MEM: VMEM);
1228
1229 // A read of an SGPR by a VMEM instruction requires 5 wait states when the
1230 // SGPR was written by a VALU Instruction.
1231 const int VmemSgprWaitStates = 5;
1232 auto IsHazardDefFn = [this](const MachineInstr &MI) {
1233 return TII.isVALU(MI, /*AllowLDSDMA=*/true);
1234 };
1235 for (const MachineOperand &Use : VMEM->uses()) {
1236 if (!Use.isReg() || TRI.isVectorRegister(MRI: MF.getRegInfo(), Reg: Use.getReg()))
1237 continue;
1238
1239 int WaitStatesNeededForUse =
1240 VmemSgprWaitStates - getWaitStatesSinceDef(Reg: Use.getReg(), IsHazardDef: IsHazardDefFn,
1241 Limit: VmemSgprWaitStates);
1242 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
1243 }
1244 return WaitStatesNeeded;
1245}
1246
1247int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) const {
1248 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1249 const SIInstrInfo *TII = ST.getInstrInfo();
1250
1251 // Check for DPP VGPR read after VALU VGPR write and EXEC write.
1252 int DppVgprWaitStates = 2;
1253 int DppExecWaitStates = 5;
1254 int WaitStatesNeeded = 0;
1255 auto IsHazardDefFn = [TII](const MachineInstr &MI) {
1256 return TII->isVALU(MI, /*AllowLDSDMA=*/true);
1257 };
1258
1259 for (const MachineOperand &Use : DPP->uses()) {
1260 if (!Use.isReg() || !TRI->isVGPR(MRI: MF.getRegInfo(), Reg: Use.getReg()))
1261 continue;
1262 int WaitStatesNeededForUse =
1263 DppVgprWaitStates - getWaitStatesSinceDef(
1264 Reg: Use.getReg(),
1265 IsHazardDef: [](const MachineInstr &) { return true; },
1266 Limit: DppVgprWaitStates);
1267 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
1268 }
1269
1270 WaitStatesNeeded = std::max(
1271 a: WaitStatesNeeded,
1272 b: DppExecWaitStates - getWaitStatesSinceDef(Reg: AMDGPU::EXEC, IsHazardDef: IsHazardDefFn,
1273 Limit: DppExecWaitStates));
1274
1275 return WaitStatesNeeded;
1276}
1277
1278int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) const {
1279 const SIInstrInfo *TII = ST.getInstrInfo();
1280
1281 // v_div_fmas requires 4 wait states after a write to vcc from a VALU
1282 // instruction.
1283 const int DivFMasWaitStates = 4;
1284 auto IsHazardDefFn = [TII](const MachineInstr &MI) {
1285 return TII->isVALU(MI, /*AllowLDSDMA=*/true);
1286 };
1287 int WaitStatesNeeded = getWaitStatesSinceDef(Reg: AMDGPU::VCC, IsHazardDef: IsHazardDefFn,
1288 Limit: DivFMasWaitStates);
1289
1290 return DivFMasWaitStates - WaitStatesNeeded;
1291}
1292
1293int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) const {
1294 const SIInstrInfo *TII = ST.getInstrInfo();
1295 unsigned GetRegHWReg = getHWReg(TII, RegInstr: *GetRegInstr);
1296
1297 const int GetRegWaitStates = 2;
1298 auto IsHazardFn = [TII, GetRegHWReg](const MachineInstr &MI) {
1299 return GetRegHWReg == getHWReg(TII, RegInstr: MI);
1300 };
1301 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazard: IsHazardFn, Limit: GetRegWaitStates);
1302
1303 return GetRegWaitStates - WaitStatesNeeded;
1304}
1305
1306int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) const {
1307 const SIInstrInfo *TII = ST.getInstrInfo();
1308 unsigned HWReg = getHWReg(TII, RegInstr: *SetRegInstr);
1309
1310 const int SetRegWaitStates = ST.getSetRegWaitStates();
1311 auto IsHazardFn = [TII, HWReg](const MachineInstr &MI) {
1312 return HWReg == getHWReg(TII, RegInstr: MI);
1313 };
1314 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazard: IsHazardFn, Limit: SetRegWaitStates);
1315 return SetRegWaitStates - WaitStatesNeeded;
1316}
1317
1318int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) const {
1319 if (!MI.mayStore())
1320 return -1;
1321
1322 const SIInstrInfo *TII = ST.getInstrInfo();
1323 unsigned Opcode = MI.getOpcode();
1324 const MCInstrDesc &Desc = MI.getDesc();
1325
1326 int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::vdata);
1327 int VDataRCID = -1;
1328 if (VDataIdx != -1)
1329 VDataRCID = TII->getOpRegClassID(OpInfo: Desc.operands()[VDataIdx]);
1330
1331 if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) {
1332 // There is no hazard if the instruction does not use vector regs
1333 // (like wbinvl1)
1334 if (VDataIdx == -1)
1335 return -1;
1336 if (AMDGPU::getRegBitWidth(RCID: VDataRCID) > 64) {
1337 // When SOFFSET-dependent wide-store windows apply, the BUFFER_STORE
1338 // source-vgpr WAR hazard exists for every SOFFSET shape; the wait-state
1339 // count differs by SOFFSET and is computed in checkVALUHazardsHelper.
1340 // Otherwise the hazard only exists if soffset is not an SGPR.
1341 if (ST.hasVDecCoExecHazard())
1342 return VDataIdx;
1343 const MachineOperand *SOffset =
1344 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
1345 if (!SOffset || !SOffset->isReg())
1346 return VDataIdx;
1347 }
1348 }
1349
1350 // MIMG instructions create a hazard if they don't use a 256-bit T# and
1351 // the store size is greater than 8 bytes and they have more than two bits
1352 // of their dmask set.
1353 // All our MIMG definitions use a 256-bit T#, so we can skip checking for them.
1354 if (TII->isMIMG(MI)) {
1355 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::srsrc);
1356 assert(SRsrcIdx != -1 && AMDGPU::getRegBitWidth(TII->getOpRegClassID(
1357 Desc.operands()[SRsrcIdx])) == 256);
1358 (void)SRsrcIdx;
1359 }
1360
1361 if (TII->isFLAT(MI)) {
1362 // There is no hazard if the instruction does not use vector regs
1363 if (VDataIdx == -1)
1364 return -1;
1365
1366 if (AMDGPU::getRegBitWidth(RCID: VDataRCID) > 64)
1367 return VDataIdx;
1368 }
1369
1370 return -1;
1371}
1372
1373int GCNHazardRecognizer::checkUniformWindowVALUHazardsHelper(
1374 Register Reg) const {
1375 // Wide stores need a single wait-state bubble before a VALU that overwrites
1376 // store data. createsVALUHazard already excludes MUBUF/MTBUF stores with an
1377 // SGPR SOFFSET.
1378 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1379
1380 auto IsHazard = [&](const MachineInstr &MI) {
1381 int DataIdx = createsVALUHazard(MI);
1382 return DataIdx >= 0 &&
1383 TRI->regsOverlap(RegA: MI.getOperand(i: DataIdx).getReg(), RegB: Reg);
1384 };
1385
1386 return std::max(a: 0, b: 1 - getWaitStatesSince(IsHazard, /*Limit=*/1));
1387}
1388
1389int GCNHazardRecognizer::checkSOFFSETWindowVALUHazardsHelper(
1390 Register Reg) const {
1391 // The required wait-state window depends on the producer's SOFFSET shape:
1392 // - MUBUF/MTBUF wide store with sgpr SOFFSET: 1 wait state.
1393 // - MUBUF/MTBUF wide store with literal/absent SOFFSET, and FLAT wide
1394 // store: 2 wait states.
1395 // The 1-cycle sgpr-SOFFSET window was measured on gfx950.
1396 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1397 const SIInstrInfo *TII = ST.getInstrInfo();
1398
1399 int WaitStatesNeeded = 0;
1400
1401 // Scan each wait-state window separately and take the max padding needed.
1402 // getWaitStatesSince supplies the minimum distance to a producer over paths.
1403 for (int Window = 1; Window <= 2; ++Window) {
1404 auto IsHazard = [&](const MachineInstr &MI) {
1405 int DataIdx = createsVALUHazard(MI);
1406 if (DataIdx < 0 ||
1407 !TRI->regsOverlap(RegA: MI.getOperand(i: DataIdx).getReg(), RegB: Reg))
1408 return false;
1409
1410 // Window 1 matches every hazard producer. Window 2 excludes BUF stores
1411 // with an SGPR SOFFSET, which only require a single wait state.
1412 if (Window == 1 || !TII->isBUF(MI))
1413 return true;
1414
1415 const MachineOperand *SOffset =
1416 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
1417 return !SOffset || !SOffset->isReg();
1418 };
1419 WaitStatesNeeded = std::max(a: WaitStatesNeeded,
1420 b: Window - getWaitStatesSince(IsHazard, Limit: Window));
1421 }
1422
1423 return WaitStatesNeeded;
1424}
1425
1426int GCNHazardRecognizer::checkVALUHazardsHelper(
1427 const MachineOperand &Def, const MachineRegisterInfo &MRI) const {
1428 // Helper to check for the hazard where VMEM instructions that store more
1429 // than 8 bytes can have their store data overwritten by the next
1430 // instruction.
1431 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1432
1433 if (!TRI->isVectorRegister(MRI, Reg: Def.getReg()))
1434 return 0;
1435
1436 if (ST.hasVDecCoExecHazard())
1437 return checkSOFFSETWindowVALUHazardsHelper(Reg: Def.getReg());
1438
1439 return checkUniformWindowVALUHazardsHelper(Reg: Def.getReg());
1440}
1441
1442/// Dest sel forwarding issue occurs if additional logic is needed to swizzle /
1443/// pack the computed value into correct bit position of the dest register. This
1444/// occurs if we have SDWA with dst_sel != DWORD or if we have op_sel with
1445/// dst_sel that is not aligned to the register. This function analayzes the \p
1446/// MI and \returns an operand with dst forwarding issue, or nullptr if
1447/// none exists.
1448static const MachineOperand *
1449getDstSelForwardingOperand(const MachineInstr &MI, const GCNSubtarget &ST) {
1450 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false))
1451 return nullptr;
1452
1453 const SIInstrInfo *TII = ST.getInstrInfo();
1454
1455 unsigned Opcode = MI.getOpcode();
1456
1457 // There are three different types of instructions
1458 // which produce forwarded dest: 1. SDWA with dst_sel != DWORD, 2. VOP3
1459 // which write hi bits (e.g. op_sel[3] == 1), and 3. FP8DstSelInst
1460 // (instructions with dest byte sel, e.g. CVT_SR_BF8_F32) and
1461 // op_sel[3:2]
1462 // != 0
1463 if (SIInstrInfo::isSDWA(MI)) {
1464 // Type 1: SDWA with dst_sel != DWORD
1465 if (auto *DstSel = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::dst_sel))
1466 if (DstSel->getImm() != AMDGPU::SDWA::DWORD)
1467 return TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1468 }
1469
1470 AMDGPU::FPType IsFP4OrFP8ConvOpc = AMDGPU::getFPDstSelType(Opc: Opcode);
1471 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::op_sel)) {
1472 // Type 2: VOP3 which write the hi bits
1473 if (TII->getNamedImmOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers) &
1474 SISrcMods::DST_OP_SEL)
1475 return TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1476
1477 // Type 3: FP8DstSelInst with op_sel[3:2] != 0)
1478 if (IsFP4OrFP8ConvOpc == AMDGPU::FPType::FP8 &&
1479 (TII->getNamedImmOperand(MI, OperandName: AMDGPU::OpName::src2_modifiers) &
1480 SISrcMods::OP_SEL_0))
1481 return TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1482 }
1483
1484 // Special case: nop is required for all the opsel values for fp4 sr variant
1485 // cvt scale instructions
1486 if (IsFP4OrFP8ConvOpc == AMDGPU::FPType::FP4)
1487 return TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1488
1489 return nullptr;
1490}
1491
1492/// Checks whether the provided \p MI "consumes" the operand with a Dest sel
1493/// fowarding issue \p Dst . We may "consume" the Dst via a standard explicit
1494/// RAW, or through irregular ways (e.g implicit RAW, certain types of WAW)
1495static bool consumesDstSelForwardingOperand(const MachineInstr *VALU,
1496 const MachineOperand *Dst,
1497 const SIRegisterInfo *TRI) {
1498 // We must consider implicit reads of the VALU. SDWA with dst_sel and
1499 // UNUSED_PRESERVE will implicitly read the result from forwarded dest,
1500 // and we must account for that hazard.
1501 // We also must account for WAW hazards. In particular, WAW with dest
1502 // preserve semantics (e.g. VOP3 with op_sel, VOP2 &&
1503 // !zeroesHigh16BitsOfDest) will read the forwarded dest for parity
1504 // check for ECC. Without accounting for this hazard, the ECC will be
1505 // wrong.
1506 // TODO: limit to RAW (including implicit reads) + problematic WAW (i.e.
1507 // complete zeroesHigh16BitsOfDest)
1508 for (auto &Operand : VALU->operands()) {
1509 if (Operand.isReg() && TRI->regsOverlap(RegA: Dst->getReg(), RegB: Operand.getReg())) {
1510 return true;
1511 }
1512 }
1513 return false;
1514}
1515
1516int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) const {
1517 int WaitStatesNeeded = 0;
1518
1519 if (ST.hasTransForwardingHazard() && !SIInstrInfo::isTRANS(MI: *VALU)) {
1520 const int TransDefWaitstates = 1;
1521
1522 auto IsTransDefFn = [this, VALU](const MachineInstr &MI) {
1523 if (!SIInstrInfo::isTRANS(MI))
1524 return false;
1525 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1526 const SIInstrInfo *TII = ST.getInstrInfo();
1527 Register Def = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst)->getReg();
1528
1529 for (const MachineOperand &Use : VALU->explicit_uses()) {
1530 if (Use.isReg() && TRI->regsOverlap(RegA: Def, RegB: Use.getReg()))
1531 return true;
1532 }
1533
1534 return false;
1535 };
1536
1537 int WaitStatesNeededForDef =
1538 TransDefWaitstates -
1539 getWaitStatesSince(IsHazard: IsTransDefFn, Limit: TransDefWaitstates);
1540 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1541 }
1542
1543 if (ST.hasDstSelForwardingHazard() || ST.hasCvtScaleForwardingHazard()) {
1544 const int Shift16DefWaitstates = 1;
1545
1546 auto IsShift16BitDefFn = [this, VALU](const MachineInstr &ProducerMI) {
1547 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1548 const MachineOperand *ForwardedDst =
1549 getDstSelForwardingOperand(MI: ProducerMI, ST);
1550 if (ForwardedDst) {
1551 return consumesDstSelForwardingOperand(VALU, Dst: ForwardedDst, TRI);
1552 }
1553
1554 if (ProducerMI.isInlineAsm()) {
1555 // Assume inline asm has dst forwarding hazard
1556 for (auto &Def : ProducerMI.all_defs()) {
1557 if (consumesDstSelForwardingOperand(VALU, Dst: &Def, TRI))
1558 return true;
1559 }
1560 }
1561
1562 return false;
1563 };
1564
1565 int WaitStatesNeededForDef =
1566 Shift16DefWaitstates -
1567 getWaitStatesSince(IsHazard: IsShift16BitDefFn, Limit: Shift16DefWaitstates);
1568 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1569 }
1570
1571 if (ST.hasVDecCoExecHazard()) {
1572 const int VALUWriteSGPRVALUReadWaitstates = 2;
1573 const int VALUWriteEXECRWLane = 4;
1574 const int VALUWriteVGPRReadlaneRead = 1;
1575
1576 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1577 const MachineRegisterInfo &MRI = MF.getRegInfo();
1578 Register UseReg;
1579 auto IsVALUDefSGPRFn = [&UseReg, TRI](const MachineInstr &MI) {
1580 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
1581 return false;
1582 return MI.modifiesRegister(Reg: UseReg, TRI);
1583 };
1584
1585 for (const MachineOperand &Use : VALU->explicit_uses()) {
1586 if (!Use.isReg())
1587 continue;
1588
1589 UseReg = Use.getReg();
1590 if (TRI->isSGPRReg(MRI, Reg: UseReg)) {
1591 int WaitStatesNeededForDef =
1592 VALUWriteSGPRVALUReadWaitstates -
1593 getWaitStatesSince(IsHazard: IsVALUDefSGPRFn,
1594 Limit: VALUWriteSGPRVALUReadWaitstates);
1595 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1596 }
1597 }
1598
1599 if (VALU->readsRegister(Reg: AMDGPU::VCC, TRI)) {
1600 UseReg = AMDGPU::VCC;
1601 int WaitStatesNeededForDef =
1602 VALUWriteSGPRVALUReadWaitstates -
1603 getWaitStatesSince(IsHazard: IsVALUDefSGPRFn, Limit: VALUWriteSGPRVALUReadWaitstates);
1604 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1605 }
1606
1607 switch (VALU->getOpcode()) {
1608 case AMDGPU::V_READLANE_B32:
1609 case AMDGPU::V_READFIRSTLANE_B32: {
1610 MachineOperand *Src = TII.getNamedOperand(MI&: *VALU, OperandName: AMDGPU::OpName::src0);
1611 UseReg = Src->getReg();
1612 int WaitStatesNeededForDef =
1613 VALUWriteVGPRReadlaneRead -
1614 getWaitStatesSince(IsHazard: IsVALUDefSGPRFn, Limit: VALUWriteVGPRReadlaneRead);
1615 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1616 }
1617 [[fallthrough]];
1618 case AMDGPU::V_WRITELANE_B32: {
1619 UseReg = AMDGPU::EXEC;
1620 int WaitStatesNeededForDef =
1621 VALUWriteEXECRWLane -
1622 getWaitStatesSince(IsHazard: IsVALUDefSGPRFn, Limit: VALUWriteEXECRWLane);
1623 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1624 break;
1625 }
1626 default:
1627 break;
1628 }
1629 }
1630
1631 // This checks for the hazard where VMEM instructions that store more than
1632 // 8 bytes can have there store data over written by the next instruction.
1633 if (!ST.has12DWordStoreHazard())
1634 return WaitStatesNeeded;
1635
1636 const MachineRegisterInfo &MRI = MF.getRegInfo();
1637
1638 for (const MachineOperand &Def : VALU->defs()) {
1639 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: checkVALUHazardsHelper(Def, MRI));
1640 }
1641
1642 return WaitStatesNeeded;
1643}
1644
1645int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) const {
1646 // This checks for hazards associated with inline asm statements.
1647 // Since inline asms can contain just about anything, we use this
1648 // to call/leverage other check*Hazard routines. Note that
1649 // this function doesn't attempt to address all possible inline asm
1650 // hazards (good luck), but is a collection of what has been
1651 // problematic thus far.
1652
1653 // see checkVALUHazards()
1654 if (!ST.has12DWordStoreHazard() && !ST.hasDstSelForwardingHazard() &&
1655 !ST.hasCvtScaleForwardingHazard())
1656 return 0;
1657
1658 const MachineRegisterInfo &MRI = MF.getRegInfo();
1659 int WaitStatesNeeded = 0;
1660
1661 for (const MachineOperand &Op :
1662 llvm::drop_begin(RangeOrContainer: IA->operands(), N: InlineAsm::MIOp_FirstOperand)) {
1663 if (Op.isReg() && Op.isDef()) {
1664 if (!TRI.isVectorRegister(MRI, Reg: Op.getReg()))
1665 continue;
1666
1667 if (ST.has12DWordStoreHazard()) {
1668 WaitStatesNeeded =
1669 std::max(a: WaitStatesNeeded, b: checkVALUHazardsHelper(Def: Op, MRI));
1670 }
1671 }
1672 }
1673
1674 if (ST.hasDstSelForwardingHazard()) {
1675 const int Shift16DefWaitstates = 1;
1676
1677 auto IsShift16BitDefFn = [this, &IA](const MachineInstr &ProducerMI) {
1678 const MachineOperand *Dst = getDstSelForwardingOperand(MI: ProducerMI, ST);
1679 // Assume inline asm reads the dst
1680 if (Dst)
1681 return IA->modifiesRegister(Reg: Dst->getReg(), TRI: &TRI) ||
1682 IA->readsRegister(Reg: Dst->getReg(), TRI: &TRI);
1683
1684 if (ProducerMI.isInlineAsm()) {
1685 // If MI is inline asm, assume it has dst forwarding hazard
1686 for (auto &Def : ProducerMI.all_defs()) {
1687 if (IA->modifiesRegister(Reg: Def.getReg(), TRI: &TRI) ||
1688 IA->readsRegister(Reg: Def.getReg(), TRI: &TRI)) {
1689 return true;
1690 }
1691 }
1692 }
1693
1694 return false;
1695 };
1696
1697 int WaitStatesNeededForDef =
1698 Shift16DefWaitstates -
1699 getWaitStatesSince(IsHazard: IsShift16BitDefFn, Limit: Shift16DefWaitstates);
1700 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1701 }
1702
1703 return WaitStatesNeeded;
1704}
1705
1706int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) const {
1707 const SIInstrInfo *TII = ST.getInstrInfo();
1708 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1709 const MachineRegisterInfo &MRI = MF.getRegInfo();
1710
1711 const MachineOperand *LaneSelectOp =
1712 TII->getNamedOperand(MI&: *RWLane, OperandName: AMDGPU::OpName::src1);
1713
1714 if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, Reg: LaneSelectOp->getReg()))
1715 return 0;
1716
1717 Register LaneSelectReg = LaneSelectOp->getReg();
1718 auto IsHazardFn = [TII](const MachineInstr &MI) {
1719 return TII->isVALU(MI, /*AllowLDSDMA=*/true);
1720 };
1721
1722 const int RWLaneWaitStates = 4;
1723 int WaitStatesSince = getWaitStatesSinceDef(Reg: LaneSelectReg, IsHazardDef: IsHazardFn,
1724 Limit: RWLaneWaitStates);
1725 return RWLaneWaitStates - WaitStatesSince;
1726}
1727
1728int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) const {
1729 if (!ST.hasRFEHazards())
1730 return 0;
1731
1732 const SIInstrInfo *TII = ST.getInstrInfo();
1733
1734 const int RFEWaitStates = 1;
1735
1736 auto IsHazardFn = [TII](const MachineInstr &MI) {
1737 return getHWReg(TII, RegInstr: MI) == AMDGPU::Hwreg::ID_TRAPSTS;
1738 };
1739 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazard: IsHazardFn, Limit: RFEWaitStates);
1740 return RFEWaitStates - WaitStatesNeeded;
1741}
1742
1743int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) const {
1744 const SIInstrInfo *TII = ST.getInstrInfo();
1745 const int ReadM0WaitStates = 1;
1746 auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isSALU(MI); };
1747 return ReadM0WaitStates -
1748 getWaitStatesSinceDef(Reg: AMDGPU::M0, IsHazardDef: IsHazardFn, Limit: ReadM0WaitStates);
1749}
1750
1751void GCNHazardRecognizer::emitVNops(MachineBasicBlock &MBB,
1752 MachineBasicBlock::iterator InsertPt,
1753 int WaitStatesNeeded, bool IsHoisting) {
1754 const DebugLoc &DL = IsHoisting ? DebugLoc() : InsertPt->getDebugLoc();
1755 for (int I = 0; I < WaitStatesNeeded; ++I)
1756 BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_NOP_e32));
1757}
1758
1759void GCNHazardRecognizer::fixHazards(MachineInstr *MI) {
1760 fixVMEMtoScalarWriteHazards(MI);
1761 fixVcmpxPermlaneHazards(MI);
1762 fixSMEMtoVectorWriteHazards(MI);
1763 fixVcmpxExecWARHazard(MI);
1764 fixLdsBranchVmemWARHazard(MI);
1765 if (ST.hasLdsDirect()) {
1766 fixLdsDirectVALUHazard(MI);
1767 fixLdsDirectVMEMHazard(MI);
1768 }
1769 fixVALUPartialForwardingHazard(MI);
1770 fixVALUTransUseHazard(MI);
1771 fixVALUTransCoexecutionHazards(MI);
1772 fixWMMAHazards(MI); // fall-through if co-execution is enabled.
1773 fixWMMACoexecutionHazards(MI);
1774 fixShift64HighRegBug(MI);
1775 fixVALUMaskWriteHazard(MI);
1776 fixRequiredExportPriority(MI);
1777 if (ST.requiresWaitIdleBeforeGetReg())
1778 fixGetRegWaitIdle(MI);
1779 if (ST.hasDsAtomicAsyncBarrierArriveB64PipeBug())
1780 fixDsAtomicAsyncBarrierArriveB64(MI);
1781 if (ST.hasScratchBaseForwardingHazard())
1782 fixScratchBaseForwardingHazard(MI);
1783 if (ST.setRegModeNeedsVNOPs())
1784 fixSetRegMode(MI);
1785 if (ST.hasNeedsTDMDrain())
1786 fixTDM(MI);
1787}
1788
1789static bool isVCmpXWritesExec(const SIInstrInfo &TII, const SIRegisterInfo &TRI,
1790 const MachineInstr &MI) {
1791 return (TII.isVOPC(MI) ||
1792 (MI.isCompare() && (TII.isVOP3(MI) || TII.isSDWA(MI)))) &&
1793 MI.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &TRI);
1794}
1795
1796bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) {
1797 if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(MI: *MI))
1798 return false;
1799
1800 const SIInstrInfo *TII = ST.getInstrInfo();
1801 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1802 auto IsHazardFn = [TII, TRI](const MachineInstr &MI) {
1803 return isVCmpXWritesExec(TII: *TII, TRI: *TRI, MI);
1804 };
1805
1806 auto IsExpiredFn = [](const MachineInstr &MI, int) {
1807 unsigned Opc = MI.getOpcode();
1808 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
1809 Opc != AMDGPU::V_NOP_e32 && Opc != AMDGPU::V_NOP_e64 &&
1810 Opc != AMDGPU::V_NOP_sdwa;
1811 };
1812
1813 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
1814 std::numeric_limits<int>::max())
1815 return false;
1816
1817 // V_NOP will be discarded by SQ.
1818 // Use V_MOV_B32 v?, v?. Register must be alive so use src0 of V_PERMLANE*
1819 // which is always a VGPR and available.
1820 auto *Src0 = TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src0);
1821 Register Reg = Src0->getReg();
1822 bool IsUndef = Src0->isUndef();
1823 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
1824 MCID: TII->get(Opcode: AMDGPU::V_MOV_B32_e32))
1825 .addReg(RegNo: Reg, Flags: RegState::Define | getDeadRegState(B: IsUndef))
1826 .addReg(RegNo: Reg, Flags: IsUndef ? RegState::Undef : RegState::Kill);
1827
1828 return true;
1829}
1830
1831bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) {
1832 if (!ST.hasVMEMtoScalarWriteHazard())
1833 return false;
1834 assert(!ST.hasExtendedWaitCounts());
1835
1836 if (!SIInstrInfo::isSALU(MI: *MI) && !SIInstrInfo::isSMRD(MI: *MI))
1837 return false;
1838
1839 if (MI->getNumDefs() == 0)
1840 return false;
1841
1842 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1843
1844 auto IsHazardFn = [TRI, MI](const MachineInstr &I) {
1845 if (!SIInstrInfo::isVMEM(MI: I) && !SIInstrInfo::isDS(MI: I))
1846 return false;
1847
1848 for (const MachineOperand &Def : MI->defs()) {
1849 const MachineOperand *Op =
1850 I.findRegisterUseOperand(Reg: Def.getReg(), TRI, isKill: false);
1851 if (!Op)
1852 continue;
1853 return true;
1854 }
1855 return false;
1856 };
1857
1858 auto IsExpiredFn = [](const MachineInstr &MI, int) {
1859 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ||
1860 (MI.getOpcode() == AMDGPU::S_WAITCNT &&
1861 !MI.getOperand(i: 0).getImm()) ||
1862 (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
1863 AMDGPU::DepCtr::decodeFieldVmVsrc(Encoded: MI.getOperand(i: 0).getImm()) == 0);
1864 };
1865
1866 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
1867 std::numeric_limits<int>::max())
1868 return false;
1869
1870 const SIInstrInfo *TII = ST.getInstrInfo();
1871 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
1872 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
1873 .addImm(Val: AMDGPU::DepCtr::encodeFieldVmVsrc(VmVsrc: 0, STI: ST));
1874 return true;
1875}
1876
1877bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) {
1878 if (!ST.hasSMEMtoVectorWriteHazard())
1879 return false;
1880 assert(!ST.hasExtendedWaitCounts());
1881
1882 if (!SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true))
1883 return false;
1884
1885 AMDGPU::OpName SDSTName;
1886 switch (MI->getOpcode()) {
1887 case AMDGPU::V_READLANE_B32:
1888 case AMDGPU::V_READFIRSTLANE_B32:
1889 SDSTName = AMDGPU::OpName::vdst;
1890 break;
1891 default:
1892 SDSTName = AMDGPU::OpName::sdst;
1893 break;
1894 }
1895
1896 const SIInstrInfo *TII = ST.getInstrInfo();
1897 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1898 const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(GPU: ST.getCPU());
1899 const MachineOperand *SDST = TII->getNamedOperand(MI&: *MI, OperandName: SDSTName);
1900 if (!SDST) {
1901 for (const auto &MO : MI->implicit_operands()) {
1902 if (MO.isDef() && TRI->isSGPRClass(RC: TRI->getPhysRegBaseClass(Reg: MO.getReg()))) {
1903 SDST = &MO;
1904 break;
1905 }
1906 }
1907 }
1908
1909 if (!SDST)
1910 return false;
1911
1912 const Register SDSTReg = SDST->getReg();
1913 auto IsHazardFn = [SDSTReg, TRI](const MachineInstr &I) {
1914 return SIInstrInfo::isSMRD(MI: I) && I.readsRegister(Reg: SDSTReg, TRI);
1915 };
1916
1917 auto IsExpiredFn = [TII, IV](const MachineInstr &MI, int) {
1918 if (TII->isSALU(MI)) {
1919 switch (MI.getOpcode()) {
1920 case AMDGPU::S_SETVSKIP:
1921 case AMDGPU::S_VERSION:
1922 case AMDGPU::S_WAITCNT_VSCNT:
1923 case AMDGPU::S_WAITCNT_VMCNT:
1924 case AMDGPU::S_WAITCNT_EXPCNT:
1925 // These instructions cannot not mitigate the hazard.
1926 return false;
1927 case AMDGPU::S_WAITCNT_LGKMCNT:
1928 // Reducing lgkmcnt count to 0 always mitigates the hazard.
1929 return (MI.getOperand(i: 1).getImm() == 0) &&
1930 (MI.getOperand(i: 0).getReg() == AMDGPU::SGPR_NULL);
1931 case AMDGPU::S_WAITCNT: {
1932 const int64_t Imm = MI.getOperand(i: 0).getImm();
1933 AMDGPU::Waitcnt Decoded = AMDGPU::decodeWaitcnt(Version: IV, Encoded: Imm);
1934 // DsCnt corresponds to LGKMCnt here.
1935 return Decoded.get(T: AMDGPU::DS_CNT) == 0;
1936 }
1937 default:
1938 assert((!SIInstrInfo::isWaitcnt(MI.getOpcode()) ||
1939 MI.getOpcode() == AMDGPU::S_WAIT_IDLE) &&
1940 "unexpected wait count instruction");
1941 // SOPP instructions cannot mitigate the hazard.
1942 if (TII->isSOPP(MI))
1943 return false;
1944 // At this point the SALU can be assumed to mitigate the hazard
1945 // because either:
1946 // (a) it is independent of the at risk SMEM (breaking chain),
1947 // or
1948 // (b) it is dependent on the SMEM, in which case an appropriate
1949 // s_waitcnt lgkmcnt _must_ exist between it and the at risk
1950 // SMEM instruction.
1951 return true;
1952 }
1953 }
1954 return false;
1955 };
1956
1957 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
1958 std::numeric_limits<int>::max())
1959 return false;
1960
1961 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
1962 MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::SGPR_NULL)
1963 .addImm(Val: 0);
1964 return true;
1965}
1966
1967bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) {
1968 if (!ST.hasVcmpxExecWARHazard())
1969 return false;
1970 assert(!ST.hasExtendedWaitCounts());
1971
1972 if (!SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true))
1973 return false;
1974
1975 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1976 if (!MI->modifiesRegister(Reg: AMDGPU::EXEC, TRI))
1977 return false;
1978
1979 auto IsHazardFn = [TRI](const MachineInstr &I) {
1980 if (SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true))
1981 return false;
1982 return I.readsRegister(Reg: AMDGPU::EXEC, TRI);
1983 };
1984
1985 const SIInstrInfo *TII = ST.getInstrInfo();
1986 auto IsExpiredFn = [TII, TRI](const MachineInstr &MI, int) {
1987 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true)) {
1988 if (TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst))
1989 return true;
1990 for (auto MO : MI.implicit_operands())
1991 if (MO.isDef() && TRI->isSGPRClass(RC: TRI->getPhysRegBaseClass(Reg: MO.getReg())))
1992 return true;
1993 }
1994 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
1995 AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: MI.getOperand(i: 0).getImm()) == 0)
1996 return true;
1997 return false;
1998 };
1999
2000 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2001 std::numeric_limits<int>::max())
2002 return false;
2003
2004 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2005 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
2006 .addImm(Val: AMDGPU::DepCtr::encodeFieldSaSdst(SaSdst: 0, STI: ST));
2007 return true;
2008}
2009
2010static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF,
2011 const GCNSubtarget &ST) {
2012 if (!ST.hasLdsBranchVmemWARHazard())
2013 return false;
2014
2015 // Check if the necessary condition for the hazard is met: both LDS and VMEM
2016 // instructions need to appear in the same function.
2017 bool HasLds = false;
2018 bool HasVmem = false;
2019 for (auto &MBB : MF) {
2020 for (auto &MI : MBB) {
2021 HasLds |= SIInstrInfo::isDS(MI) || SIInstrInfo::isLDSDMA(MI);
2022 HasVmem |= SIInstrInfo::isVMEM(MI);
2023 if (HasLds && HasVmem)
2024 return true;
2025 }
2026 }
2027 return false;
2028}
2029
2030static bool isStoreCountWaitZero(const MachineInstr &I) {
2031 return I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
2032 I.getOperand(i: 0).getReg() == AMDGPU::SGPR_NULL &&
2033 !I.getOperand(i: 1).getImm();
2034}
2035
2036bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) {
2037 if (!RunLdsBranchVmemWARHazardFixup)
2038 return false;
2039
2040 assert(ST.hasLdsBranchVmemWARHazard());
2041 assert(!ST.hasExtendedWaitCounts());
2042
2043 auto IsHazardInst = [](const MachineInstr &MI) {
2044 if (SIInstrInfo::isDS(MI) || SIInstrInfo::isLDSDMA(MI))
2045 return 1;
2046 if (SIInstrInfo::isVMEM(MI))
2047 return 2;
2048 return 0;
2049 };
2050
2051 auto InstType = IsHazardInst(*MI);
2052 if (!InstType)
2053 return false;
2054
2055 auto IsExpiredFn = [&IsHazardInst](const MachineInstr &I, int) {
2056 return IsHazardInst(I) || isStoreCountWaitZero(I);
2057 };
2058
2059 auto IsHazardFn = [InstType, &IsHazardInst](const MachineInstr &I) {
2060 if (!I.isBranch())
2061 return false;
2062
2063 auto IsHazardFn = [InstType, IsHazardInst](const MachineInstr &I) {
2064 auto InstType2 = IsHazardInst(I);
2065 return InstType2 && InstType != InstType2;
2066 };
2067
2068 auto IsExpiredFn = [InstType, &IsHazardInst](const MachineInstr &I, int) {
2069 auto InstType2 = IsHazardInst(I);
2070 if (InstType == InstType2)
2071 return true;
2072
2073 return isStoreCountWaitZero(I);
2074 };
2075
2076 return ::getWaitStatesSince(IsHazard: IsHazardFn, MI: &I, IsExpired: IsExpiredFn) !=
2077 std::numeric_limits<int>::max();
2078 };
2079
2080 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2081 std::numeric_limits<int>::max())
2082 return false;
2083
2084 const SIInstrInfo *TII = ST.getInstrInfo();
2085 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2086 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_VSCNT))
2087 .addReg(RegNo: AMDGPU::SGPR_NULL, Flags: RegState::Undef)
2088 .addImm(Val: 0);
2089
2090 return true;
2091}
2092
2093bool GCNHazardRecognizer::fixLdsDirectVALUHazard(MachineInstr *MI) {
2094 if (!SIInstrInfo::isLDSDIR(MI: *MI))
2095 return false;
2096
2097 const int NoHazardWaitStates = 15;
2098 const MachineOperand *VDST = TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::vdst);
2099 const Register VDSTReg = VDST->getReg();
2100
2101 bool VisitedTrans = false;
2102 auto IsHazardFn = [this, VDSTReg, &VisitedTrans](const MachineInstr &I) {
2103 if (!SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true))
2104 return false;
2105 VisitedTrans = VisitedTrans || SIInstrInfo::isTRANS(MI: I);
2106 // Cover both WAR and WAW
2107 return I.readsRegister(Reg: VDSTReg, TRI: &TRI) || I.modifiesRegister(Reg: VDSTReg, TRI: &TRI);
2108 };
2109 auto IsExpiredFn = [&](const MachineInstr &I, int WaitStates) {
2110 if (WaitStates >= NoHazardWaitStates)
2111 return true;
2112 // Instructions which cause va_vdst==0 expire hazard
2113 return SIInstrInfo::isVMEM(MI: I) || SIInstrInfo::isDS(MI: I) ||
2114 SIInstrInfo::isEXP(MI: I);
2115 };
2116 auto GetWaitStatesFn = [](const MachineInstr &MI) {
2117 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ? 1 : 0;
2118 };
2119
2120 DenseSet<const MachineBasicBlock *> Visited;
2121 auto Count = ::getWaitStatesSince(IsHazard: IsHazardFn, MBB: MI->getParent(),
2122 I: std::next(x: MI->getReverseIterator()), WaitStates: 0,
2123 IsExpired: IsExpiredFn, Visited, GetNumWaitStates: GetWaitStatesFn);
2124
2125 // Transcendentals can execute in parallel to other VALUs.
2126 // This makes va_vdst count unusable with a mixture of VALU and TRANS.
2127 if (VisitedTrans)
2128 Count = 0;
2129
2130 MachineOperand *WaitVdstOp =
2131 TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::waitvdst);
2132 WaitVdstOp->setImm(std::min(a: Count, b: NoHazardWaitStates));
2133
2134 return true;
2135}
2136
2137bool GCNHazardRecognizer::fixLdsDirectVMEMHazard(MachineInstr *MI) {
2138 if (!SIInstrInfo::isLDSDIR(MI: *MI))
2139 return false;
2140
2141 const MachineOperand *VDST = TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::vdst);
2142 const Register VDSTReg = VDST->getReg();
2143
2144 auto IsHazardFn = [this, VDSTReg](const MachineInstr &I) {
2145 if (!SIInstrInfo::isVMEM(MI: I) && !SIInstrInfo::isDS(MI: I))
2146 return false;
2147 return I.readsRegister(Reg: VDSTReg, TRI: &TRI) || I.modifiesRegister(Reg: VDSTReg, TRI: &TRI);
2148 };
2149 bool LdsdirCanWait = ST.hasLdsWaitVMSRC();
2150 // TODO: On GFX12 the hazard should expire on S_WAIT_LOADCNT/SAMPLECNT/BVHCNT
2151 // according to the type of VMEM instruction.
2152 auto IsExpiredFn = [this, LdsdirCanWait](const MachineInstr &I, int) {
2153 return SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true) ||
2154 SIInstrInfo::isEXP(MI: I) ||
2155 (I.getOpcode() == AMDGPU::S_WAITCNT && !I.getOperand(i: 0).getImm()) ||
2156 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2157 AMDGPU::DepCtr::decodeFieldVmVsrc(Encoded: I.getOperand(i: 0).getImm()) == 0) ||
2158 (LdsdirCanWait && SIInstrInfo::isLDSDIR(MI: I) &&
2159 !TII.getNamedOperand(MI: I, OperandName: AMDGPU::OpName::waitvsrc)->getImm());
2160 };
2161
2162 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2163 std::numeric_limits<int>::max())
2164 return false;
2165
2166 if (LdsdirCanWait) {
2167 TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::waitvsrc)->setImm(0);
2168 } else {
2169 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2170 MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
2171 .addImm(Val: AMDGPU::DepCtr::encodeFieldVmVsrc(VmVsrc: 0, STI: ST));
2172 }
2173
2174 return true;
2175}
2176
2177bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(MachineInstr *MI) {
2178 if (!ST.hasVALUPartialForwardingHazard())
2179 return false;
2180 assert(!ST.hasExtendedWaitCounts());
2181
2182 if (!ST.isWave64() || !SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true))
2183 return false;
2184
2185 SmallSetVector<Register, 4> SrcVGPRs;
2186
2187 for (const MachineOperand &Use : MI->explicit_uses()) {
2188 if (Use.isReg() && TRI.isVGPR(MRI: MF.getRegInfo(), Reg: Use.getReg()))
2189 SrcVGPRs.insert(X: Use.getReg());
2190 }
2191
2192 // Only applies with >= 2 unique VGPR sources
2193 if (SrcVGPRs.size() <= 1)
2194 return false;
2195
2196 // Look for the following pattern:
2197 // Va <- VALU [PreExecPos]
2198 // intv1
2199 // Exec <- SALU [ExecPos]
2200 // intv2
2201 // Vb <- VALU [PostExecPos]
2202 // intv3
2203 // MI Va, Vb (WaitState = 0)
2204 //
2205 // Where:
2206 // intv1 + intv2 <= 2 VALUs
2207 // intv3 <= 4 VALUs
2208 //
2209 // If found, insert an appropriate S_WAITCNT_DEPCTR before MI.
2210
2211 const int Intv1plus2MaxVALUs = 2;
2212 const int Intv3MaxVALUs = 4;
2213 const int IntvMaxVALUs = 6;
2214 const int NoHazardVALUWaitStates = IntvMaxVALUs + 2;
2215
2216 struct StateType {
2217 SmallDenseMap<Register, int, 4> DefPos;
2218 int ExecPos = std::numeric_limits<int>::max();
2219 int VALUs = 0;
2220
2221 static unsigned getHashValue(const StateType &State) {
2222 hash_code H = hash_combine(args: State.ExecPos, args: State.VALUs);
2223 for (const auto &[Reg, Pos] : State.DefPos)
2224 H = hash_combine(args: H, args: Reg, args: Pos);
2225 return H;
2226 }
2227 static bool isEqual(const StateType &LHS, const StateType &RHS) {
2228 return LHS.DefPos == RHS.DefPos && LHS.ExecPos == RHS.ExecPos &&
2229 LHS.VALUs == RHS.VALUs;
2230 }
2231 };
2232
2233 StateType State;
2234
2235 // This overloads expiry testing with all the hazard detection
2236 auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) {
2237 // Too many VALU states have passed
2238 if (State.VALUs > NoHazardVALUWaitStates)
2239 return HazardExpired;
2240
2241 // Instructions which cause va_vdst==0 expire hazard
2242 if (SIInstrInfo::isVMEM(MI: I) || SIInstrInfo::isDS(MI: I) ||
2243 SIInstrInfo::isEXP(MI: I) ||
2244 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2245 AMDGPU::DepCtr::decodeFieldVaVdst(Encoded: I.getOperand(i: 0).getImm()) == 0))
2246 return HazardExpired;
2247
2248 // Track registers writes
2249 bool Changed = false;
2250 if (SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true)) {
2251 for (Register Src : SrcVGPRs) {
2252 if (!State.DefPos.count(Val: Src) && I.modifiesRegister(Reg: Src, TRI: &TRI)) {
2253 State.DefPos[Src] = State.VALUs;
2254 Changed = true;
2255 }
2256 }
2257 } else if (SIInstrInfo::isSALU(MI: I)) {
2258 if (State.ExecPos == std::numeric_limits<int>::max()) {
2259 if (!State.DefPos.empty() && I.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &TRI)) {
2260 State.ExecPos = State.VALUs;
2261 Changed = true;
2262 }
2263 }
2264 }
2265
2266 // Early expiration: too many VALUs in intv3
2267 if (State.VALUs > Intv3MaxVALUs && State.DefPos.empty())
2268 return HazardExpired;
2269
2270 // Only evaluate state if something changed
2271 if (!Changed)
2272 return NoHazardFound;
2273
2274 // Determine positions of VALUs pre/post exec change
2275 if (State.ExecPos == std::numeric_limits<int>::max())
2276 return NoHazardFound;
2277
2278 int PreExecPos = std::numeric_limits<int>::max();
2279 int PostExecPos = std::numeric_limits<int>::max();
2280
2281 for (auto Entry : State.DefPos) {
2282 int DefVALUs = Entry.second;
2283 if (DefVALUs != std::numeric_limits<int>::max()) {
2284 if (DefVALUs >= State.ExecPos)
2285 PreExecPos = std::min(a: PreExecPos, b: DefVALUs);
2286 else
2287 PostExecPos = std::min(a: PostExecPos, b: DefVALUs);
2288 }
2289 }
2290
2291 // Need a VALUs post exec change
2292 if (PostExecPos == std::numeric_limits<int>::max())
2293 return NoHazardFound;
2294
2295 // Too many VALUs in intv3?
2296 int Intv3VALUs = PostExecPos;
2297 if (Intv3VALUs > Intv3MaxVALUs)
2298 return HazardExpired;
2299
2300 // Too many VALUs in intv2?
2301 int Intv2VALUs = (State.ExecPos - PostExecPos) - 1;
2302 if (Intv2VALUs > Intv1plus2MaxVALUs)
2303 return HazardExpired;
2304
2305 // Need a VALUs pre exec change
2306 if (PreExecPos == std::numeric_limits<int>::max())
2307 return NoHazardFound;
2308
2309 // Too many VALUs in intv1?
2310 int Intv1VALUs = PreExecPos - State.ExecPos;
2311 if (Intv1VALUs > Intv1plus2MaxVALUs)
2312 return HazardExpired;
2313
2314 // Too many VALUs in intv1 + intv2
2315 if (Intv1VALUs + Intv2VALUs > Intv1plus2MaxVALUs)
2316 return HazardExpired;
2317
2318 return HazardFound;
2319 };
2320 auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) {
2321 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
2322 State.VALUs += 1;
2323 };
2324
2325 if (!hasHazard<StateType>(InitialState: State, IsHazard: IsHazardFn, UpdateState: UpdateStateFn, InitialMBB: MI->getParent(),
2326 InitialI: std::next(x: MI->getReverseIterator())))
2327 return false;
2328
2329 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2330 MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
2331 .addImm(Val: AMDGPU::DepCtr::encodeFieldVaVdst(VaVdst: 0, STI: ST));
2332
2333 return true;
2334}
2335
2336bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) {
2337 if (!ST.hasVALUTransUseHazard())
2338 return false;
2339 assert(!ST.hasExtendedWaitCounts());
2340
2341 if (!SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true))
2342 return false;
2343
2344 SmallSet<Register, 4> SrcVGPRs;
2345
2346 for (const MachineOperand &Use : MI->explicit_uses()) {
2347 if (Use.isReg() && TRI.isVGPR(MRI: MF.getRegInfo(), Reg: Use.getReg()))
2348 SrcVGPRs.insert(V: Use.getReg());
2349 }
2350
2351 // Look for the following pattern:
2352 // Va <- TRANS VALU
2353 // intv
2354 // MI Va (WaitState = 0)
2355 //
2356 // Where:
2357 // intv <= 5 VALUs / 1 TRANS
2358 //
2359 // If found, insert an appropriate S_WAITCNT_DEPCTR before MI.
2360
2361 const int IntvMaxVALUs = 5;
2362 const int IntvMaxTRANS = 1;
2363
2364 struct StateType {
2365 int VALUs = 0;
2366 int TRANS = 0;
2367
2368 static unsigned getHashValue(const StateType &State) {
2369 return hash_combine(args: State.VALUs, args: State.TRANS);
2370 }
2371 static bool isEqual(const StateType &LHS, const StateType &RHS) {
2372 return LHS.VALUs == RHS.VALUs && LHS.TRANS == RHS.TRANS;
2373 }
2374 };
2375
2376 StateType State;
2377
2378 // This overloads expiry testing with all the hazard detection
2379 auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) {
2380 // Too many VALU states have passed
2381 if (State.VALUs > IntvMaxVALUs || State.TRANS > IntvMaxTRANS)
2382 return HazardExpired;
2383
2384 // Instructions which cause va_vdst==0 expire hazard
2385 if (SIInstrInfo::isVMEM(MI: I) || SIInstrInfo::isDS(MI: I) ||
2386 SIInstrInfo::isEXP(MI: I) ||
2387 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2388 AMDGPU::DepCtr::decodeFieldVaVdst(Encoded: I.getOperand(i: 0).getImm()) == 0))
2389 return HazardExpired;
2390
2391 // Track registers writes
2392 if (SIInstrInfo::isTRANS(MI: I)) {
2393 for (Register Src : SrcVGPRs) {
2394 if (I.modifiesRegister(Reg: Src, TRI: &TRI)) {
2395 return HazardFound;
2396 }
2397 }
2398 }
2399
2400 return NoHazardFound;
2401 };
2402 auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) {
2403 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
2404 State.VALUs += 1;
2405 if (SIInstrInfo::isTRANS(MI))
2406 State.TRANS += 1;
2407 };
2408
2409 if (!hasHazard<StateType>(InitialState: State, IsHazard: IsHazardFn, UpdateState: UpdateStateFn, InitialMBB: MI->getParent(),
2410 InitialI: std::next(x: MI->getReverseIterator())))
2411 return false;
2412
2413 // Hazard is observed - insert a wait on va_dst counter to ensure hazard is
2414 // avoided.
2415 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2416 MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
2417 .addImm(Val: AMDGPU::DepCtr::encodeFieldVaVdst(VaVdst: 0, STI: ST));
2418
2419 return true;
2420}
2421
2422bool GCNHazardRecognizer::fixVALUTransCoexecutionHazards(MachineInstr *MI) {
2423 if (!ST.hasTransCoexecutionHazard() || // Coexecution disabled.
2424 !SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true) ||
2425 SIInstrInfo::isTRANS(MI: *MI))
2426 return false;
2427
2428 const SIInstrInfo *TII = ST.getInstrInfo();
2429 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2430
2431 auto IsTransHazardFn = [MI, TII, TRI](const MachineInstr &I) {
2432 if (!SIInstrInfo::isTRANS(MI: I))
2433 return false;
2434
2435 // RAW: Trans(I) writes, VALU(MI) reads.
2436 Register TransDef = TII->getNamedOperand(MI: I, OperandName: AMDGPU::OpName::vdst)->getReg();
2437 for (const MachineOperand &ValuUse : MI->explicit_uses()) {
2438 if (ValuUse.isReg() && TRI->regsOverlap(RegA: TransDef, RegB: ValuUse.getReg()))
2439 return true;
2440 }
2441
2442 auto *ValuDst = TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::vdst);
2443 if (!ValuDst || !ValuDst->isReg())
2444 return false;
2445
2446 // WAR: Trans(I) reads, VALU(MI) writes.
2447 Register ValuDef = ValuDst->getReg();
2448 for (const MachineOperand &TransUse : I.explicit_uses()) {
2449 if (TransUse.isReg() && TRI->regsOverlap(RegA: ValuDef, RegB: TransUse.getReg()))
2450 return true;
2451 }
2452
2453 return false;
2454 };
2455
2456 auto IsExpiredFn = [](const MachineInstr &I, int) {
2457 return SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true);
2458 };
2459
2460 const int HasVALU = std::numeric_limits<int>::max();
2461 if (::getWaitStatesSince(IsHazard: IsTransHazardFn, MI, IsExpired: IsExpiredFn) == HasVALU)
2462 return false;
2463
2464 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::V_NOP_e32));
2465 return true;
2466}
2467
2468bool GCNHazardRecognizer::fixWMMAHazards(MachineInstr *MI) {
2469 if (!SIInstrInfo::isWMMA(MI: *MI) && !SIInstrInfo::isSWMMAC(MI: *MI))
2470 return false;
2471
2472 const SIInstrInfo *TII = ST.getInstrInfo();
2473 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2474
2475 auto IsHazardFn = [MI, TII, TRI, this](const MachineInstr &I) {
2476 if (!SIInstrInfo::isWMMA(MI: I) && !SIInstrInfo::isSWMMAC(MI: I))
2477 return false;
2478
2479 // Src0(matrix A) or Src1(matrix B) of the current wmma instruction overlaps
2480 // with the dest(matrix D) of the previous wmma.
2481 const Register CurSrc0Reg =
2482 TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src0)->getReg();
2483 const Register CurSrc1Reg =
2484 TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src1)->getReg();
2485
2486 const Register PrevDstReg =
2487 TII->getNamedOperand(MI: I, OperandName: AMDGPU::OpName::vdst)->getReg();
2488
2489 if (TRI->regsOverlap(RegA: PrevDstReg, RegB: CurSrc0Reg) ||
2490 TRI->regsOverlap(RegA: PrevDstReg, RegB: CurSrc1Reg)) {
2491 return true;
2492 }
2493
2494 // GFX12+ allows overlap of matrix C with PrevDstReg (hardware will stall)
2495 // but Index can't overlap with PrevDstReg.
2496 if (AMDGPU::isGFX12Plus(STI: ST)) {
2497 if (SIInstrInfo::isSWMMAC(MI: *MI)) {
2498 const Register CurIndex =
2499 TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src2)->getReg();
2500 if (TRI->regsOverlap(RegA: PrevDstReg, RegB: CurIndex))
2501 return true;
2502 }
2503 return false;
2504 }
2505
2506 return false;
2507 };
2508
2509 auto IsExpiredFn = [](const MachineInstr &I, int) {
2510 return SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true);
2511 };
2512
2513 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2514 std::numeric_limits<int>::max())
2515 return false;
2516
2517 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::V_NOP_e32));
2518
2519 return true;
2520}
2521
2522static bool isCoexecutableVALUInst(const MachineInstr &MI) {
2523 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false) &&
2524 !SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI);
2525}
2526
2527// Classify XDL WMMA instructions into co-execution hazard categories
2528// (Refer to SPG 4.6.12.1), mainly based on instruction latency.
2529//
2530// Category 0: WMMA with Latency 8
2531// WMMA_*F16, WMMA_*BF16
2532// WMMA_*_16X16X128_{FP8,BF8}
2533// WMMA_*F8F6F4 if SRCA & SRCB are not both F4
2534//
2535// Category 1: WMMA Latency 16
2536// WMMA_IU8
2537//
2538// Category 2: SWMMAC with Latency 8
2539// SWMMAC_*F16, SWMMAC_*BF16,
2540// SWMMAC_*FP8FP8
2541// SWMMAC_*BF8FP8
2542// SWMMAC_*FP8BF8
2543// SWMMAC_*BF8BF8
2544//
2545// Category 3: SWMMAC with Latency 16
2546// SWMMAC_IU8
2547//
2548// Category 4: 16 Pass GFX1251 WMMA with latency 16
2549// V_WMMA_*_16X16X32_{F16,BF16}
2550// V_WMMA_{F32,F16}_16X16X64_{FP8,BF8}*
2551// V_WMMA_F32_16x16x128_F8F6F4 (F4 only)
2552// V_SWMMAC_*_16X16X64_{F16,BF16}
2553// V_SWMMAC_{F32,F16}_16X16X128_{FP8,BF8}*
2554//
2555// Category 5: 32 Pass GFX1251 WMMA with latency 32
2556// V_WMMA_F32_16x16x128_F8F6F4 (not all F4)
2557// V_WMMA_{F32,F16}_16X16X128_{FP8,BF8}*
2558// V_WMMA_F32_32X16X128_F4
2559// V_WMMA_I32_16X16X64_IU8
2560// V_WMMA_I32_16X16X64_IU8
2561//
2562// Category 6: gfx1250 WMMA with Latency 4 (one co-execution slot)
2563// WMMA_*_16X16X64_{FP8,BF8}
2564// WMMA_*F8F6F4 if SRCA & SRCB are both F4
2565static unsigned getWMMAHazardInstInCategory(const MachineInstr &MI,
2566 const SIInstrInfo *TII,
2567 const TargetSchedModel &SchedModel,
2568 const GCNSubtarget &ST) {
2569 assert(TII->isXDLWMMA(MI) && "must be xdl wmma");
2570 bool IsSWMMAC = SIInstrInfo::isSWMMAC(MI);
2571 bool IsLowestRateWMMA = ST.hasGFX125xLowestRateWMMA();
2572 unsigned Category = 0;
2573
2574 unsigned Latency = SchedModel.computeInstrLatency(MI: &MI);
2575 switch (Latency) {
2576 case 4:
2577 // Dense 4-cycle WMMA (gfx1250 16x16x64 FP8/BF8 and f8f6f4 with both
2578 // inputs F4). One co-execution slot; there is no 4-cycle SWMMAC.
2579 assert(!IsSWMMAC && "no 4-cycle SWMMAC expected");
2580 Category = 6;
2581 break;
2582 case 8:
2583 Category = IsSWMMAC ? 2 : 0;
2584 break;
2585 case 16:
2586 Category = IsLowestRateWMMA ? 4 : (IsSWMMAC ? 3 : 1);
2587 break;
2588 case 32:
2589 assert(IsLowestRateWMMA && "latency 32 is not expected");
2590 Category = 5;
2591 break;
2592 default:
2593 llvm_unreachable("unexpected xdl wmma latency");
2594 } // end switch.
2595
2596 return Category;
2597}
2598
2599int GCNHazardRecognizer::checkWMMACoexecutionHazards(MachineInstr *MI) const {
2600 if (!ST.hasWMMACoexecutionHazards())
2601 return 0;
2602
2603 const SIInstrInfo *TII = ST.getInstrInfo();
2604 if (!TII->isXDLWMMA(MI: *MI) && !isCoexecutableVALUInst(MI: *MI))
2605 return 0;
2606
2607 // WaitStates here is the number of V_NOPs or unrelated VALU instructions must
2608 // be in between the first WMMA and the second instruction to cover the hazard
2609 // (WMMAWaitStates if the second is also a WMMA, VALUWaitStates if the second
2610 // is a VALU). Refer to SPG 4.6.12.1. "Requirements for WMMA data hazards" for
2611 // numbers, which depends on the category of the first WMMA.
2612 const int WMMAWaitStates[] = {5, 9, 3, 5, 9, 17, 2};
2613 const int VALUWaitStates[] = {4, 8, 2, 4, 8, 16, 1};
2614 unsigned Category = 0;
2615
2616 auto IsWMMAHazardFn = [MI, TII, &Category, this](const MachineInstr &I) {
2617 if (!TII->isXDLWMMA(MI: I))
2618 return false;
2619
2620 Category = getWMMAHazardInstInCategory(MI: I, TII, SchedModel: TSchedModel, ST);
2621 return hasWMMAToWMMARegOverlap(WMMA: I, MI: *MI);
2622 };
2623
2624 auto IsVALUHazardFn = [MI, TII, &Category, this](const MachineInstr &I) {
2625 if (!TII->isXDLWMMA(MI: I))
2626 return false;
2627
2628 Category = getWMMAHazardInstInCategory(MI: I, TII, SchedModel: TSchedModel, ST);
2629 return hasWMMAToVALURegOverlap(WMMA: I, MI: *MI);
2630 };
2631
2632 int WaitStatesNeeded = -1;
2633 int ExistingVALUs = 0; // Existing number of VALU ops in between.
2634 bool IsLowestRateWMMA = ST.hasGFX125xLowestRateWMMA();
2635
2636 // getWaitStatesSinceVALU checks for a hazard between instruction 'I' and
2637 // 'MI':
2638 // - If a hazard exists: returns the number of VALUs in between and sets
2639 // 'Category' via IsWMMAHazardFn/IsVALUHazardFn for instruction 'I'.
2640 // - If no hazard exists: returns INT_MAX, making WaitStatesNeeded negative,
2641 // so no V_NOP insertion is needed.
2642 if (TII->isXDLWMMA(MI: *MI)) {
2643 // Maximum of MMAWaitStates.
2644 const int WMMAWaitsLimit = IsLowestRateWMMA ? 17 : 9;
2645 ExistingVALUs = getWaitStatesSinceVALU(IsHazard: IsWMMAHazardFn, Limit: WMMAWaitsLimit);
2646 WaitStatesNeeded = WMMAWaitStates[Category] - ExistingVALUs;
2647 } else { // Must be a co-executable VALU.
2648 // Maximum of VALUWaitStates.
2649 const int VALUWaitsLimit = IsLowestRateWMMA ? 16 : 8;
2650 ExistingVALUs = getWaitStatesSinceVALU(IsHazard: IsVALUHazardFn, Limit: VALUWaitsLimit);
2651 WaitStatesNeeded = VALUWaitStates[Category] - ExistingVALUs;
2652 }
2653
2654 return WaitStatesNeeded;
2655}
2656
2657bool GCNHazardRecognizer::hasWMMAToWMMARegOverlap(
2658 const MachineInstr &WMMA, const MachineInstr &MI) const {
2659 Register D0 = TII.getNamedOperand(MI: WMMA, OperandName: AMDGPU::OpName::vdst)->getReg();
2660 Register A1 = TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src0)->getReg();
2661 Register B1 = TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src1)->getReg();
2662
2663 // WMMA0 writes (D0), WMMA1 reads (A1/B1/Idx1).
2664 if (TRI.regsOverlap(RegA: D0, RegB: A1) || TRI.regsOverlap(RegA: D0, RegB: B1))
2665 return true;
2666
2667 if (SIInstrInfo::isSWMMAC(MI)) {
2668 Register Idx1 = TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src2)->getReg();
2669 if (TRI.regsOverlap(RegA: D0, RegB: Idx1))
2670 return true;
2671 }
2672 return false;
2673}
2674
2675bool GCNHazardRecognizer::hasWMMAToVALURegOverlap(
2676 const MachineInstr &WMMA, const MachineInstr &MI) const {
2677 // WMMA writes, VALU reads.
2678 Register D0 = TII.getNamedOperand(MI: WMMA, OperandName: AMDGPU::OpName::vdst)->getReg();
2679 for (const MachineOperand &ValuUse : MI.explicit_uses()) {
2680 if (ValuUse.isReg() && TRI.regsOverlap(RegA: D0, RegB: ValuUse.getReg()))
2681 return true;
2682 }
2683
2684 // WMMA reads or writes, VALU writes.
2685 Register A0 = TII.getNamedOperand(MI: WMMA, OperandName: AMDGPU::OpName::src0)->getReg();
2686 Register B0 = TII.getNamedOperand(MI: WMMA, OperandName: AMDGPU::OpName::src1)->getReg();
2687 SmallVector<Register, 4> WMMARegs({D0, A0, B0});
2688
2689 if (SIInstrInfo::isSWMMAC(MI: WMMA)) {
2690 Register Idx0 = TII.getNamedOperand(MI: WMMA, OperandName: AMDGPU::OpName::src2)->getReg();
2691 WMMARegs.push_back(Elt: Idx0);
2692 }
2693
2694 for (const MachineOperand &ValuDef : MI.defs()) {
2695 Register VDstReg = ValuDef.getReg();
2696 for (Register WMMAReg : WMMARegs) {
2697 if (TRI.regsOverlap(RegA: VDstReg, RegB: WMMAReg))
2698 return true;
2699 }
2700 }
2701 return false;
2702}
2703
2704bool GCNHazardRecognizer::isCoexecutionHazardFor(const MachineInstr &I,
2705 const MachineInstr &MI) const {
2706 // I is the potential WMMA hazard source, MI is the instruction being checked
2707 // for hazard.
2708 if (!TII.isXDLWMMA(MI: I))
2709 return false;
2710
2711 // Dispatch based on MI type
2712 if (TII.isXDLWMMA(MI))
2713 return hasWMMAToWMMARegOverlap(WMMA: I, MI);
2714 if (isCoexecutableVALUInst(MI))
2715 return hasWMMAToVALURegOverlap(WMMA: I, MI);
2716
2717 return false;
2718}
2719
2720bool GCNHazardRecognizer::hasWMMAHazardInLoop(MachineLoop *L, MachineInstr *MI,
2721 bool IncludeSubloops) {
2722 // Scan loop for any WMMA that hazards MI.
2723 // TODO: Avoid full loop scan when WMMA is beyond VALU distance.
2724 for (MachineBasicBlock *MBB : L->getBlocks()) {
2725 if (!IncludeSubloops && MLI->getLoopFor(BB: MBB) != L)
2726 continue;
2727 for (MachineInstr &I : *MBB) {
2728 if (&I == MI)
2729 continue;
2730 if (isCoexecutionHazardFor(I, MI: *MI))
2731 return true;
2732 }
2733 }
2734 return false;
2735}
2736
2737bool GCNHazardRecognizer::tryHoistWMMAVnopsFromLoop(MachineInstr *MI,
2738 int WaitStatesNeeded) {
2739 if (!MLI)
2740 return false;
2741
2742 MachineLoop *L = MLI->getLoopFor(BB: MI->getParent());
2743 if (!L) {
2744 ++NumWMMAHoistingBailed;
2745 return false;
2746 }
2747
2748 // If innermost loop has WMMA hazard, we can't hoist at all
2749 if (hasWMMAHazardInLoop(L, MI)) {
2750 ++NumWMMAHoistingBailed;
2751 return false;
2752 }
2753
2754 // Find outermost loop with no internal hazard
2755 MachineLoop *TargetLoop = L;
2756 while (MachineLoop *Parent = TargetLoop->getParentLoop()) {
2757 if (hasWMMAHazardInLoop(L: Parent, MI, IncludeSubloops: false))
2758 break; // Parent has hazard in its own blocks, stop here
2759 TargetLoop = Parent; // Safe to hoist further out
2760 }
2761
2762 // Need valid preheader to insert V_NOPs
2763 MachineBasicBlock *Preheader = TargetLoop->getLoopPreheader();
2764 if (!Preheader) {
2765 ++NumWMMAHoistingBailed;
2766 return false;
2767 }
2768
2769 LLVM_DEBUG(dbgs() << "WMMA V_NOP Hoisting: Moving " << WaitStatesNeeded
2770 << " V_NOPs from loop to " << printMBBReference(*Preheader)
2771 << "\n");
2772
2773 emitVNops(MBB&: *Preheader, InsertPt: Preheader->getFirstTerminator(), WaitStatesNeeded,
2774 /*IsHoisting=*/true);
2775 NumWMMANopsHoisted += WaitStatesNeeded;
2776 return true;
2777}
2778
2779bool GCNHazardRecognizer::fixWMMACoexecutionHazards(MachineInstr *MI) {
2780 int WaitStatesNeeded = checkWMMACoexecutionHazards(MI);
2781 if (WaitStatesNeeded <= 0)
2782 return false;
2783
2784 if (EnableWMMAVnopHoisting && tryHoistWMMAVnopsFromLoop(MI, WaitStatesNeeded))
2785 return true;
2786
2787 emitVNops(MBB&: *MI->getParent(), InsertPt: MI->getIterator(), WaitStatesNeeded);
2788 return true;
2789}
2790
2791bool GCNHazardRecognizer::fixShift64HighRegBug(MachineInstr *MI) {
2792 if (!ST.hasShift64HighRegBug())
2793 return false;
2794 assert(!ST.hasExtendedWaitCounts());
2795
2796 switch (MI->getOpcode()) {
2797 default:
2798 return false;
2799 case AMDGPU::V_LSHLREV_B64_e64:
2800 case AMDGPU::V_LSHRREV_B64_e64:
2801 case AMDGPU::V_ASHRREV_I64_e64:
2802 break;
2803 }
2804
2805 MachineOperand *Amt = TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src0);
2806 if (!Amt->isReg())
2807 return false;
2808
2809 Register AmtReg = Amt->getReg();
2810 const MachineRegisterInfo &MRI = MF.getRegInfo();
2811 // Check if this is a last VGPR in the allocation block.
2812 if (!TRI.isVGPR(MRI, Reg: AmtReg) || ((AmtReg - AMDGPU::VGPR0) & 7) != 7)
2813 return false;
2814
2815 if (AmtReg != AMDGPU::VGPR255 && MRI.isPhysRegUsed(PhysReg: AmtReg + 1))
2816 return false;
2817
2818 assert(ST.needsAlignedVGPRs());
2819 static_assert(AMDGPU::VGPR0 + 1 == AMDGPU::VGPR1);
2820
2821 const DebugLoc &DL = MI->getDebugLoc();
2822 MachineBasicBlock *MBB = MI->getParent();
2823 MachineOperand *Src1 = TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src1);
2824
2825 // In:
2826 //
2827 // Dst = shiftrev64 Amt, Src1
2828 //
2829 // if Dst!=Src1 then avoid the bug with:
2830 //
2831 // Dst.sub0 = Amt
2832 // Dst = shift64 Dst.sub0, Src1
2833
2834 Register DstReg = MI->getOperand(i: 0).getReg();
2835 if (!Src1->isReg() || Src1->getReg() != DstReg) {
2836 Register DstLo = TRI.getSubReg(Reg: DstReg, Idx: AMDGPU::sub0);
2837 runOnInstruction(
2838 MI: BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstLo).add(MO: *Amt));
2839 Amt->setReg(DstLo);
2840 Amt->setIsKill(true);
2841 return true;
2842 }
2843
2844 bool Overlapped = MI->modifiesRegister(Reg: AmtReg, TRI: &TRI);
2845 Register NewReg;
2846 for (MCRegister Reg : Overlapped ? AMDGPU::VReg_64_Align2RegClass
2847 : AMDGPU::VGPR_32RegClass) {
2848 if (!MI->modifiesRegister(Reg, TRI: &TRI) && !MI->readsRegister(Reg, TRI: &TRI)) {
2849 NewReg = Reg;
2850 break;
2851 }
2852 }
2853
2854 Register NewAmt = Overlapped ? (Register)TRI.getSubReg(Reg: NewReg, Idx: AMDGPU::sub1)
2855 : NewReg;
2856 Register NewAmtLo;
2857
2858 if (Overlapped)
2859 NewAmtLo = TRI.getSubReg(Reg: NewReg, Idx: AMDGPU::sub0);
2860
2861 // Insert a full wait count because found register might be pending a wait.
2862 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT))
2863 .addImm(Val: 0);
2864
2865 // Insert V_SWAP_B32 instruction(s) and run hazard recognizer on them.
2866 if (Overlapped)
2867 runOnInstruction(
2868 MI: BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SWAP_B32), DestReg: NewAmtLo)
2869 .addDef(RegNo: AmtReg - 1)
2870 .addReg(RegNo: AmtReg - 1, Flags: RegState::Undef)
2871 .addReg(RegNo: NewAmtLo, Flags: RegState::Undef));
2872 runOnInstruction(MI: BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SWAP_B32), DestReg: NewAmt)
2873 .addDef(RegNo: AmtReg)
2874 .addReg(RegNo: AmtReg, Flags: RegState::Undef)
2875 .addReg(RegNo: NewAmt, Flags: RegState::Undef));
2876
2877 // Instructions emitted after the current instruction will be processed by the
2878 // parent loop of the hazard recognizer in a natural way.
2879 BuildMI(BB&: *MBB, I: std::next(x: MI->getIterator()), MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SWAP_B32),
2880 DestReg: AmtReg)
2881 .addDef(RegNo: NewAmt)
2882 .addReg(RegNo: NewAmt)
2883 .addReg(RegNo: AmtReg);
2884 if (Overlapped)
2885 BuildMI(BB&: *MBB, I: std::next(x: MI->getIterator()), MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SWAP_B32),
2886 DestReg: AmtReg - 1)
2887 .addDef(RegNo: NewAmtLo)
2888 .addReg(RegNo: NewAmtLo)
2889 .addReg(RegNo: AmtReg - 1);
2890
2891 // Re-running hazard recognizer on the modified instruction is not necessary,
2892 // inserted V_SWAP_B32 has already both read and write new registers so
2893 // hazards related to these register has already been handled.
2894 Amt->setReg(NewAmt);
2895 Amt->setIsKill(false);
2896 // We do not update liveness, so verifier may see it as undef.
2897 Amt->setIsUndef();
2898 if (Overlapped) {
2899 MI->getOperand(i: 0).setReg(NewReg);
2900 Src1->setReg(NewReg);
2901 Src1->setIsKill(false);
2902 Src1->setIsUndef();
2903 }
2904
2905 return true;
2906}
2907
2908int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) const {
2909 int NSAtoVMEMWaitStates = 1;
2910
2911 if (!ST.hasNSAtoVMEMBug())
2912 return 0;
2913
2914 if (!SIInstrInfo::isMUBUF(MI: *MI) && !SIInstrInfo::isMTBUF(MI: *MI))
2915 return 0;
2916
2917 const SIInstrInfo *TII = ST.getInstrInfo();
2918 const auto *Offset = TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::offset);
2919 if (!Offset || (Offset->getImm() & 6) == 0)
2920 return 0;
2921
2922 auto IsHazardFn = [TII](const MachineInstr &I) {
2923 if (!SIInstrInfo::isMIMG(MI: I))
2924 return false;
2925 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc: I.getOpcode());
2926 return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA &&
2927 TII->getInstSizeInBytes(MI: I) >= 16;
2928 };
2929
2930 return NSAtoVMEMWaitStates - getWaitStatesSince(IsHazard: IsHazardFn, Limit: 1);
2931}
2932
2933int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(
2934 MachineInstr *MI) const {
2935 int FPAtomicToDenormModeWaitStates = 3;
2936
2937 if (!ST.hasFPAtomicToDenormModeHazard())
2938 return 0;
2939 assert(!ST.hasExtendedWaitCounts());
2940
2941 if (MI->getOpcode() != AMDGPU::S_DENORM_MODE)
2942 return 0;
2943
2944 auto IsHazardFn = [](const MachineInstr &I) {
2945 if (!SIInstrInfo::isVMEM(MI: I))
2946 return false;
2947 return SIInstrInfo::isFPAtomic(MI: I);
2948 };
2949
2950 auto IsExpiredFn = [](const MachineInstr &MI, int WaitStates) {
2951 if (WaitStates >= 3 || SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
2952 return true;
2953
2954 return SIInstrInfo::isWaitcnt(Opcode: MI.getOpcode());
2955 };
2956
2957 return FPAtomicToDenormModeWaitStates -
2958 ::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn);
2959}
2960
2961int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) const {
2962 assert(SIInstrInfo::isMAI(*MI));
2963
2964 return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI);
2965}
2966
2967int GCNHazardRecognizer::checkMFMAPadding(MachineInstr *MI) const {
2968 // Early exit if no padding is requested.
2969 if (MFMAPaddingRatio == 0)
2970 return 0;
2971
2972 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2973 if (!SIInstrInfo::isMFMA(MI: *MI) || MFI->getOccupancy() < 2)
2974 return 0;
2975
2976 int NeighborMFMALatency = 0;
2977 auto IsNeighboringMFMA = [&NeighborMFMALatency,
2978 this](const MachineInstr &MI) {
2979 if (!SIInstrInfo::isMFMA(MI))
2980 return false;
2981
2982 NeighborMFMALatency = this->getMFMAPipelineWaitStates(MI);
2983 return true;
2984 };
2985
2986 const int MaxMFMAPipelineWaitStates = 16;
2987 int WaitStatesSinceNeighborMFMA =
2988 getWaitStatesSince(IsHazard: IsNeighboringMFMA, Limit: MaxMFMAPipelineWaitStates);
2989
2990 int NeighborMFMAPaddingNeeded =
2991 (NeighborMFMALatency * MFMAPaddingRatio / 100) -
2992 WaitStatesSinceNeighborMFMA;
2993
2994 return std::max(a: 0, b: NeighborMFMAPaddingNeeded);
2995}
2996
2997int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) const {
2998 int WaitStatesNeeded = 0;
2999 unsigned Opc = MI->getOpcode();
3000
3001 auto IsVALUFn = [](const MachineInstr &MI) {
3002 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) || MI.isInlineAsm();
3003 };
3004
3005 if (Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) { // MFMA or v_accvgpr_write
3006 const int LegacyVALUWritesVGPRWaitStates = 2;
3007 const int VALUWritesExecWaitStates = 4;
3008 const int MaxWaitStates = 4;
3009
3010 int WaitStatesNeededForUse = VALUWritesExecWaitStates -
3011 getWaitStatesSinceDef(Reg: AMDGPU::EXEC, IsHazardDef: IsVALUFn, Limit: MaxWaitStates);
3012 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3013
3014 if (WaitStatesNeeded < MaxWaitStates) {
3015 for (const MachineOperand &Use : MI->explicit_uses()) {
3016 const int MaxWaitStates = 2;
3017
3018 if (!Use.isReg() || !TRI.isVGPR(MRI: MF.getRegInfo(), Reg: Use.getReg()))
3019 continue;
3020
3021 int WaitStatesNeededForUse = LegacyVALUWritesVGPRWaitStates -
3022 getWaitStatesSinceDef(Reg: Use.getReg(), IsHazardDef: IsVALUFn, Limit: MaxWaitStates);
3023 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3024
3025 if (WaitStatesNeeded == MaxWaitStates)
3026 break;
3027 }
3028 }
3029 }
3030
3031 for (const MachineOperand &Op : MI->explicit_operands()) {
3032 if (!Op.isReg() || !TRI.isAGPR(MRI: MF.getRegInfo(), Reg: Op.getReg()))
3033 continue;
3034
3035 if (Op.isDef() && Opc != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
3036 continue;
3037
3038 const int MFMAWritesAGPROverlappedSrcABWaitStates = 4;
3039 const int MFMAWritesAGPROverlappedSrcCWaitStates = 2;
3040 const int MFMA4x4WritesAGPRAccVgprReadWaitStates = 4;
3041 const int MFMA16x16WritesAGPRAccVgprReadWaitStates = 10;
3042 const int MFMA32x32WritesAGPRAccVgprReadWaitStates = 18;
3043 const int MFMA4x4WritesAGPRAccVgprWriteWaitStates = 1;
3044 const int MFMA16x16WritesAGPRAccVgprWriteWaitStates = 7;
3045 const int MFMA32x32WritesAGPRAccVgprWriteWaitStates = 15;
3046 const int MaxWaitStates = 18;
3047 Register Reg = Op.getReg();
3048 unsigned HazardDefLatency = 0;
3049
3050 auto IsOverlappedMFMAFn = [Reg, &HazardDefLatency,
3051 this](const MachineInstr &MI) {
3052 if (!SIInstrInfo::isMFMA(MI))
3053 return false;
3054 Register DstReg = MI.getOperand(i: 0).getReg();
3055 if (DstReg == Reg)
3056 return false;
3057 HazardDefLatency =
3058 std::max(a: HazardDefLatency, b: TSchedModel.computeInstrLatency(MI: &MI));
3059 return TRI.regsOverlap(RegA: DstReg, RegB: Reg);
3060 };
3061
3062 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsHazardDef: IsOverlappedMFMAFn,
3063 Limit: MaxWaitStates);
3064 int NeedWaitStates = MFMAWritesAGPROverlappedSrcABWaitStates;
3065 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2);
3066 int OpNo = Op.getOperandNo();
3067 if (OpNo == SrcCIdx) {
3068 NeedWaitStates = MFMAWritesAGPROverlappedSrcCWaitStates;
3069 } else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) {
3070 switch (HazardDefLatency) {
3071 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprReadWaitStates;
3072 break;
3073 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprReadWaitStates;
3074 break;
3075 case 16: [[fallthrough]];
3076 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprReadWaitStates;
3077 break;
3078 }
3079 } else if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
3080 switch (HazardDefLatency) {
3081 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprWriteWaitStates;
3082 break;
3083 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprWriteWaitStates;
3084 break;
3085 case 16: [[fallthrough]];
3086 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprWriteWaitStates;
3087 break;
3088 }
3089 }
3090
3091 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
3092 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3093
3094 if (WaitStatesNeeded == MaxWaitStates)
3095 return WaitStatesNeeded; // Early exit.
3096
3097 auto IsAccVgprWriteFn = [Reg, this](const MachineInstr &MI) {
3098 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
3099 return false;
3100 Register DstReg = MI.getOperand(i: 0).getReg();
3101 return TRI.regsOverlap(RegA: Reg, RegB: DstReg);
3102 };
3103
3104 const int AccVGPRWriteMFMAReadSrcCWaitStates = 1;
3105 const int AccVGPRWriteMFMAReadSrcABWaitStates = 3;
3106 const int AccVGPRWriteAccVgprReadWaitStates = 3;
3107 NeedWaitStates = AccVGPRWriteMFMAReadSrcABWaitStates;
3108 if (OpNo == SrcCIdx)
3109 NeedWaitStates = AccVGPRWriteMFMAReadSrcCWaitStates;
3110 else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64)
3111 NeedWaitStates = AccVGPRWriteAccVgprReadWaitStates;
3112
3113 WaitStatesNeededForUse = NeedWaitStates -
3114 getWaitStatesSinceDef(Reg, IsHazardDef: IsAccVgprWriteFn, Limit: MaxWaitStates);
3115 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3116
3117 if (WaitStatesNeeded == MaxWaitStates)
3118 return WaitStatesNeeded; // Early exit.
3119 }
3120
3121 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
3122 const int MFMA4x4ReadSrcCAccVgprWriteWaitStates = 0;
3123 const int MFMA16x16ReadSrcCAccVgprWriteWaitStates = 5;
3124 const int MFMA32x32ReadSrcCAccVgprWriteWaitStates = 13;
3125 const int MaxWaitStates = 13;
3126 Register DstReg = MI->getOperand(i: 0).getReg();
3127 unsigned HazardDefLatency = 0;
3128
3129 auto IsSrcCMFMAFn = [DstReg, &HazardDefLatency,
3130 this](const MachineInstr &MI) {
3131 if (!SIInstrInfo::isMFMA(MI))
3132 return false;
3133 Register Reg = TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src2)->getReg();
3134 HazardDefLatency =
3135 std::max(a: HazardDefLatency, b: TSchedModel.computeInstrLatency(MI: &MI));
3136 return TRI.regsOverlap(RegA: Reg, RegB: DstReg);
3137 };
3138
3139 int WaitStatesSince = getWaitStatesSince(IsHazard: IsSrcCMFMAFn, Limit: MaxWaitStates);
3140 int NeedWaitStates;
3141 switch (HazardDefLatency) {
3142 case 2: NeedWaitStates = MFMA4x4ReadSrcCAccVgprWriteWaitStates;
3143 break;
3144 case 8: NeedWaitStates = MFMA16x16ReadSrcCAccVgprWriteWaitStates;
3145 break;
3146 case 16: [[fallthrough]];
3147 default: NeedWaitStates = MFMA32x32ReadSrcCAccVgprWriteWaitStates;
3148 break;
3149 }
3150
3151 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSince;
3152 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3153 }
3154
3155 // Pad neighboring MFMA with noops for better inter-wave performance.
3156 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: checkMFMAPadding(MI));
3157
3158 return WaitStatesNeeded;
3159}
3160
3161static int
3162GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates(int NumPasses,
3163 bool IsGFX950) {
3164 // xdl def cycles | gfx940 | gfx950
3165 // 2 pass | 3 4
3166 // 4 pass | 5 6
3167 // 8 pass | 9 10
3168 // 16 pass | 17 18
3169 return NumPasses + 1 + IsGFX950;
3170}
3171
3172static int
3173GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates(int NumPasses,
3174 bool IsGFX950) {
3175 // xdl def cycles | gfx940 | gfx950
3176 // 2 pass | 3 3
3177 // 4 pass | 5 6
3178 // 8 pass | 9 10
3179 // 16 pass | 17 18
3180 return NumPasses + 1 + (NumPasses != 2 && IsGFX950);
3181}
3182
3183static int
3184GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates(int NumPasses) {
3185 // 2 pass -> 2
3186 // 4 pass -> 4
3187 // 8 pass -> 8
3188 // 16 pass -> 16
3189 return NumPasses;
3190}
3191
3192static int
3193GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses) {
3194 // 2 pass -> 4
3195 // 4 pass -> 6
3196 // 8 pass -> 10
3197 // 16 pass -> 18
3198 return NumPasses + 2;
3199}
3200
3201static int GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses,
3202 bool IsGFX950) {
3203 // xdl def cycles | gfx942 | gfx950
3204 // 2 pass | 5 5
3205 // 4 pass | 7 8
3206 // 8 pass | 11 12
3207 // 16 pass | 19 20
3208 return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
3209}
3210
3211int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) const {
3212 int WaitStatesNeeded = 0;
3213 unsigned Opc = MI->getOpcode();
3214
3215 auto IsLegacyVALUFn = [](const MachineInstr &MI) {
3216 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
3217 !SIInstrInfo::isMFMA(MI);
3218 };
3219
3220 auto IsLegacyVALUNotDotFn = [](const MachineInstr &MI) {
3221 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
3222 !SIInstrInfo::isMFMA(MI) && !SIInstrInfo::isDOT(MI);
3223 };
3224
3225 if (!SIInstrInfo::isMFMA(MI: *MI))
3226 return WaitStatesNeeded;
3227
3228 const int VALUWritesExecWaitStates = 4;
3229 int WaitStatesNeededForUse = VALUWritesExecWaitStates -
3230 getWaitStatesSinceDef(Reg: AMDGPU::EXEC, IsHazardDef: IsLegacyVALUFn,
3231 Limit: VALUWritesExecWaitStates);
3232 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3233
3234 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2);
3235
3236 // Loop for both DGEMM and S/HGEMM 2nd instruction.
3237 for (const MachineOperand &Use : MI->explicit_uses()) {
3238 const int LegacyVALUNotDotWritesVGPRWaitStates = 2;
3239 const int SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates = 2;
3240 const int SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates = 8;
3241 const int SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates = 16;
3242 const int SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates = 3;
3243 const int SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates = 9;
3244 const int SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates = 17;
3245 const int DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 9;
3246 const int GFX950_DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 17;
3247 const int DMFMA4x4WritesVGPROverlappedSrcCWaitStates = 4;
3248 const int SMFMA4x4WritesVGPROverlappedSrcABWaitStates = 5;
3249 const int SMFMA16x16WritesVGPROverlappedSrcABWaitStates = 11;
3250 const int SMFMA32x32WritesVGPROverlappedSrcABWaitStates = 19;
3251 const int DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates = 6;
3252 const int DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 11;
3253 const int GFX950_DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 19;
3254 const int DMFMA4x4WritesVGPRFullSrcCWaitStates = 4;
3255 const int GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates = 2;
3256 const int MaxWaitStates =
3257 GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(
3258 NumPasses: 16, IsGFX950: ST.hasGFX950Insts());
3259
3260 if (!Use.isReg())
3261 continue;
3262 Register Reg = Use.getReg();
3263 bool FullReg;
3264 const MachineInstr *MI1;
3265
3266 auto IsOverlappedMFMAFn = [Reg, &FullReg, &MI1,
3267 this](const MachineInstr &MI) {
3268 if (!SIInstrInfo::isMFMA(MI))
3269 return false;
3270 Register DstReg = MI.getOperand(i: 0).getReg();
3271 FullReg = (DstReg == Reg);
3272 MI1 = &MI;
3273 return TRI.regsOverlap(RegA: DstReg, RegB: Reg);
3274 };
3275
3276 WaitStatesNeededForUse = LegacyVALUNotDotWritesVGPRWaitStates -
3277 getWaitStatesSinceDef(Reg, IsHazardDef: IsLegacyVALUNotDotFn, Limit: MaxWaitStates);
3278 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3279
3280 int NumWaitStates =
3281 getWaitStatesSinceDef(Reg, IsHazardDef: IsOverlappedMFMAFn, Limit: MaxWaitStates);
3282 if (NumWaitStates == std::numeric_limits<int>::max())
3283 continue;
3284
3285 int OpNo = Use.getOperandNo();
3286 unsigned Opc1 = MI1->getOpcode();
3287 int NeedWaitStates = 0;
3288 if (OpNo == SrcCIdx) {
3289 if (!SIInstrInfo::isDGEMM(Opcode: Opc) &&
3290 (!ST.hasGFX940Insts() && SIInstrInfo::isDGEMM(Opcode: Opc1))) {
3291 NeedWaitStates = 0;
3292 } else if (FullReg) {
3293 if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
3294 Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) &&
3295 (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
3296 Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64))
3297 NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates;
3298 else if (ST.hasGFX940Insts() &&
3299 TSchedModel.computeInstrLatency(MI: MI1) == 2)
3300 NeedWaitStates = GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates;
3301 } else {
3302 switch (Opc1) {
3303 case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
3304 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
3305 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
3306 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
3307 if (!TII.isXDL(MI: *MI))
3308 NeedWaitStates =
3309 ST.hasGFX950Insts()
3310 ? GFX950_DMFMA16x16WritesVGPROverlappedSrcCWaitStates
3311 : DMFMA16x16WritesVGPROverlappedSrcCWaitStates;
3312 break;
3313 case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
3314 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
3315 if (!TII.isXDL(MI: *MI))
3316 NeedWaitStates = DMFMA4x4WritesVGPROverlappedSrcCWaitStates;
3317 break;
3318 default:
3319 int NumPasses = TSchedModel.computeInstrLatency(MI: MI1);
3320 if (ST.hasGFX940Insts()) {
3321 if (TII.isXDL(MI: *MI) && !TII.isXDL(MI: *MI1))
3322 break;
3323
3324 NeedWaitStates =
3325 TII.isXDL(MI: *MI1)
3326 ? (TII.isXDL(MI: *MI)
3327 ? GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates(
3328 NumPasses, IsGFX950: ST.hasGFX950Insts())
3329 : GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates(
3330 NumPasses, IsGFX950: ST.hasGFX950Insts()))
3331 : GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates(
3332 NumPasses);
3333 break;
3334 }
3335
3336 switch (NumPasses) {
3337 case 2:
3338 NeedWaitStates =
3339 SIInstrInfo::isDGEMM(Opcode: Opc)
3340 ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates
3341 : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates;
3342 break;
3343 case 8:
3344 NeedWaitStates =
3345 SIInstrInfo::isDGEMM(Opcode: Opc)
3346 ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates
3347 : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates;
3348 break;
3349 case 16:
3350 NeedWaitStates =
3351 SIInstrInfo::isDGEMM(Opcode: Opc)
3352 ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates
3353 : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates;
3354 break;
3355 default:
3356 llvm_unreachable("unexpected number of passes");
3357 }
3358 }
3359 }
3360 } else {
3361 switch (Opc1) {
3362 case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
3363 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
3364 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
3365 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
3366 NeedWaitStates =
3367 ST.hasGFX950Insts()
3368 ? GFX950_DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates
3369 : DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates;
3370 break;
3371 case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
3372 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
3373 NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates;
3374 break;
3375 default:
3376 int NumPasses = TSchedModel.computeInstrLatency(MI: MI1);
3377
3378 if (ST.hasGFX940Insts()) {
3379 NeedWaitStates =
3380 TII.isXDL(MI: *MI1)
3381 ? GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(
3382 NumPasses, IsGFX950: ST.hasGFX950Insts())
3383 : GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates(
3384 NumPasses);
3385 break;
3386 }
3387
3388 switch (NumPasses) {
3389 case 2:
3390 NeedWaitStates = SMFMA4x4WritesVGPROverlappedSrcABWaitStates;
3391 break;
3392 case 4:
3393 llvm_unreachable("unexpected number of passes for mfma");
3394 case 8:
3395 NeedWaitStates = SMFMA16x16WritesVGPROverlappedSrcABWaitStates;
3396 break;
3397 case 16:
3398 default:
3399 NeedWaitStates = SMFMA32x32WritesVGPROverlappedSrcABWaitStates;
3400 }
3401 }
3402 }
3403 assert(NeedWaitStates <= MaxWaitStates &&
3404 "hazard requirement exceeds the scan window");
3405 if (WaitStatesNeeded >= NeedWaitStates)
3406 continue;
3407
3408 WaitStatesNeededForUse = NeedWaitStates - NumWaitStates;
3409 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3410
3411 if (WaitStatesNeeded == MaxWaitStates)
3412 break;
3413 }
3414
3415 // Pad neighboring MFMA with noops for better inter-wave performance.
3416 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: checkMFMAPadding(MI));
3417
3418 return WaitStatesNeeded;
3419}
3420
3421int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) const {
3422 // On gfx90a+ relevant hazards are checked in checkMAIVALUHazards()
3423 if (!ST.hasMAIInsts() || ST.hasGFX90AInsts())
3424 return 0;
3425
3426 int WaitStatesNeeded = 0;
3427
3428 auto IsAccVgprReadFn = [](const MachineInstr &MI) {
3429 return MI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64;
3430 };
3431
3432 for (const MachineOperand &Op : MI->explicit_uses()) {
3433 if (!Op.isReg() || !TRI.isVGPR(MRI: MF.getRegInfo(), Reg: Op.getReg()))
3434 continue;
3435
3436 Register Reg = Op.getReg();
3437
3438 const int AccVgprReadLdStWaitStates = 2;
3439 const int VALUWriteAccVgprRdWrLdStDepVALUWaitStates = 1;
3440 const int MaxWaitStates = 2;
3441
3442 int WaitStatesNeededForUse = AccVgprReadLdStWaitStates -
3443 getWaitStatesSinceDef(Reg, IsHazardDef: IsAccVgprReadFn, Limit: MaxWaitStates);
3444 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3445
3446 if (WaitStatesNeeded == MaxWaitStates)
3447 return WaitStatesNeeded; // Early exit.
3448
3449 auto IsVALUAccVgprRdWrCheckFn = [Reg, this](const MachineInstr &MI) {
3450 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 &&
3451 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
3452 return false;
3453 auto IsVALUFn = [](const MachineInstr &MI) {
3454 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
3455 !SIInstrInfo::isMAI(MI);
3456 };
3457 return getWaitStatesSinceDef(Reg, IsHazardDef: IsVALUFn, Limit: 2 /*MaxWaitStates*/) <
3458 std::numeric_limits<int>::max();
3459 };
3460
3461 WaitStatesNeededForUse = VALUWriteAccVgprRdWrLdStDepVALUWaitStates -
3462 getWaitStatesSince(IsHazard: IsVALUAccVgprRdWrCheckFn, Limit: MaxWaitStates);
3463 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3464 }
3465
3466 return WaitStatesNeeded;
3467}
3468
3469int GCNHazardRecognizer::checkPermlaneHazards(MachineInstr *MI) const {
3470 assert(!ST.hasVcmpxPermlaneHazard() &&
3471 "this is a different vcmpx+permlane hazard");
3472 const SIRegisterInfo *TRI = ST.getRegisterInfo();
3473 const SIInstrInfo *TII = ST.getInstrInfo();
3474
3475 auto IsVCmpXWritesExecFn = [TII, TRI](const MachineInstr &MI) {
3476 return isVCmpXWritesExec(TII: *TII, TRI: *TRI, MI);
3477 };
3478
3479 auto IsVALUFn = [](const MachineInstr &MI) {
3480 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true);
3481 };
3482
3483 const int VCmpXWritesExecWaitStates = 4;
3484 const int VALUWritesVDstWaitStates = 2;
3485 int WaitStatesNeeded = 0;
3486
3487 for (const MachineOperand &Op : MI->explicit_uses()) {
3488 if (!Op.isReg() || !TRI->isVGPR(MRI: MF.getRegInfo(), Reg: Op.getReg()))
3489 continue;
3490 Register Reg = Op.getReg();
3491
3492 int WaitStatesSinceDef =
3493 VALUWritesVDstWaitStates -
3494 getWaitStatesSinceDef(Reg, IsHazardDef: IsVALUFn,
3495 /*MaxWaitStates=*/Limit: VALUWritesVDstWaitStates);
3496 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesSinceDef);
3497 if (WaitStatesNeeded >= VALUWritesVDstWaitStates)
3498 break;
3499 }
3500
3501 int VCmpXHazardWaits =
3502 VCmpXWritesExecWaitStates -
3503 getWaitStatesSince(IsHazard: IsVCmpXWritesExecFn, Limit: VCmpXWritesExecWaitStates);
3504
3505 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: VCmpXHazardWaits);
3506 return WaitStatesNeeded;
3507}
3508
3509static int GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(int NumPasses) {
3510 // 2 pass -> 4
3511 // 4 pass -> 6
3512 // 8 pass -> 10
3513 // 16 pass -> 18
3514 return NumPasses + 2;
3515}
3516
3517static int GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(int NumPasses,
3518 bool IsGFX950) {
3519 // xdl def cycles | gfx942 | gfx950
3520 // 2 pass | 5 5
3521 // 4 pass | 7 8
3522 // 8 pass | 11 12
3523 // 16 pass | 19 20
3524 return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
3525}
3526
3527static int GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses,
3528 bool IsGFX950) {
3529 // xdl def cycles | gfx942 | gfx950
3530 // 2 pass | 5 5
3531 // 4 pass | 7 8
3532 // 8 pass | 11 12
3533 // 16 pass | 19 20
3534 return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
3535}
3536
3537static int GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses) {
3538 // 2 pass -> 4
3539 // 4 pass -> 6
3540 // 8 pass -> 10
3541 // 16 pass -> 18
3542 return NumPasses + 2;
3543}
3544
3545int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const {
3546 if (!ST.hasGFX90AInsts())
3547 return 0;
3548
3549 auto IsDGEMMFn = [](const MachineInstr &MI) -> bool {
3550 return SIInstrInfo::isDGEMM(Opcode: MI.getOpcode());
3551 };
3552
3553 // This is checked in checkMAIHazards90A()
3554 if (SIInstrInfo::isMFMA(MI: *MI))
3555 return 0;
3556
3557 const MachineRegisterInfo &MRI = MF.getRegInfo();
3558
3559 int WaitStatesNeeded = 0;
3560
3561 bool IsMem = SIInstrInfo::isVMEM(MI: *MI) || SIInstrInfo::isDS(MI: *MI);
3562 bool IsMemOrExport = IsMem || SIInstrInfo::isEXP(MI: *MI);
3563 bool IsVALU = SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true);
3564
3565 const MachineInstr *MFMA = nullptr;
3566 unsigned Reg;
3567 auto IsMFMAWriteFn = [&Reg, &MFMA, this](const MachineInstr &MI) {
3568 if (!SIInstrInfo::isMFMA(MI) ||
3569 !TRI.regsOverlap(RegA: MI.getOperand(i: 0).getReg(), RegB: Reg))
3570 return false;
3571 MFMA = &MI;
3572 return true;
3573 };
3574
3575 const MachineInstr *DOT = nullptr;
3576 auto IsDotWriteFn = [&Reg, &DOT, this](const MachineInstr &MI) {
3577 if (!SIInstrInfo::isDOT(MI) ||
3578 !TRI.regsOverlap(RegA: MI.getOperand(i: 0).getReg(), RegB: Reg))
3579 return false;
3580 DOT = &MI;
3581 return true;
3582 };
3583
3584 bool DGEMMAfterVALUWrite = false;
3585 auto IsDGEMMHazard = [&DGEMMAfterVALUWrite, this](const MachineInstr &MI) {
3586 // Found DGEMM on reverse traversal to def.
3587 if (SIInstrInfo::isDGEMM(Opcode: MI.getOpcode()))
3588 DGEMMAfterVALUWrite = true;
3589
3590 // Only hazard if register is defined by a VALU and a DGEMM is found after
3591 // after the def.
3592 if (!TII.isVALU(MI, /*AllowLDSDMA=*/true) || !DGEMMAfterVALUWrite)
3593 return false;
3594
3595 return true;
3596 };
3597
3598 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opcode: MI->getOpcode(),
3599 Name: AMDGPU::OpName::src2);
3600
3601 if (IsMemOrExport || IsVALU) {
3602 const int SMFMA4x4WriteVgprVALUMemExpReadWaitStates = 5;
3603 const int SMFMA16x16WriteVgprVALUMemExpReadWaitStates = 11;
3604 const int SMFMA32x32WriteVgprVALUMemExpReadWaitStates = 19;
3605 const int DMFMA4x4WriteVgprMemExpReadWaitStates = 9;
3606 const int DMFMA16x16WriteVgprMemExpReadWaitStates = 18;
3607 const int DMFMA4x4WriteVgprVALUReadWaitStates = 6;
3608 const int DMFMA16x16WriteVgprVALUReadWaitStates = 11;
3609 const int GFX950_DMFMA16x16WriteVgprVALUReadWaitStates = 19;
3610 const int DotWriteSameDotReadSrcAB = 3;
3611 const int DotWriteDifferentVALURead = 3;
3612 const int DMFMABetweenVALUWriteVMEMRead = 2;
3613 const int MaxWaitStates =
3614 GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(NumPasses: 16,
3615 IsGFX950: ST.hasGFX950Insts());
3616
3617 for (const MachineOperand &Use : MI->explicit_uses()) {
3618 if (!Use.isReg())
3619 continue;
3620 Reg = Use.getReg();
3621
3622 DOT = nullptr;
3623 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsHazardDef: IsDotWriteFn,
3624 Limit: MaxWaitStates);
3625 if (DOT) {
3626 int NeedWaitStates = 0;
3627 if (DOT->getOpcode() == MI->getOpcode()) {
3628 if (&Use - &MI->getOperand(i: 0) != SrcCIdx)
3629 NeedWaitStates = DotWriteSameDotReadSrcAB;
3630 } else {
3631 NeedWaitStates = DotWriteDifferentVALURead;
3632 }
3633
3634 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
3635 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3636 }
3637
3638 // Workaround for HW data hazard bug observed only in GFX90A. When there
3639 // is a DGEMM instruction in-between a VALU and a VMEM instruction it
3640 // causes the SQ to incorrectly not insert two wait states between the two
3641 // instructions needed to avoid data hazard.
3642 if (IsMem && ST.hasGFX90AInsts() && !ST.hasGFX940Insts()) {
3643 DGEMMAfterVALUWrite = false;
3644 if (TRI.isVectorRegister(MRI, Reg)) {
3645 int WaitStatesNeededForUse =
3646 DMFMABetweenVALUWriteVMEMRead -
3647 getWaitStatesSinceDef(Reg, IsHazardDef: IsDGEMMHazard,
3648 Limit: DMFMABetweenVALUWriteVMEMRead);
3649
3650 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3651 }
3652 }
3653
3654 MFMA = nullptr;
3655 WaitStatesSinceDef =
3656 getWaitStatesSinceDef(Reg, IsHazardDef: IsMFMAWriteFn, Limit: MaxWaitStates);
3657 if (!MFMA)
3658 continue;
3659
3660 unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MI: MFMA);
3661 int NumPasses = HazardDefLatency;
3662 int NeedWaitStates = MaxWaitStates;
3663
3664 if (SIInstrInfo::isDGEMM(Opcode: MFMA->getOpcode())) {
3665 switch (HazardDefLatency) {
3666 case 4:
3667 NeedWaitStates = IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates
3668 : DMFMA4x4WriteVgprVALUReadWaitStates;
3669 break;
3670 case 8:
3671 case 16:
3672 NeedWaitStates =
3673 IsMemOrExport
3674 ? DMFMA16x16WriteVgprMemExpReadWaitStates
3675 : (ST.hasGFX950Insts()
3676 ? GFX950_DMFMA16x16WriteVgprVALUReadWaitStates
3677 : DMFMA16x16WriteVgprVALUReadWaitStates);
3678 break;
3679 default:
3680 llvm_unreachable("unexpected dgemm");
3681 }
3682 } else if (ST.hasGFX940Insts()) {
3683 NeedWaitStates =
3684 TII.isXDL(MI: *MFMA)
3685 ? GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(
3686 NumPasses, IsGFX950: ST.hasGFX950Insts())
3687 : GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates(
3688 NumPasses);
3689 } else {
3690 switch (HazardDefLatency) {
3691 case 2:
3692 NeedWaitStates = SMFMA4x4WriteVgprVALUMemExpReadWaitStates;
3693 break;
3694 case 8:
3695 NeedWaitStates = SMFMA16x16WriteVgprVALUMemExpReadWaitStates;
3696 break;
3697 case 16:
3698 NeedWaitStates = SMFMA32x32WriteVgprVALUMemExpReadWaitStates;
3699 break;
3700 default:
3701 llvm_unreachable("unexpected number of passes for mfma");
3702 }
3703 }
3704
3705 assert(NeedWaitStates <= MaxWaitStates &&
3706 "hazard requirement exceeds the scan window");
3707 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
3708 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3709
3710 if (WaitStatesNeeded == MaxWaitStates)
3711 break;
3712 }
3713 }
3714
3715 unsigned Opc = MI->getOpcode();
3716 const int DMFMAToFMA64WaitStates = 2;
3717 if ((Opc == AMDGPU::V_FMA_F64_e64 ||
3718 Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64 ||
3719 Opc == AMDGPU::V_FMAC_F64_dpp) &&
3720 WaitStatesNeeded < DMFMAToFMA64WaitStates) {
3721 int WaitStatesNeededForUse = DMFMAToFMA64WaitStates -
3722 getWaitStatesSince(IsHazard: IsDGEMMFn, Limit: DMFMAToFMA64WaitStates);
3723 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3724 }
3725
3726 if (!IsVALU && !IsMemOrExport)
3727 return WaitStatesNeeded;
3728
3729 for (const MachineOperand &Def : MI->defs()) {
3730 const int SMFMA4x4WriteVgprVALUWawWaitStates = 5;
3731 const int SMFMA16x16WriteVgprVALUWawWaitStates = 11;
3732 const int SMFMA32x32WriteVgprVALUWawWaitStates = 19;
3733 const int SMFMA4x4ReadVgprVALUWarWaitStates = 1;
3734 const int GFX940_XDL4PassReadVgprVALUWarWaitStates = 3;
3735 const int SMFMA16x16ReadVgprVALUWarWaitStates = 7;
3736 const int SMFMA32x32ReadVgprVALUWarWaitStates = 15;
3737 const int DMFMA4x4WriteVgprVALUWriteWaitStates = 6;
3738 const int DMFMA16x16WriteVgprVALUWriteWaitStates = 11;
3739 const int DotWriteDifferentVALUWrite = 3;
3740 const int MaxWaitStates =
3741 GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(NumPasses: 16, IsGFX950: ST.hasGFX950Insts());
3742 const int MaxWarWaitStates = 15;
3743
3744 Reg = Def.getReg();
3745
3746 DOT = nullptr;
3747 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsHazardDef: IsDotWriteFn,
3748 Limit: MaxWaitStates);
3749 if (DOT && DOT->getOpcode() != MI->getOpcode())
3750 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: DotWriteDifferentVALUWrite -
3751 WaitStatesSinceDef);
3752
3753 MFMA = nullptr;
3754 WaitStatesSinceDef =
3755 getWaitStatesSinceDef(Reg, IsHazardDef: IsMFMAWriteFn, Limit: MaxWaitStates);
3756 if (MFMA) {
3757 int NeedWaitStates = MaxWaitStates;
3758 int NumPasses = TSchedModel.computeInstrLatency(MI: MFMA);
3759
3760 if (SIInstrInfo::isDGEMM(Opcode: MFMA->getOpcode())) {
3761 switch (NumPasses) {
3762 case 4:
3763 NeedWaitStates = DMFMA4x4WriteVgprVALUWriteWaitStates;
3764 break;
3765 case 8:
3766 case 16:
3767 NeedWaitStates = DMFMA16x16WriteVgprVALUWriteWaitStates;
3768 break;
3769 default:
3770 llvm_unreachable("unexpected number of cycles for dgemm");
3771 }
3772 } else if (ST.hasGFX940Insts()) {
3773 NeedWaitStates =
3774 TII.isXDL(MI: *MFMA)
3775 ? GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(
3776 NumPasses, IsGFX950: ST.hasGFX950Insts())
3777 : GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(NumPasses);
3778 } else {
3779 switch (NumPasses) {
3780 case 2:
3781 NeedWaitStates = SMFMA4x4WriteVgprVALUWawWaitStates;
3782 break;
3783 case 8:
3784 NeedWaitStates = SMFMA16x16WriteVgprVALUWawWaitStates;
3785 break;
3786 case 16:
3787 NeedWaitStates = SMFMA32x32WriteVgprVALUWawWaitStates;
3788 break;
3789 default:
3790 llvm_unreachable("Unexpected number of passes for mfma");
3791 }
3792 }
3793
3794 assert(NeedWaitStates <= MaxWaitStates &&
3795 "hazard requirement exceeds the scan window");
3796 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
3797 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3798
3799 if (WaitStatesNeeded == MaxWaitStates)
3800 break;
3801 }
3802
3803 auto IsSMFMAReadAsCFn = [&Reg, &MFMA, this](const MachineInstr &MI) {
3804 if (!SIInstrInfo::isMFMA(MI) || SIInstrInfo::isDGEMM(Opcode: MI.getOpcode()) ||
3805 !MI.readsRegister(Reg, TRI: &TRI))
3806 return false;
3807
3808 if (ST.hasGFX940Insts() && !TII.isXDL(MI))
3809 return false;
3810
3811 const MachineOperand *SrcC =
3812 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
3813 assert(SrcC);
3814 if (!SrcC->isReg() || !TRI.regsOverlap(RegA: SrcC->getReg(), RegB: Reg))
3815 return false;
3816
3817 MFMA = &MI;
3818 return true;
3819 };
3820
3821 MFMA = nullptr;
3822 int WaitStatesSinceUse = getWaitStatesSince(IsHazard: IsSMFMAReadAsCFn,
3823 Limit: MaxWarWaitStates);
3824 if (!MFMA)
3825 continue;
3826
3827 unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MI: MFMA);
3828 int NeedWaitStates = MaxWaitStates;
3829 switch (HazardDefLatency) {
3830 case 2: NeedWaitStates = SMFMA4x4ReadVgprVALUWarWaitStates;
3831 break;
3832 case 4: assert(ST.hasGFX940Insts());
3833 NeedWaitStates = GFX940_XDL4PassReadVgprVALUWarWaitStates;
3834 break;
3835 case 8: NeedWaitStates = SMFMA16x16ReadVgprVALUWarWaitStates;
3836 break;
3837 case 16: [[fallthrough]];
3838 default: NeedWaitStates = SMFMA32x32ReadVgprVALUWarWaitStates;
3839 break;
3840 }
3841
3842 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceUse;
3843 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3844 }
3845
3846 return WaitStatesNeeded;
3847}
3848
3849bool GCNHazardRecognizer::ShouldPreferAnother(SUnit *SU) const {
3850 if (!SU->isInstr())
3851 return false;
3852
3853 const MachineInstr *MAI = nullptr;
3854
3855 auto IsMFMAFn = [&MAI](const MachineInstr &MI) {
3856 MAI = nullptr;
3857 if (SIInstrInfo::isMFMA(MI))
3858 MAI = &MI;
3859 return MAI != nullptr;
3860 };
3861
3862 MachineInstr *MI = SU->getInstr();
3863 if (IsMFMAFn(*MI)) {
3864 int W = getWaitStatesSince(IsHazard: IsMFMAFn, Limit: 16);
3865 if (MAI)
3866 return W < (int)TSchedModel.computeInstrLatency(MI: MAI);
3867 }
3868
3869 return false;
3870}
3871
3872// Adjust global offsets for instructions bundled with S_GETPC_B64 after
3873// insertion of a new instruction.
3874static void updateGetPCBundle(MachineInstr *NewMI) {
3875 if (!NewMI->isBundled())
3876 return;
3877
3878 // Find start of bundle.
3879 auto I = NewMI->getIterator();
3880 while (I->isBundledWithPred())
3881 I--;
3882 if (I->isBundle())
3883 I++;
3884
3885 // Bail if this is not an S_GETPC bundle.
3886 if (I->getOpcode() != AMDGPU::S_GETPC_B64)
3887 return;
3888
3889 // Update offsets of any references in the bundle.
3890 const unsigned NewBytes = 4;
3891 assert(NewMI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
3892 "Unexpected instruction insertion in bundle");
3893 auto NextMI = std::next(x: NewMI->getIterator());
3894 auto End = NewMI->getParent()->end();
3895 while (NextMI != End && NextMI->isBundledWithPred()) {
3896 for (auto &Operand : NextMI->operands()) {
3897 if (Operand.isGlobal())
3898 Operand.setOffset(Operand.getOffset() + NewBytes);
3899 }
3900 NextMI++;
3901 }
3902}
3903
3904bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) {
3905 if (!ST.hasVALUMaskWriteHazard())
3906 return false;
3907 assert(!ST.hasExtendedWaitCounts());
3908
3909 if (!ST.isWave64())
3910 return false;
3911
3912 const bool IsSALU = SIInstrInfo::isSALU(MI: *MI);
3913 const bool IsVALU = SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true);
3914 if (!IsSALU && !IsVALU)
3915 return false;
3916
3917 // The hazard sequence is three instructions:
3918 // 1. VALU reads SGPR as mask
3919 // 2. VALU/SALU writes SGPR
3920 // 3. VALU/SALU reads SGPR
3921 // The hazard can expire if the distance between 2 and 3 is sufficient,
3922 // or (2) is VALU and (3) is SALU.
3923 // In practice this happens <10% of the time, hence always assume the hazard
3924 // exists if (1) and (2) are present to avoid searching all SGPR reads.
3925
3926 const SIRegisterInfo *TRI = ST.getRegisterInfo();
3927 const MachineRegisterInfo &MRI = MF.getRegInfo();
3928
3929 auto IgnoreableSGPR = [](const Register Reg) {
3930 switch (Reg) {
3931 case AMDGPU::EXEC:
3932 case AMDGPU::EXEC_LO:
3933 case AMDGPU::EXEC_HI:
3934 case AMDGPU::M0:
3935 case AMDGPU::SGPR_NULL:
3936 case AMDGPU::SGPR_NULL64:
3937 case AMDGPU::SCC:
3938 return true;
3939 default:
3940 return false;
3941 }
3942 };
3943 auto IsVCC = [](const Register Reg) {
3944 return Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::VCC_HI;
3945 };
3946
3947 struct StateType {
3948 SmallSet<Register, 2> HazardSGPRs;
3949
3950 static unsigned getHashValue(const StateType &State) {
3951 return hash_combine_range(R: State.HazardSGPRs);
3952 }
3953 static bool isEqual(const StateType &LHS, const StateType &RHS) {
3954 return LHS.HazardSGPRs == RHS.HazardSGPRs;
3955 }
3956 };
3957
3958 SmallVector<const MachineInstr *> WaitInstrs;
3959 StateType InitialState;
3960
3961 // Look for SGPR write.
3962 MachineOperand *HazardDef = nullptr;
3963 for (MachineOperand &Op : MI->all_defs()) {
3964 Register Reg = Op.getReg();
3965 if (IgnoreableSGPR(Reg))
3966 continue;
3967 if (!IsVCC(Reg)) {
3968 if (Op.isImplicit())
3969 continue;
3970 if (!TRI->isSGPRReg(MRI, Reg))
3971 continue;
3972 }
3973
3974 HazardDef = &Op;
3975 break;
3976 }
3977
3978 if (!HazardDef)
3979 return false;
3980
3981 // Setup to track writes to individual SGPRs
3982 const Register HazardReg = HazardDef->getReg();
3983 if (AMDGPU::SReg_32RegClass.contains(Reg: HazardReg)) {
3984 InitialState.HazardSGPRs.insert(V: HazardReg);
3985 } else {
3986 assert(AMDGPU::SReg_64RegClass.contains(HazardReg));
3987 InitialState.HazardSGPRs.insert(V: TRI->getSubReg(Reg: HazardReg, Idx: AMDGPU::sub0));
3988 InitialState.HazardSGPRs.insert(V: TRI->getSubReg(Reg: HazardReg, Idx: AMDGPU::sub1));
3989 }
3990
3991 auto IsHazardFn = [&](StateType &State, const MachineInstr &I) {
3992 if (State.HazardSGPRs.empty())
3993 return HazardExpired;
3994
3995 switch (I.getOpcode()) {
3996 case AMDGPU::V_ADDC_U32_e32:
3997 case AMDGPU::V_ADDC_U32_dpp:
3998 case AMDGPU::V_CNDMASK_B16_t16_e32:
3999 case AMDGPU::V_CNDMASK_B16_fake16_e32:
4000 case AMDGPU::V_CNDMASK_B16_t16_dpp:
4001 case AMDGPU::V_CNDMASK_B16_fake16_dpp:
4002 case AMDGPU::V_CNDMASK_B32_e32:
4003 case AMDGPU::V_CNDMASK_B32_dpp:
4004 case AMDGPU::V_DIV_FMAS_F32_e64:
4005 case AMDGPU::V_DIV_FMAS_F64_e64:
4006 case AMDGPU::V_SUBB_U32_e32:
4007 case AMDGPU::V_SUBB_U32_dpp:
4008 case AMDGPU::V_SUBBREV_U32_e32:
4009 case AMDGPU::V_SUBBREV_U32_dpp: {
4010 // These implicitly read VCC as mask source.
4011 return IsVCC(HazardReg) ? HazardFound : NoHazardFound;
4012 }
4013 case AMDGPU::V_ADDC_U32_e64:
4014 case AMDGPU::V_ADDC_U32_e64_dpp:
4015 case AMDGPU::V_CNDMASK_B16_t16_e64:
4016 case AMDGPU::V_CNDMASK_B16_fake16_e64:
4017 case AMDGPU::V_CNDMASK_B16_t16_e64_dpp:
4018 case AMDGPU::V_CNDMASK_B16_fake16_e64_dpp:
4019 case AMDGPU::V_CNDMASK_B32_e64:
4020 case AMDGPU::V_CNDMASK_B32_e64_dpp:
4021 case AMDGPU::V_SUBB_U32_e64:
4022 case AMDGPU::V_SUBB_U32_e64_dpp:
4023 case AMDGPU::V_SUBBREV_U32_e64:
4024 case AMDGPU::V_SUBBREV_U32_e64_dpp: {
4025 // Only check mask register overlaps.
4026 const MachineOperand *SSRCOp = TII.getNamedOperand(MI: I, OperandName: AMDGPU::OpName::src2);
4027 assert(SSRCOp);
4028 bool Result = TRI->regsOverlap(RegA: SSRCOp->getReg(), RegB: HazardReg);
4029 return Result ? HazardFound : NoHazardFound;
4030 }
4031 default:
4032 return NoHazardFound;
4033 }
4034 };
4035
4036 auto UpdateStateFn = [&](StateType &State, const MachineInstr &I) {
4037 // Update tracking of SGPR writes.
4038 for (auto &Op : I.all_defs()) {
4039 Register Reg = Op.getReg();
4040 if (IgnoreableSGPR(Reg))
4041 continue;
4042 if (!IsVCC(Reg)) {
4043 if (Op.isImplicit())
4044 continue;
4045 if (!TRI->isSGPRReg(MRI, Reg))
4046 continue;
4047 }
4048
4049 // Stop tracking any SGPRs with writes on the basis that they will
4050 // already have an appropriate wait inserted afterwards.
4051 SmallVector<Register, 2> Found;
4052 for (Register SGPR : State.HazardSGPRs) {
4053 if (Reg == SGPR || TRI->regsOverlap(RegA: Reg, RegB: SGPR))
4054 Found.push_back(Elt: SGPR);
4055 }
4056 for (Register SGPR : Found)
4057 State.HazardSGPRs.erase(V: SGPR);
4058 }
4059 };
4060
4061 // Check for hazard
4062 if (!hasHazard<StateType>(InitialState, IsHazard: IsHazardFn, UpdateState: UpdateStateFn,
4063 InitialMBB: MI->getParent(),
4064 InitialI: std::next(x: MI->getReverseIterator())))
4065 return false;
4066
4067 // Compute counter mask
4068 unsigned DepCtr =
4069 IsVALU ? (IsVCC(HazardReg) ? AMDGPU::DepCtr::encodeFieldVaVcc(VaVcc: 0, STI: ST)
4070 : AMDGPU::DepCtr::encodeFieldVaSdst(VaSdst: 0, STI: ST))
4071 : AMDGPU::DepCtr::encodeFieldSaSdst(SaSdst: 0, STI: ST);
4072
4073 // Add s_waitcnt_depctr after SGPR write.
4074 auto NextMI = std::next(x: MI->getIterator());
4075 auto NewMI = BuildMI(BB&: *MI->getParent(), I: NextMI, MIMD: MI->getDebugLoc(),
4076 MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4077 .addImm(Val: DepCtr);
4078
4079 // SALU write may be s_getpc in a bundle.
4080 updateGetPCBundle(NewMI);
4081
4082 return true;
4083}
4084
4085static bool ensureEntrySetPrio(MachineFunction *MF, int Priority,
4086 const SIInstrInfo &TII) {
4087 MachineBasicBlock &EntryMBB = MF->front();
4088 if (EntryMBB.begin() != EntryMBB.end()) {
4089 auto &EntryMI = *EntryMBB.begin();
4090 if (EntryMI.getOpcode() == AMDGPU::S_SETPRIO &&
4091 EntryMI.getOperand(i: 0).getImm() >= Priority)
4092 return false;
4093 }
4094
4095 BuildMI(BB&: EntryMBB, I: EntryMBB.begin(), MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_SETPRIO))
4096 .addImm(Val: Priority);
4097 return true;
4098}
4099
4100bool GCNHazardRecognizer::fixRequiredExportPriority(MachineInstr *MI) {
4101 if (!ST.hasRequiredExportPriority())
4102 return false;
4103
4104 // Assume the following shader types will never have exports,
4105 // and avoid adding or adjusting S_SETPRIO.
4106 MachineBasicBlock *MBB = MI->getParent();
4107 MachineFunction *MF = MBB->getParent();
4108 auto CC = MF->getFunction().getCallingConv();
4109 switch (CC) {
4110 case CallingConv::AMDGPU_CS:
4111 case CallingConv::AMDGPU_CS_Chain:
4112 case CallingConv::AMDGPU_CS_ChainPreserve:
4113 case CallingConv::AMDGPU_KERNEL:
4114 return false;
4115 default:
4116 break;
4117 }
4118
4119 const int MaxPriority = 3;
4120 const int NormalPriority = 2;
4121 const int PostExportPriority = 0;
4122
4123 auto It = MI->getIterator();
4124 switch (MI->getOpcode()) {
4125 case AMDGPU::S_ENDPGM:
4126 case AMDGPU::S_ENDPGM_SAVED:
4127 case AMDGPU::S_ENDPGM_ORDERED_PS_DONE:
4128 case AMDGPU::SI_RETURN_TO_EPILOG:
4129 // Ensure shader with calls raises priority at entry.
4130 // This ensures correct priority if exports exist in callee.
4131 if (MF->getFrameInfo().hasCalls())
4132 return ensureEntrySetPrio(MF, Priority: NormalPriority, TII);
4133 return false;
4134 case AMDGPU::S_SETPRIO: {
4135 // Raise minimum priority unless in workaround.
4136 auto &PrioOp = MI->getOperand(i: 0);
4137 int Prio = PrioOp.getImm();
4138 bool InWA = (Prio == PostExportPriority) &&
4139 (It != MBB->begin() && TII.isEXP(MI: *std::prev(x: It)));
4140 if (InWA || Prio >= NormalPriority)
4141 return false;
4142 PrioOp.setImm(std::min(a: Prio + NormalPriority, b: MaxPriority));
4143 return true;
4144 }
4145 default:
4146 if (!TII.isEXP(MI: *MI))
4147 return false;
4148 break;
4149 }
4150
4151 // Check entry priority at each export (as there will only be a few).
4152 // Note: amdgpu_gfx can only be a callee, so defer to caller setprio.
4153 bool Changed = false;
4154 if (CC != CallingConv::AMDGPU_Gfx && CC != CallingConv::AMDGPU_Gfx_WholeWave)
4155 Changed = ensureEntrySetPrio(MF, Priority: NormalPriority, TII);
4156
4157 auto NextMI = std::next(x: It);
4158 bool EndOfShader = false;
4159 if (NextMI != MBB->end()) {
4160 // Only need WA at end of sequence of exports.
4161 if (TII.isEXP(MI: *NextMI))
4162 return Changed;
4163 // Assume appropriate S_SETPRIO after export means WA already applied.
4164 if (NextMI->getOpcode() == AMDGPU::S_SETPRIO &&
4165 NextMI->getOperand(i: 0).getImm() == PostExportPriority)
4166 return Changed;
4167 EndOfShader = NextMI->getOpcode() == AMDGPU::S_ENDPGM;
4168 }
4169
4170 const DebugLoc &DL = MI->getDebugLoc();
4171
4172 // Lower priority.
4173 BuildMI(BB&: *MBB, I: NextMI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_SETPRIO))
4174 .addImm(Val: PostExportPriority);
4175
4176 if (!EndOfShader) {
4177 // Wait for exports to complete.
4178 BuildMI(BB&: *MBB, I: NextMI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_EXPCNT))
4179 .addReg(RegNo: AMDGPU::SGPR_NULL)
4180 .addImm(Val: 0);
4181 }
4182
4183 BuildMI(BB&: *MBB, I: NextMI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_NOP)).addImm(Val: 0);
4184 BuildMI(BB&: *MBB, I: NextMI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_NOP)).addImm(Val: 0);
4185
4186 if (!EndOfShader) {
4187 // Return to normal (higher) priority.
4188 BuildMI(BB&: *MBB, I: NextMI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_SETPRIO))
4189 .addImm(Val: NormalPriority);
4190 }
4191
4192 return true;
4193}
4194
4195bool GCNHazardRecognizer::fixGetRegWaitIdle(MachineInstr *MI) {
4196 if (!isSGetReg(Opcode: MI->getOpcode()))
4197 return false;
4198
4199 const SIInstrInfo *TII = ST.getInstrInfo();
4200 switch (getHWReg(TII, RegInstr: *MI)) {
4201 default:
4202 return false;
4203 case AMDGPU::Hwreg::ID_STATUS:
4204 case AMDGPU::Hwreg::ID_STATE_PRIV:
4205 case AMDGPU::Hwreg::ID_EXCP_FLAG_PRIV:
4206 case AMDGPU::Hwreg::ID_EXCP_FLAG_USER:
4207 break;
4208 }
4209
4210 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
4211 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4212 .addImm(Val: 0);
4213 return true;
4214}
4215
4216bool GCNHazardRecognizer::fixDsAtomicAsyncBarrierArriveB64(MachineInstr *MI) {
4217 if (MI->getOpcode() != AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
4218 return false;
4219
4220 const SIInstrInfo *TII = ST.getInstrInfo();
4221 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
4222 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4223 .addImm(Val: AMDGPU::DepCtr::encodeFieldVmVsrc(VmVsrc: 0, STI: ST));
4224 BuildMI(BB&: *MI->getParent(), I: std::next(x: MI->getIterator()), MIMD: MI->getDebugLoc(),
4225 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4226 .addImm(Val: AMDGPU::DepCtr::encodeFieldVmVsrc(VmVsrc: 0, STI: ST));
4227
4228 return true;
4229}
4230
4231bool GCNHazardRecognizer::fixScratchBaseForwardingHazard(MachineInstr *MI) {
4232 // No reason to check this in pre-RA scheduling, SGPRs have to be allocated
4233 // for hazard to trigger.
4234 if (!isHazardRecognizerMode())
4235 return false;
4236
4237 const SIRegisterInfo *TRI = ST.getRegisterInfo();
4238 const SIInstrInfo *TII = ST.getInstrInfo();
4239 // Hazard expires after 10 SGPR writes by SALU or 8 SGPR writes by VALU.
4240 const int FlatScrBaseWaitStates = 10;
4241
4242 bool ReadsFlatScrLo =
4243 MI->readsRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_LO, TRI);
4244 bool ReadsFlatScrHi =
4245 MI->readsRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_HI, TRI);
4246 if (isSGetReg(Opcode: MI->getOpcode())) {
4247 switch (getHWReg(TII, RegInstr: *MI)) {
4248 default:
4249 break;
4250 case AMDGPU::Hwreg::ID_FLAT_SCR_LO:
4251 ReadsFlatScrLo = true;
4252 break;
4253 case AMDGPU::Hwreg::ID_FLAT_SCR_HI:
4254 ReadsFlatScrHi = true;
4255 break;
4256 }
4257 }
4258
4259 const MachineRegisterInfo &MRI = MF.getRegInfo();
4260
4261 auto IsRegDefHazard = [&](Register Reg) -> bool {
4262 DenseSet<const MachineBasicBlock *> Visited;
4263 auto IsHazardFn = [TRI, Reg](const MachineInstr &MI) {
4264 return MI.modifiesRegister(Reg, TRI);
4265 };
4266
4267 // This literally abuses the idea of waitstates. Instead of waitstates it
4268 // returns 1 for SGPR written and 0 otherwise.
4269 auto IsSGPRDef = [TII, TRI, &MRI](const MachineInstr &MI) -> unsigned {
4270 if (!TII->isSALU(MI) && !TII->isVALU(MI, /*AllowLDSDMA=*/true))
4271 return 0;
4272 for (const MachineOperand &MO : MI.all_defs()) {
4273 if (TRI->isSGPRReg(MRI, Reg: MO.getReg()))
4274 return 1;
4275 }
4276 return 0;
4277 };
4278
4279 auto IsExpiredFn = [=](const MachineInstr &MI, int SgprWrites) {
4280 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR) {
4281 unsigned Wait = MI.getOperand(i: 0).getImm();
4282 if (AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: Wait) == 0 &&
4283 AMDGPU::DepCtr::decodeFieldVaSdst(Encoded: Wait) == 0)
4284 return true;
4285 }
4286 return SgprWrites >= FlatScrBaseWaitStates;
4287 };
4288
4289 return ::getWaitStatesSince(
4290 IsHazard: IsHazardFn, MBB: MI->getParent(), I: std::next(x: MI->getReverseIterator()),
4291 WaitStates: 0, IsExpired: IsExpiredFn, Visited, GetNumWaitStates: IsSGPRDef) < FlatScrBaseWaitStates;
4292 };
4293
4294 if ((!ReadsFlatScrLo || MRI.isConstantPhysReg(PhysReg: AMDGPU::SGPR102) ||
4295 !IsRegDefHazard(AMDGPU::SGPR102)) &&
4296 (!ReadsFlatScrHi || MRI.isConstantPhysReg(PhysReg: AMDGPU::SGPR103) ||
4297 !IsRegDefHazard(AMDGPU::SGPR103)))
4298 return false;
4299
4300 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
4301 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4302 .addImm(Val: AMDGPU::DepCtr::encodeFieldVaSdst(
4303 Encoded: AMDGPU::DepCtr::encodeFieldSaSdst(SaSdst: 0, STI: ST), VaSdst: 0));
4304 return true;
4305}
4306
4307bool GCNHazardRecognizer::fixSetRegMode(MachineInstr *MI) {
4308 if (!isSSetReg(Opcode: MI->getOpcode()) ||
4309 MI->getOperand(i: 1).getImm() != AMDGPU::Hwreg::ID_MODE)
4310 return false;
4311
4312 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_NOP_e32));
4313 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_NOP_e32));
4314 return true;
4315}
4316
4317bool GCNHazardRecognizer::fixTDM(MachineInstr *MI) {
4318 auto IsTDM = [&](const MachineInstr &MI) -> bool {
4319 return SIInstrInfo::usesTENSOR_CNT(MI) &&
4320 MI.getOpcode() != AMDGPU::S_WAIT_TENSORCNT;
4321 };
4322
4323 if (!IsTDM(*MI))
4324 return false;
4325
4326 auto IsExpiredFn = [](const MachineInstr &MI, int) {
4327 if (MI.getOpcode() != AMDGPU::S_WAIT_TENSORCNT)
4328 return false;
4329 return MI.getOperand(i: 0).getImm() <= 10;
4330 };
4331
4332 if (::getWaitStatesSince(IsHazard: IsTDM, MI, IsExpired: IsExpiredFn) ==
4333 std::numeric_limits<int>::max())
4334 return false;
4335
4336 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
4337 MCID: TII.get(Opcode: AMDGPU::S_WAIT_TENSORCNT))
4338 .addImm(Val: 10);
4339 return true;
4340}
4341