1//===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements hazard recognizers for scheduling on GCN processors.
10//
11//===----------------------------------------------------------------------===//
12
13#include "GCNHazardRecognizer.h"
14#include "AMDGPUTargetMachine.h"
15#include "AMDGPUWaitcntUtils.h"
16#include "GCNSubtarget.h"
17#include "SIMachineFunctionInfo.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/CodeGen/MachineFrameInfo.h"
20#include "llvm/CodeGen/MachineFunction.h"
21#include "llvm/CodeGen/MachineInstrBuilder.h"
22#include "llvm/CodeGen/ScheduleDAG.h"
23#include "llvm/Support/Debug.h"
24#include "llvm/TargetParser/AMDGPUTargetParser.h"
25
26using namespace llvm;
27
28#define DEBUG_TYPE "gcn-hazard-recognizer"
29// Opt-in debug type for the per-candidate co-execution slot traces, which are
30// far too noisy for the normal debug output. Pass both types to get everything.
31#define DEBUG_TYPE_VERBOSE "gcn-hazard-recognizer-verbose"
32
33STATISTIC(NumWMMANopsHoisted,
34 "Number of WMMA hazard V_NOPs hoisted from loops");
35STATISTIC(NumWMMAHoistingBailed,
36 "Number of WMMA hazards where V_NOP hoisting was not possible");
37
38namespace {
39
40struct MFMAPaddingRatioParser : public cl::parser<unsigned> {
41 MFMAPaddingRatioParser(cl::Option &O) : cl::parser<unsigned>(O) {}
42
43 bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) {
44 if (Arg.getAsInteger(Radix: 0, Result&: Value))
45 return O.error(Message: "'" + Arg + "' value invalid for uint argument!");
46
47 if (Value > 100)
48 return O.error(Message: "'" + Arg + "' value must be in the range [0, 100]!");
49
50 return false;
51 }
52};
53
54} // end anonymous namespace
55
56static cl::opt<unsigned, false, MFMAPaddingRatioParser>
57 MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(Val: 0), cl::Hidden,
58 cl::desc("Fill a percentage of the latency between "
59 "neighboring MFMA with s_nops."));
60
61//===----------------------------------------------------------------------===//
62// Hazard Recognizer Implementation
63//===----------------------------------------------------------------------===//
64
65static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF,
66 const GCNSubtarget &ST);
67
68GCNHazardRecognizer::GCNHazardRecognizer(
69 const MachineFunction &MF, GCNHazardRecognizer::OperatingMode Mode,
70 MachineLoopInfo *MLI)
71 : Mode(Mode), CurrCycleInstr(nullptr), MF(MF),
72 ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()),
73 TRI(TII.getRegisterInfo()), TSchedModel(TII.getSchedModel()), MLI(MLI),
74 ClauseUses(TRI.getNumRegUnits()), ClauseDefs(TRI.getNumRegUnits()) {
75 MaxLookAhead = MF.getRegInfo().isPhysRegUsed(PhysReg: AMDGPU::AGPR0) ? 19 : 5;
76 RunLdsBranchVmemWARHazardFixup = shouldRunLdsBranchVmemWARHazardFixup(MF, ST);
77 LLVM_DEBUG({
78 if (isPreRA())
79 dbgs() << " PreRA hazard recognizer: " << MF.getName() << "\n";
80 });
81}
82
83GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF,
84 MachineLoopInfo *MLI)
85 : GCNHazardRecognizer(MF, OperatingMode::PostRA, MLI) {}
86
87GCNHazardRecognizer::~GCNHazardRecognizer() {
88 // Dump any active co-execution window that did not complete naturally
89 // (e.g. region ended before the window expired).
90 LLVM_DEBUG({
91 if (CurrentCoExecStage.has_value()) {
92 unsigned Stage = *CurrentCoExecStage;
93 if (Stage < AMDGPU::MaxCoExecStages)
94 CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
95 dbgs() << " CoExec window ended at stage " << Stage << ":\n";
96 dumpCoExecWindow();
97 }
98 });
99}
100
101void GCNHazardRecognizer::Reset() {
102 EmittedInstrs.clear();
103 EmittedVALUInstrs.clear();
104 HasPendingWMMACoexecHazard = false;
105 if (isSchedulerMode())
106 schedulerReset();
107}
108
109void GCNHazardRecognizer::schedulerReset() {
110 LLVM_DEBUG({
111 if (CurrentCoExecStage.has_value() || CyclesUntilTRANS > 0 ||
112 CyclesUntilVALU > 0)
113 dbgs() << " Scheduler Reset: clearing co-exec window, TRANS="
114 << CyclesUntilTRANS << ", VALU=" << CyclesUntilVALU << "\n";
115 });
116 CurrentCoExecStage = std::nullopt;
117 CoExecWindowStartCycle = 0;
118 CyclesUntilTRANS = 0;
119 CyclesUntilVALU = 0;
120 ActiveCoExecInfo = AMDGPU::CoExecInfo();
121 CoExecWindowLog.fill(u: '.');
122}
123
124void GCNHazardRecognizer::dumpCoExecWindow() const {
125 unsigned W = ActiveCoExecInfo.TotalWindow;
126 if (W == 0)
127 return;
128
129 // Print the stage numbers row.
130 dbgs() << " Stages: ";
131 for (unsigned I = 0; I < W; ++I)
132 dbgs() << I % 10 << ' ';
133 dbgs() << '\n';
134
135 // Print the pattern row.
136 dbgs() << " Slots: ";
137 for (unsigned I = 0; I < W; ++I)
138 dbgs() << ActiveCoExecInfo.Pattern[I] << ' ';
139 dbgs() << '\n';
140
141 // Print the scheduled row.
142 dbgs() << " Scheduled: ";
143 for (unsigned I = 0; I < W; ++I)
144 dbgs() << CoExecWindowLog[I] << ' ';
145 dbgs() << '\n';
146}
147
148void GCNHazardRecognizer::schedulerAdvanceCycle() {
149 // Record what happened at the current stage of the co-exec window.
150 if (CurrentCoExecStage.has_value()) {
151 unsigned Stage = *CurrentCoExecStage;
152 if (Stage < AMDGPU::MaxCoExecStages) {
153 if (CurrCycleInstr)
154 CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
155 else
156 CoExecWindowLog[Stage] = '-';
157 }
158 }
159
160 LLVM_DEBUG({
161 bool HasState = CurrentCoExecStage.has_value() || CyclesUntilTRANS > 0 ||
162 CyclesUntilVALU > 0;
163 if (HasState) {
164 dbgs() << " Scheduler AdvanceCycle:";
165 if (CurrentCoExecStage.has_value()) {
166 unsigned Stage = *CurrentCoExecStage;
167 unsigned Next = Stage + 1;
168 if (Next >= ActiveCoExecInfo.TotalWindow)
169 dbgs() << " stage " << Stage << "->expired";
170 else
171 dbgs() << " stage " << Stage << "->" << Next;
172 }
173 if (CyclesUntilTRANS > 0)
174 dbgs() << " TRANS=" << CyclesUntilTRANS << "->"
175 << (CyclesUntilTRANS - 1);
176 if (CyclesUntilVALU > 0)
177 dbgs() << " VALU=" << CyclesUntilVALU << "->" << (CyclesUntilVALU - 1);
178 dbgs() << "\n";
179 }
180 });
181
182 // Decrement hazard counters.
183 if (CyclesUntilTRANS > 0)
184 --CyclesUntilTRANS;
185 if (CyclesUntilVALU > 0)
186 --CyclesUntilVALU;
187
188 // Advance WMMA co-execution window.
189 if (CurrentCoExecStage.has_value()) {
190 unsigned Stage = *CurrentCoExecStage + 1;
191 if (Stage >= ActiveCoExecInfo.TotalWindow) {
192 // Window expired.
193 LLVM_DEBUG({
194 dbgs() << " CoExec window complete:\n";
195 dumpCoExecWindow();
196 });
197 CurrentCoExecStage = std::nullopt;
198 } else {
199 CurrentCoExecStage = Stage;
200 }
201 }
202}
203
204bool GCNHazardRecognizer::hasCoExecWindowModel() const {
205 // The co-execution slot patterns returned by getCoExecInfo() are derived from
206 // gfx1250 timings, so the window model is restricted to gfx1250 for now.
207 // gfx1251 and gfx12.5-generic report the same co-execution hazard features
208 // but have different WMMA latencies, so they need their own slot patterns
209 // before they can be modeled here.
210 if (ST.hasWMMACoexecutionHazards() && ST.hasTransCoexecutionHazard() &&
211 AMDGPU::isGFX1250(STI: ST))
212 return true;
213
214 if (ST.hasGFX950Insts() &&
215 AMDGPU::getSchedStrategy(CLOpts: ST.getCLOpts(), F: MF.getFunction()) == "coexec")
216 return true;
217
218 return false;
219}
220
221void GCNHazardRecognizer::updateWMMAWindowState(const MachineInstr &MI) {
222 if (!hasCoExecWindowModel())
223 return;
224
225 if (!SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI) &&
226 !SIInstrInfo::isMFMA(MI))
227 return;
228
229 // If a previous window was still active, dump it before starting a new one.
230 // Record the current stage (filled by this new WMMA) before dumping.
231 LLVM_DEBUG({
232 if (CurrentCoExecStage.has_value()) {
233 unsigned Stage = *CurrentCoExecStage;
234 if (Stage < AMDGPU::MaxCoExecStages)
235 CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
236 dbgs() << " CoExec window interrupted at stage " << Stage << ":\n";
237 dumpCoExecWindow();
238 }
239 });
240
241 // Start a new co-execution window.
242 ActiveCoExecInfo = AMDGPU::getCoExecInfo(MI, TII);
243 CurrentCoExecStage = 0;
244 CoExecWindowLog.fill(u: '.');
245
246 LLVM_DEBUG(dbgs() << " WMMA window started: " << ActiveCoExecInfo.Pattern
247 << " (window=" << ActiveCoExecInfo.TotalWindow << ")\n"
248 << " " << MI);
249}
250
251void GCNHazardRecognizer::updateTRANSState(const MachineInstr &MI) {
252 if (!hasCoExecWindowModel())
253 return;
254 if (!SIInstrInfo::isTRANS(MI))
255 return;
256
257 // Back-to-back TRANS instructions have a 1-cycle hazard.
258 // This is checked via checkTRANSHazard() and does not create a co-exec
259 // window. The TRANS shadow slot allows anything except TRANS and
260 // multi-cycle VALU.
261 // Set to 2: bumpCycle advances to the next pick's cycle (decrementing
262 // by 1 via AdvanceCycle) before the next instruction's hazard check, so
263 // the counter is observed at 1 there. That 1-cycle stall lets the
264 // strategy pick a non-TRANS, non-multi-cycle-VALU candidate to fill the
265 // shadow slot.
266 CyclesUntilTRANS = 2;
267 LLVM_DEBUG(dbgs() << " TRANS hazard set: CyclesUntilTRANS=2\n");
268}
269
270void GCNHazardRecognizer::updateMultiCycleVALUState(const MachineInstr &MI) {
271 if (!hasCoExecWindowModel())
272 return;
273 // Multi-cycle VALU (CVT, etc.) blocks subsequent VALU for repeat rate cycles.
274 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
275 return;
276
277 // Skip WMMA, MFMA, and TRANS - they have their own tracking.
278 if (SIInstrInfo::isWMMA(MI) || SIInstrInfo::isSWMMAC(MI) ||
279 SIInstrInfo::isMFMA(MI) || SIInstrInfo::isTRANS(MI))
280 return;
281
282 unsigned RepeatRate = TII.getRepeatRate(MI);
283 if (RepeatRate > 1) {
284 // bumpCycle's AdvanceCycle decrements once before the next pick's
285 // hazard check (same convention as CyclesUntilTRANS), so to expose
286 // RepeatRate-1 cycles of shadow we must seed with RepeatRate.
287 CyclesUntilVALU = RepeatRate;
288 LLVM_DEBUG(dbgs() << " Multi-cycle VALU: repeat=" << RepeatRate
289 << ", CyclesUntilVALU=" << CyclesUntilVALU << "\n");
290 }
291}
292
293AMDGPU::CoExecMaskT
294GCNHazardRecognizer::getCoExecMaskForMI(const MachineInstr &MI,
295 const SIInstrInfo &TII) {
296 return AMDGPU::getCoExecMask(F: AMDGPU::classifyFlavor(MI, SII: TII));
297}
298
299unsigned GCNHazardRecognizer::checkTRANSHazard(const MachineInstr &MI) const {
300 if (!CyclesUntilTRANS)
301 return 0;
302
303 // Only TRANS and multi-cycle VALU are blocked by the TRANS shadow.
304 if (SIInstrInfo::isTRANS(MI))
305 return CyclesUntilTRANS;
306
307 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
308 !SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI) &&
309 TII.getRepeatRate(MI) > 1)
310 return CyclesUntilTRANS;
311
312 return 0;
313}
314
315unsigned
316GCNHazardRecognizer::checkMultiCycleVALUHazard(const MachineInstr &MI) const {
317 if (!CyclesUntilVALU)
318 return 0;
319
320 // Multi-cycle VALU blocks anything on the VALU pipe - VALU, WMMA, SWMMAC,
321 // and TRANS - for RepeatRate-1 cycles. Only off-pipe instructions (MEM,
322 // SALU, control) can fill the shadow.
323 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
324 !SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI) &&
325 !SIInstrInfo::isTRANS(MI))
326 return 0;
327
328 return CyclesUntilVALU;
329}
330
331unsigned
332GCNHazardRecognizer::checkWMMACoexecSlot(const MachineInstr &MI) const {
333 // No hazard if not in a WMMA window.
334 if (!CurrentCoExecStage.has_value())
335 return 0;
336
337 unsigned Stage = *CurrentCoExecStage;
338 AMDGPU::CoExecMaskT InstMask = getCoExecMaskForMI(MI, TII);
339 unsigned StallCycles = ActiveCoExecInfo.getStallCycles(InstMask, Stage);
340
341 // No stall required if the instruction can co-execute at the current stage.
342 if (StallCycles == 0)
343 return 0;
344
345 // Stall for the required number of cycles until the next allowed stage.
346 unsigned NextStage = Stage + StallCycles;
347 if (NextStage < ActiveCoExecInfo.TotalWindow) {
348 DEBUG_WITH_TYPE(
349 DEBUG_TYPE_VERBOSE,
350 dbgs() << " CoExec stall: stage=" << Stage << "("
351 << AMDGPU::getStageTypeName(ActiveCoExecInfo.getType(Stage))
352 << ") mask=" << AMDGPU::getCoExecMaskName(InstMask)
353 << " -> stall " << StallCycles << " (next allowed=" << NextStage
354 << ")\n"
355 << " " << MI);
356 return StallCycles;
357 }
358
359 // No compatible slot in window - stall until window ends.
360 DEBUG_WITH_TYPE(
361 DEBUG_TYPE_VERBOSE,
362 dbgs() << " CoExec stall: stage=" << Stage << "("
363 << AMDGPU::getStageTypeName(ActiveCoExecInfo.getType(Stage))
364 << ") mask=" << AMDGPU::getCoExecMaskName(InstMask) << " -> stall "
365 << StallCycles << " (window ends)\n"
366 << " " << MI);
367 return StallCycles;
368}
369
370unsigned
371GCNHazardRecognizer::checkMultiShadowHazard(const MachineInstr &MI) const {
372 // This models a VALU caught in both a WMMA and a TRANS shadow.
373 if (!hasCoExecWindowModel())
374 return 0;
375
376 // No hazard if not in a WMMA window.
377 if (!CurrentCoExecStage.has_value())
378 return 0;
379
380 if (!CyclesUntilTRANS)
381 return 0;
382
383 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false))
384 return 0;
385
386 // We have a VALU instruction that is under both a TRANS and WMMA shadow.
387 // We need to wait for at least one to clear.
388
389 unsigned LookAheadStage = *CurrentCoExecStage + CyclesUntilTRANS;
390 AMDGPU::CoExecMaskT InstMask = getCoExecMaskForMI(MI, TII);
391 return CyclesUntilTRANS +
392 ActiveCoExecInfo.getStallCycles(InstMask, Stage: LookAheadStage);
393}
394
395void GCNHazardRecognizer::schedulerEmitInstruction(MachineInstr *MI) {
396 LLVM_DEBUG({
397 bool InWindow = CurrentCoExecStage.has_value();
398 bool HasActiveState =
399 InWindow || CyclesUntilTRANS > 0 || CyclesUntilVALU > 0;
400 if (HasActiveState) {
401 if (InWindow) {
402 unsigned Stage = *CurrentCoExecStage;
403 dbgs() << " Stage " << Stage << "("
404 << AMDGPU::getStageTypeName(ActiveCoExecInfo.getType(Stage))
405 << ") Emit ["
406 << AMDGPU::getCoExecMaskName(getCoExecMaskForMI(*MI, TII))
407 << "]: " << *MI;
408 } else {
409 dbgs() << " Emit ["
410 << AMDGPU::getCoExecMaskName(getCoExecMaskForMI(*MI, TII))
411 << "]: " << *MI;
412 }
413 }
414 });
415 DEBUG_WITH_TYPE(DEBUG_TYPE_VERBOSE, {
416 bool HasActiveState = CurrentCoExecStage.has_value() ||
417 CyclesUntilTRANS > 0 || CyclesUntilVALU > 0;
418 if (!HasActiveState)
419 dbgs() << " Emit ["
420 << AMDGPU::getCoExecMaskName(getCoExecMaskForMI(*MI, TII))
421 << "]: " << *MI;
422 });
423 updateWMMAWindowState(MI: *MI);
424 updateTRANSState(MI: *MI);
425 updateMultiCycleVALUState(MI: *MI);
426}
427
428void GCNHazardRecognizer::EmitInstruction(SUnit *SU) {
429 EmitInstruction(MI: SU->getInstr());
430}
431
432void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) {
433 CurrCycleInstr = MI;
434 if (isSchedulerMode())
435 schedulerEmitInstruction(MI);
436}
437
438static bool isDivFMas(unsigned Opcode) {
439 return Opcode == AMDGPU::V_DIV_FMAS_F32_e64 || Opcode == AMDGPU::V_DIV_FMAS_F64_e64;
440}
441
442static bool isSGetReg(unsigned Opcode) {
443 return Opcode == AMDGPU::S_GETREG_B32 || Opcode == AMDGPU::S_GETREG_B32_const;
444}
445
446static bool isSSetReg(unsigned Opcode) {
447 switch (Opcode) {
448 case AMDGPU::S_SETREG_B32:
449 case AMDGPU::S_SETREG_B32_mode:
450 case AMDGPU::S_SETREG_IMM32_B32:
451 case AMDGPU::S_SETREG_IMM32_B32_mode:
452 return true;
453 }
454 return false;
455}
456
457static bool isRWLane(unsigned Opcode) {
458 return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32;
459}
460
461static bool isRFE(unsigned Opcode) {
462 return Opcode == AMDGPU::S_RFE_B64;
463}
464
465static bool isSMovRel(unsigned Opcode) {
466 switch (Opcode) {
467 case AMDGPU::S_MOVRELS_B32:
468 case AMDGPU::S_MOVRELS_B64:
469 case AMDGPU::S_MOVRELD_B32:
470 case AMDGPU::S_MOVRELD_B64:
471 return true;
472 default:
473 return false;
474 }
475}
476
477static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII,
478 const MachineInstr &MI) {
479 if (TII.isAlwaysGDS(Opcode: MI.getOpcode()))
480 return true;
481
482 switch (MI.getOpcode()) {
483 case AMDGPU::S_SENDMSG:
484 case AMDGPU::S_SENDMSGHALT:
485 case AMDGPU::S_TTRACEDATA:
486 return true;
487 // These DS opcodes don't support GDS.
488 case AMDGPU::DS_NOP:
489 case AMDGPU::DS_PERMUTE_B32:
490 case AMDGPU::DS_BPERMUTE_B32:
491 return false;
492 default:
493 if (TII.isDS(Opcode: MI.getOpcode())) {
494 int GDS = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
495 Name: AMDGPU::OpName::gds);
496 if (MI.getOperand(i: GDS).getImm())
497 return true;
498 }
499 return false;
500 }
501}
502
503static bool isPermlane(const MachineInstr &MI) {
504 unsigned Opcode = MI.getOpcode();
505 return Opcode == AMDGPU::V_PERMLANE16_B32_e64 ||
506 Opcode == AMDGPU::V_PERMLANE64_B32 ||
507 Opcode == AMDGPU::V_PERMLANEX16_B32_e64 ||
508 Opcode == AMDGPU::V_PERMLANE16_VAR_B32_e64 ||
509 Opcode == AMDGPU::V_PERMLANEX16_VAR_B32_e64 ||
510 Opcode == AMDGPU::V_PERMLANE16_SWAP_B32_e32 ||
511 Opcode == AMDGPU::V_PERMLANE16_SWAP_B32_e64 ||
512 Opcode == AMDGPU::V_PERMLANE32_SWAP_B32_e32 ||
513 Opcode == AMDGPU::V_PERMLANE32_SWAP_B32_e64 ||
514 Opcode == AMDGPU::V_PERMLANE_BCAST_B32_e64 ||
515 Opcode == AMDGPU::V_PERMLANE_UP_B32_e64 ||
516 Opcode == AMDGPU::V_PERMLANE_DOWN_B32_e64 ||
517 Opcode == AMDGPU::V_PERMLANE_XOR_B32_e64 ||
518 Opcode == AMDGPU::V_PERMLANE_IDX_GEN_B32_e64;
519}
520
521static bool isLdsDma(const MachineInstr &MI) {
522 return SIInstrInfo::isLDSDMA(MI);
523}
524
525static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) {
526 const MachineOperand *RegOp = TII->getNamedOperand(MI: RegInstr,
527 OperandName: AMDGPU::OpName::simm16);
528 return std::get<0>(t: AMDGPU::Hwreg::HwregEncoding::decode(Encoded: RegOp->getImm()));
529}
530
531ScheduleHazardRecognizer::HazardType
532GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
533 MachineInstr *MI = SU->getInstr();
534 // If we are not in "HazardRecognizerMode" and therefore not being run from
535 // the scheduler, track possible stalls from hazards but don't insert noops.
536 auto HazardType = isHazardRecognizerMode() ? NoopHazard : Hazard;
537
538 if (MI->isBundle())
539 return NoHazard;
540
541 // Check co-execution slot hazards and pipeline stalls in scheduler modes.
542 if (isSchedulerMode()) {
543 if (checkMultiShadowHazard(MI: *MI) > 0)
544 return Hazard;
545 if (checkWMMACoexecSlot(MI: *MI) > 0)
546 return Hazard;
547 if (checkTRANSHazard(MI: *MI) > 0)
548 return Hazard;
549 if (checkMultiCycleVALUHazard(MI: *MI) > 0)
550 return Hazard;
551 // The remaining checks are all defined by register dependences.
552 if (!hasPhysRegs())
553 return NoHazard;
554 }
555
556 if (SIInstrInfo::isSMRD(MI: *MI) && checkSMRDHazards(SMRD: MI) > 0)
557 return HazardType;
558
559 if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0)
560 return HazardType;
561
562 if (checkFPAtomicToDenormModeHazard(MI) > 0)
563 return HazardType;
564
565 // Hazards which cannot be mitigated with S_NOPs.
566 if (!isHazardRecognizerMode()) {
567 if (checkWMMACoexecutionHazards(MI) > 0) {
568 HasPendingWMMACoexecHazard = true;
569 return Hazard;
570 }
571 }
572
573 if (ST.hasNoDataDepHazard())
574 return NoHazard;
575
576 if (SIInstrInfo::isVMEM(MI: *MI) && checkVMEMHazards(VMEM: MI) > 0)
577 return HazardType;
578
579 if (SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true) &&
580 checkVALUHazards(VALU: MI) > 0)
581 return HazardType;
582
583 if (SIInstrInfo::isDPP(MI: *MI) && checkDPPHazards(DPP: MI) > 0)
584 return HazardType;
585
586 if (isDivFMas(Opcode: MI->getOpcode()) && checkDivFMasHazards(DivFMas: MI) > 0)
587 return HazardType;
588
589 if (isRWLane(Opcode: MI->getOpcode()) && checkRWLaneHazards(RWLane: MI) > 0)
590 return HazardType;
591
592 if ((SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/false) ||
593 SIInstrInfo::isVMEM(MI: *MI) || SIInstrInfo::isDS(MI: *MI) ||
594 SIInstrInfo::isEXP(MI: *MI)) &&
595 checkMAIVALUHazards(MI) > 0)
596 return HazardType;
597
598 if (isSGetReg(Opcode: MI->getOpcode()) && checkGetRegHazards(GetRegInstr: MI) > 0)
599 return HazardType;
600
601 if (isSSetReg(Opcode: MI->getOpcode()) && checkSetRegHazards(SetRegInstr: MI) > 0)
602 return HazardType;
603
604 if (isRFE(Opcode: MI->getOpcode()) && checkRFEHazards(RFE: MI) > 0)
605 return HazardType;
606
607 if (((ST.hasReadM0MovRelInterpHazard() &&
608 (TII.isVINTRP(MI: *MI) || isSMovRel(Opcode: MI->getOpcode()) ||
609 MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 ||
610 MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) ||
611 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, MI: *MI)) ||
612 (ST.hasReadM0LdsDmaHazard() && isLdsDma(MI: *MI)) ||
613 (ST.hasReadM0LdsDirectHazard() &&
614 MI->readsRegister(Reg: AMDGPU::LDS_DIRECT, /*TRI=*/nullptr))) &&
615 checkReadM0Hazards(SMovRel: MI) > 0)
616 return HazardType;
617
618 if (SIInstrInfo::isMAI(MI: *MI) && checkMAIHazards(MI) > 0)
619 return HazardType;
620
621 if ((SIInstrInfo::isVMEM(MI: *MI) || SIInstrInfo::isDS(MI: *MI)) &&
622 checkMAILdStHazards(MI) > 0)
623 return HazardType;
624
625 if (ST.hasPermlaneForwardingHazard() && isPermlane(MI: *MI) &&
626 checkPermlaneHazards(MI) > 0)
627 return HazardType;
628
629 if (MI->isInlineAsm() && checkInlineAsmHazards(IA: MI) > 0)
630 return HazardType;
631
632 return NoHazard;
633}
634
635static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII,
636 unsigned Quantity) {
637 while (Quantity > 0) {
638 unsigned Arg = std::min(a: Quantity, b: 8u);
639 Quantity -= Arg;
640 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_NOP))
641 .addImm(Val: Arg - 1);
642 }
643}
644
645unsigned
646GCNHazardRecognizer::getMFMAPipelineWaitStates(const MachineInstr &MI) const {
647 const MCSchedClassDesc *SC = TSchedModel.resolveSchedClass(MI: &MI);
648 assert(TSchedModel.getWriteProcResBegin(SC) !=
649 TSchedModel.getWriteProcResEnd(SC));
650 return TSchedModel.getWriteProcResBegin(SC)->ReleaseAtCycle;
651}
652
653void GCNHazardRecognizer::processBundle() {
654 MachineBasicBlock::instr_iterator MI = std::next(x: CurrCycleInstr->getIterator());
655 MachineBasicBlock::instr_iterator E = CurrCycleInstr->getParent()->instr_end();
656 // Check bundled MachineInstr's for hazards.
657 for (; MI != E && MI->isInsideBundle(); ++MI) {
658 CurrCycleInstr = &*MI;
659 unsigned WaitStates = PreEmitNoopsCommon(CurrCycleInstr);
660
661 if (isHazardRecognizerMode()) {
662 fixHazards(MI: CurrCycleInstr);
663
664 insertNoopsInBundle(MI: CurrCycleInstr, TII, Quantity: WaitStates);
665 }
666
667 // It’s unnecessary to track more than MaxLookAhead instructions. Since we
668 // include the bundled MI directly after, only add a maximum of
669 // (MaxLookAhead - 1) noops to EmittedInstrs.
670 for (unsigned i = 0, e = std::min(a: WaitStates, b: MaxLookAhead - 1); i < e; ++i)
671 EmittedInstrs.push_front(x: nullptr);
672
673 EmittedInstrs.push_front(x: CurrCycleInstr);
674 EmittedInstrs.resize(new_size: MaxLookAhead);
675 }
676 CurrCycleInstr = nullptr;
677}
678
679void GCNHazardRecognizer::runOnInstruction(MachineInstr *MI) {
680 assert(isHazardRecognizerMode());
681
682 unsigned NumPreNoops = PreEmitNoops(MI);
683 EmitNoops(Quantity: NumPreNoops);
684 if (MI->isInsideBundle())
685 insertNoopsInBundle(MI, TII, Quantity: NumPreNoops);
686 else
687 TII.insertNoops(MBB&: *MI->getParent(), MI: MachineBasicBlock::iterator(MI),
688 Quantity: NumPreNoops);
689 EmitInstruction(MI);
690 AdvanceCycle();
691}
692
693unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
694 assert(isHazardRecognizerMode());
695 CurrCycleInstr = MI;
696 unsigned W = PreEmitNoopsCommon(MI);
697 fixHazards(MI);
698 CurrCycleInstr = nullptr;
699 return std::max(a: W, b: ST.getCLOpts().snop_padding);
700}
701
702unsigned GCNHazardRecognizer::getHazardWaitStates(MachineInstr *MI) const {
703 unsigned W = 0;
704
705 // Check co-execution slot hazards and pipeline stalls in scheduler modes.
706 if (isSchedulerMode()) {
707 W = checkWMMACoexecSlot(MI: *MI);
708 W = std::max(a: W, b: checkTRANSHazard(MI: *MI));
709 W = std::max(a: W, b: checkMultiCycleVALUHazard(MI: *MI));
710 W = std::max(a: W, b: checkMultiShadowHazard(MI: *MI));
711 // The remaining checks are all defined by register dependences.
712 if (!hasPhysRegs())
713 return W;
714 }
715
716 return std::max(a: W, b: PreEmitNoopsCommon(MI));
717}
718
719unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) const {
720 if (MI->isBundle())
721 return 0;
722
723 int WaitStates = 0;
724
725 if (SIInstrInfo::isSMRD(MI: *MI))
726 return std::max(a: WaitStates, b: checkSMRDHazards(SMRD: MI));
727
728 if (ST.hasNSAtoVMEMBug())
729 WaitStates = std::max(a: WaitStates, b: checkNSAtoVMEMHazard(MI));
730
731 WaitStates = std::max(a: WaitStates, b: checkFPAtomicToDenormModeHazard(MI));
732
733 if (ST.hasNoDataDepHazard())
734 return WaitStates;
735
736 if (SIInstrInfo::isVMEM(MI: *MI))
737 WaitStates = std::max(a: WaitStates, b: checkVMEMHazards(VMEM: MI));
738
739 if (SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true))
740 WaitStates = std::max(a: WaitStates, b: checkVALUHazards(VALU: MI));
741
742 if (SIInstrInfo::isDPP(MI: *MI))
743 WaitStates = std::max(a: WaitStates, b: checkDPPHazards(DPP: MI));
744
745 if (isDivFMas(Opcode: MI->getOpcode()))
746 WaitStates = std::max(a: WaitStates, b: checkDivFMasHazards(DivFMas: MI));
747
748 if (isRWLane(Opcode: MI->getOpcode()))
749 WaitStates = std::max(a: WaitStates, b: checkRWLaneHazards(RWLane: MI));
750
751 if (SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/false) ||
752 SIInstrInfo::isVMEM(MI: *MI) || SIInstrInfo::isDS(MI: *MI) ||
753 SIInstrInfo::isEXP(MI: *MI))
754 WaitStates = std::max(a: WaitStates, b: checkMAIVALUHazards(MI));
755
756 if (MI->isInlineAsm())
757 return std::max(a: WaitStates, b: checkInlineAsmHazards(IA: MI));
758
759 if (isSGetReg(Opcode: MI->getOpcode()))
760 return std::max(a: WaitStates, b: checkGetRegHazards(GetRegInstr: MI));
761
762 if (isSSetReg(Opcode: MI->getOpcode()))
763 return std::max(a: WaitStates, b: checkSetRegHazards(SetRegInstr: MI));
764
765 if (isRFE(Opcode: MI->getOpcode()))
766 return std::max(a: WaitStates, b: checkRFEHazards(RFE: MI));
767
768 if ((ST.hasReadM0MovRelInterpHazard() &&
769 (TII.isVINTRP(MI: *MI) || isSMovRel(Opcode: MI->getOpcode()) ||
770 MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 ||
771 MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) ||
772 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, MI: *MI)) ||
773 (ST.hasReadM0LdsDmaHazard() && isLdsDma(MI: *MI)) ||
774 (ST.hasReadM0LdsDirectHazard() &&
775 MI->readsRegister(Reg: AMDGPU::LDS_DIRECT, /*TRI=*/nullptr)))
776 return std::max(a: WaitStates, b: checkReadM0Hazards(SMovRel: MI));
777
778 if (SIInstrInfo::isMAI(MI: *MI))
779 return std::max(a: WaitStates, b: checkMAIHazards(MI));
780
781 if (SIInstrInfo::isVMEM(MI: *MI) || SIInstrInfo::isDS(MI: *MI))
782 return std::max(a: WaitStates, b: checkMAILdStHazards(MI));
783
784 if (ST.hasPermlaneForwardingHazard() && isPermlane(MI: *MI))
785 return std::max(a: WaitStates, b: checkPermlaneHazards(MI));
786
787 return WaitStates;
788}
789
790void GCNHazardRecognizer::EmitNoop() {
791 EmittedInstrs.push_front(x: nullptr);
792}
793
794void GCNHazardRecognizer::AdvanceCycle() {
795 if (isSchedulerMode())
796 schedulerAdvanceCycle();
797
798 // When the scheduler detects a stall, it will call AdvanceCycle() without
799 // emitting any instructions.
800 if (!CurrCycleInstr) {
801 EmittedInstrs.push_front(x: nullptr);
802
803 if (HasPendingWMMACoexecHazard)
804 EmittedVALUInstrs.push_front(x: nullptr);
805 return;
806 }
807
808 HasPendingWMMACoexecHazard = false;
809
810 if (CurrCycleInstr->isBundle()) {
811 processBundle();
812 return;
813 }
814
815 unsigned NumWaitStates = TII.getNumWaitStates(MI: *CurrCycleInstr);
816 if (!NumWaitStates) {
817 CurrCycleInstr = nullptr;
818 return;
819 }
820
821 // Keep track of emitted instructions
822 EmittedInstrs.push_front(x: CurrCycleInstr);
823
824 bool IsVALUOrWMMA =
825 SIInstrInfo::isVALU(MI: *CurrCycleInstr, /*AllowLDSDMA=*/true) ||
826 SIInstrInfo::isWMMA(MI: *CurrCycleInstr) ||
827 SIInstrInfo::isSWMMAC(MI: *CurrCycleInstr);
828 if (IsVALUOrWMMA) {
829 EmittedVALUInstrs.push_front(x: CurrCycleInstr);
830 } else {
831 // A pending WMMA co-execution hazard optimistically records stall cycles as
832 // future V_NOPs. If the scheduler instead stalls for a different
833 // (S_NOP-resolvable) hazard and schedules a non-VALU into those cycles,
834 // they will not resolve the VALU-pipe hazard, so drop them here.
835 while (!EmittedVALUInstrs.empty() && EmittedVALUInstrs.front() == nullptr)
836 EmittedVALUInstrs.pop_front();
837 }
838
839 // Add a nullptr for each additional wait state after the first. Make sure
840 // not to add more than getMaxLookAhead() items to the list, since we
841 // truncate the list to that size right after this loop.
842 for (unsigned i = 1, e = std::min(a: NumWaitStates, b: getMaxLookAhead());
843 i < e; ++i) {
844 EmittedInstrs.push_front(x: nullptr);
845 }
846
847 // getMaxLookahead() is the largest number of wait states we will ever need
848 // to insert, so there is no point in keeping track of more than that many
849 // wait states.
850 EmittedInstrs.resize(new_size: getMaxLookAhead());
851 if (EmittedVALUInstrs.size() > MaxVALULookAhead)
852 EmittedVALUInstrs.resize(new_size: MaxVALULookAhead);
853
854 CurrCycleInstr = nullptr;
855}
856
857void GCNHazardRecognizer::RecedeCycle() {
858 assert(!isHazardRecognizerMode() &&
859 "Bottom-up scheduling shouldn't run in hazard recognizer mode");
860}
861
862//===----------------------------------------------------------------------===//
863// Helper Functions
864//===----------------------------------------------------------------------===//
865
866enum HazardFnResult { HazardFound, HazardExpired, NoHazardFound };
867
868// Search for a hazard in a block and its predecessors.
869template <typename StateT>
870static bool
871hasHazard(StateT InitialState,
872 function_ref<HazardFnResult(StateT &, const MachineInstr &)> IsHazard,
873 function_ref<void(StateT &, const MachineInstr &)> UpdateState,
874 const MachineBasicBlock *InitialMBB,
875 MachineBasicBlock::const_reverse_instr_iterator InitialI) {
876 struct StateMapKey {
877 SmallVectorImpl<StateT> *States;
878 size_t Idx;
879 static bool isEqual(const StateMapKey &LHS, const StateMapKey &RHS) {
880 return LHS.States == RHS.States && LHS.Idx == RHS.Idx;
881 }
882 };
883 struct StateMapKeyTraits : DenseMapInfo<StateMapKey> {
884 static unsigned getHashValue(const StateMapKey &Key) {
885 return StateT::getHashValue((*Key.States)[Key.Idx]);
886 }
887 static unsigned getHashValue(const StateT &State) {
888 return StateT::getHashValue(State);
889 }
890 static bool isEqual(const StateMapKey &LHS, const StateMapKey &RHS) {
891 return StateT::isEqual((*LHS.States)[LHS.Idx], (*RHS.States)[RHS.Idx]);
892 }
893 static bool isEqual(const StateT &LHS, const StateMapKey &RHS) {
894 return StateT::isEqual(LHS, (*RHS.States)[RHS.Idx]);
895 }
896 };
897
898 SmallDenseMap<StateMapKey, size_t, 8, StateMapKeyTraits> StateMap;
899 SmallVector<StateT, 8> States;
900
901 MachineBasicBlock::const_reverse_instr_iterator I = InitialI;
902 const MachineBasicBlock *MBB = InitialMBB;
903 StateT State = InitialState;
904
905 SmallSetVector<std::pair<const MachineBasicBlock *, size_t>, 16> Worklist;
906 unsigned WorkIdx = 0;
907 for (;;) {
908 bool Expired = false;
909 for (auto E = MBB->instr_rend(); I != E; ++I) {
910 // No need to look at parent BUNDLE instructions.
911 if (I->isBundle())
912 continue;
913
914 auto Result = IsHazard(State, *I);
915 if (Result == HazardFound)
916 return true;
917 if (Result == HazardExpired) {
918 Expired = true;
919 break;
920 }
921
922 if (I->isInlineAsm() || I->isMetaInstruction())
923 continue;
924
925 UpdateState(State, *I);
926 }
927
928 if (!Expired) {
929 size_t StateIdx = States.size();
930 StateMapKey Key = {&States, StateIdx};
931 auto Insertion = StateMap.insert_as(std::pair(Key, StateIdx), State);
932 if (Insertion.second) {
933 States.emplace_back(State);
934 } else {
935 StateIdx = Insertion.first->second;
936 }
937 for (MachineBasicBlock *Pred : MBB->predecessors())
938 Worklist.insert(X: std::pair(Pred, StateIdx));
939 }
940
941 if (WorkIdx == Worklist.size())
942 break;
943
944 size_t StateIdx;
945 std::tie(args&: MBB, args&: StateIdx) = Worklist[WorkIdx++];
946 State = States[StateIdx];
947 I = MBB->instr_rbegin();
948 }
949
950 return false;
951}
952
953// Number of trailing terminators of \p Pred not executed on the edge to \p MBB:
954// control leaves at the first terminator branching to \p MBB, so the ones after
955// it are dead on that edge. A fallthrough executes them all.
956static unsigned countUnexecutedTerminators(const MachineBasicBlock &Pred,
957 const MachineBasicBlock *MBB) {
958 const SIInstrInfo &TII =
959 *Pred.getParent()->getSubtarget<GCNSubtarget>().getInstrInfo();
960 unsigned Unexecuted = 0, Passed = 0;
961 for (auto I = Pred.instr_rbegin(), E = Pred.instr_rend();
962 I != E && I->isTerminator(); ++I, ++Passed) {
963 // Only an explicit MBB destination identifies this edge; the last match is
964 // the earliest such branch in program order.
965 if (I->isBranch() && I->getNumOperands() > 0 && I->getOperand(i: 0).isMBB() &&
966 TII.getBranchDestBlock(MI: *I) == MBB)
967 Unexecuted = Passed;
968 }
969 return Unexecuted;
970}
971
972// Wait states \p MI adds to a distance. Inline asm may be empty, so it adds
973// none, which is the worst case.
974static unsigned
975getCountedWaitStates(const MachineInstr &MI,
976 GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates) {
977 return MI.isInlineAsm() ? 0 : GetNumWaitStates(MI);
978}
979
980namespace {
981
982// Answers GCNHazardRecognizer::getMax*WindowDeficit, which documents the
983// result, by walking the CFG backwards from the instruction being checked,
984// counting distances with GetNumWaitStates. That needs the instruction in its
985// final place, so hazard recognizer mode only. Each object answers one query.
986class WindowDeficitSearch {
987public:
988 WindowDeficitSearch(
989 function_ref<std::optional<int>(const MachineInstr &)> WindowFor,
990 int MaxWindow,
991 GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates =
992 SIInstrInfo::getNumWaitStates)
993 : WindowFor(WindowFor), GetNumWaitStates(GetNumWaitStates),
994 MaxWindow(MaxWindow) {}
995
996 int run(const MachineInstr &MI);
997
998private:
999 // A block to scan upwards from Start, which is Distance away from the
1000 // instruction being checked. In a predecessor, Start is the last instruction
1001 // above the terminators, and Distance includes the ones the edge executes.
1002 struct Arrival {
1003 const MachineBasicBlock *MBB;
1004 MachineBasicBlock::const_reverse_instr_iterator Start;
1005 int Distance;
1006 };
1007
1008 void arrive(const MachineBasicBlock *MBB,
1009 MachineBasicBlock::const_reverse_instr_iterator Start,
1010 int Distance);
1011 void scan(Arrival A);
1012
1013 const function_ref<std::optional<int>(const MachineInstr &)> WindowFor;
1014 const GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates;
1015 const int MaxWindow;
1016 int Deficit = 0;
1017 // Pending[D] holds the arrivals at distance D, and Best the smallest distance
1018 // each block has been queued at. Buckets are emptied in increasing distance,
1019 // so the first arrival scanned at a block is the shortest way into it, and
1020 // any later one is stale.
1021 SmallVector<SmallVector<Arrival, 2>, 4> Pending;
1022 DenseMap<const MachineBasicBlock *, int> Best;
1023};
1024
1025} // end anonymous namespace
1026
1027void WindowDeficitSearch::arrive(
1028 const MachineBasicBlock *MBB,
1029 MachineBasicBlock::const_reverse_instr_iterator Start, int Distance) {
1030 // No window exceeds MaxWindow, so nothing this far back can beat the deficit
1031 // already found.
1032 if (MaxWindow - Distance <= Deficit)
1033 return;
1034
1035 if (Pending.empty())
1036 Pending.resize(N: MaxWindow);
1037
1038 auto [It, Inserted] = Best.try_emplace(Key: MBB, Args&: Distance);
1039 if (!Inserted) {
1040 if (It->second <= Distance)
1041 return;
1042 It->second = Distance;
1043 }
1044
1045 assert(Distance < MaxWindow && "arrival outside the window");
1046 Pending[Distance].push_back(Elt: {.MBB: MBB, .Start: Start, .Distance: Distance});
1047}
1048
1049// \p A is a copy because arrive() may append to the bucket it came from, which
1050// can move that bucket's elements.
1051void WindowDeficitSearch::scan(Arrival A) {
1052 // A shorter way into this block has been queued since.
1053 if (Best.lookup(Val: A.MBB) < A.Distance)
1054 return;
1055
1056 int Distance = A.Distance;
1057 for (auto I = A.Start, E = A.MBB->instr_rend(); I != E; ++I) {
1058 if (I->isBundle())
1059 continue;
1060
1061 if (std::optional<int> Window = WindowFor(*I)) {
1062 assert(*Window >= 0 && *Window <= MaxWindow && "window out of bounds");
1063 Deficit = std::max(a: Deficit, b: *Window - Distance);
1064 }
1065
1066 Distance += getCountedWaitStates(MI: *I, GetNumWaitStates);
1067
1068 if (Distance >= MaxWindow)
1069 return;
1070 }
1071
1072 for (const MachineBasicBlock *Pred : A.MBB->predecessors()) {
1073 auto Start = Pred->instr_rbegin(), End = Pred->instr_rend();
1074 for (unsigned N = countUnexecutedTerminators(Pred: *Pred, MBB: A.MBB); N; --N)
1075 ++Start;
1076
1077 int Executed = 0;
1078 for (; Start != End && Start->isTerminator(); ++Start) {
1079 assert(!WindowFor(*Start) && "terminators must not ask for a window");
1080 Executed += getCountedWaitStates(MI: *Start, GetNumWaitStates);
1081 }
1082
1083 arrive(MBB: Pred, Start, Distance: Distance + Executed);
1084 }
1085}
1086
1087int WindowDeficitSearch::run(const MachineInstr &MI) {
1088 // Only the part above MI is scanned here, so the block stays out of Best: a
1089 // backedge into it must still scan it from its end.
1090 scan(A: {.MBB: MI.getParent(), .Start: std::next(x: MI.getReverseIterator()), .Distance: 0});
1091
1092 // Distances never shrink along a path, so a scan only queues arrivals at the
1093 // distance being swept or further, and one pass over the buckets is enough.
1094 // The inner loop re-reads the size because an edge that adds nothing appends
1095 // to the bucket in hand; Best keeps a cycle that adds nothing from looping.
1096 for (int Distance = 0; Distance < static_cast<int>(Pending.size());
1097 ++Distance)
1098 for (unsigned I = 0; I != Pending[Distance].size(); ++I)
1099 scan(A: Pending[Distance][I]);
1100
1101 return Deficit;
1102}
1103
1104// Returns a minimum wait states since \p I walking all predecessors.
1105// Only scans until \p IsExpired does not return true.
1106// Can only be run in a hazard recognizer mode.
1107static int
1108getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
1109 const MachineBasicBlock *MBB,
1110 MachineBasicBlock::const_reverse_instr_iterator I,
1111 int WaitStates, GCNHazardRecognizer::IsExpiredFn IsExpired,
1112 DenseSet<const MachineBasicBlock *> &Visited,
1113 GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates =
1114 SIInstrInfo::getNumWaitStates) {
1115 for (auto E = MBB->instr_rend(); I != E; ++I) {
1116 // Don't add WaitStates for parent BUNDLE instructions.
1117 if (I->isBundle())
1118 continue;
1119
1120 if (I->isMetaInstruction())
1121 continue;
1122
1123 if (IsHazard(*I))
1124 return WaitStates;
1125
1126 if (I->isInlineAsm())
1127 continue;
1128
1129 WaitStates += GetNumWaitStates(*I);
1130
1131 if (IsExpired(*I, WaitStates))
1132 return std::numeric_limits<int>::max();
1133 }
1134
1135 int MinWaitStates = std::numeric_limits<int>::max();
1136 for (MachineBasicBlock *Pred : MBB->predecessors()) {
1137 if (!Visited.insert(V: Pred).second)
1138 continue;
1139
1140 int W = getWaitStatesSince(IsHazard, MBB: Pred, I: Pred->instr_rbegin(), WaitStates,
1141 IsExpired, Visited, GetNumWaitStates);
1142
1143 MinWaitStates = std::min(a: MinWaitStates, b: W);
1144 }
1145
1146 return MinWaitStates;
1147}
1148
1149static int
1150getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
1151 const MachineInstr *MI,
1152 GCNHazardRecognizer::IsExpiredFn IsExpired,
1153 GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates =
1154 SIInstrInfo::getNumWaitStates) {
1155 DenseSet<const MachineBasicBlock *> Visited;
1156 return getWaitStatesSince(IsHazard, MBB: MI->getParent(),
1157 I: std::next(x: MI->getReverseIterator()), WaitStates: 0, IsExpired,
1158 Visited, GetNumWaitStates);
1159}
1160
1161int GCNHazardRecognizer::getMaxWindowDeficit(int MaxWindow,
1162 WindowForFn WindowFor) const {
1163 if (isHazardRecognizerMode())
1164 return WindowDeficitSearch(WindowFor, MaxWindow).run(MI: *CurrCycleInstr);
1165
1166 // EmittedInstrs is capped and can be shorter than the widest window, which
1167 // costs scheduling quality only: the standalone pass still pads.
1168 int Deficit = 0, Distance = 0;
1169 for (MachineInstr *MI : EmittedInstrs) {
1170 if (MI) {
1171 if (std::optional<int> Window = WindowFor(*MI)) {
1172 assert(*Window >= 0 && *Window <= MaxWindow && "window out of bounds");
1173 Deficit = std::max(a: Deficit, b: *Window - Distance);
1174 }
1175 }
1176
1177 Distance +=
1178 MI ? getCountedWaitStates(MI: *MI, GetNumWaitStates: SIInstrInfo::getNumWaitStates) : 1;
1179
1180 if (Distance >= MaxWindow)
1181 break;
1182 }
1183 return Deficit;
1184}
1185
1186int GCNHazardRecognizer::getWaitStatesSince(
1187 IsHazardFn IsHazard, int Limit, GetNumWaitStatesFn GetNumWaitStates) const {
1188 if (isHazardRecognizerMode()) {
1189 auto IsExpiredFn = [Limit](const MachineInstr &, int WaitStates) {
1190 return WaitStates >= Limit;
1191 };
1192 return ::getWaitStatesSince(IsHazard, MI: CurrCycleInstr, IsExpired: IsExpiredFn,
1193 GetNumWaitStates);
1194 }
1195
1196 int WaitStates = 0;
1197 for (MachineInstr *MI : EmittedInstrs) {
1198 if (MI) {
1199 if (IsHazard(*MI))
1200 return WaitStates;
1201
1202 if (MI->isInlineAsm())
1203 continue;
1204 }
1205 WaitStates += MI ? GetNumWaitStates(*MI) : 1;
1206
1207 if (WaitStates >= Limit)
1208 break;
1209 }
1210 return std::numeric_limits<int>::max();
1211}
1212
1213int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard,
1214 int Limit) const {
1215 return getWaitStatesSince(IsHazard, Limit, GetNumWaitStates: SIInstrInfo::getNumWaitStates);
1216}
1217
1218int GCNHazardRecognizer::getMaxVALUWindowDeficit(int MaxWindow,
1219 WindowForFn WindowFor) const {
1220 if (isHazardRecognizerMode()) {
1221 auto GetVALUWaitStates = [](const MachineInstr &MI) -> unsigned {
1222 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ? 1 : 0;
1223 };
1224 return WindowDeficitSearch(WindowFor, MaxWindow, GetVALUWaitStates)
1225 .run(MI: *CurrCycleInstr);
1226 }
1227
1228 // EmittedVALUInstrs is capped at MaxVALULookAhead, so a MaxWindow beyond that
1229 // window could miss a hazard. Keep the cap in sync with the wait-state
1230 // tables.
1231 assert(MaxWindow <= static_cast<int>(MaxVALULookAhead) &&
1232 "MaxWindow exceeds the EmittedVALUInstrs lookahead window");
1233 int WaitStates = 0;
1234 for (MachineInstr *MI : EmittedVALUInstrs) {
1235 if (MI) {
1236 if (std::optional<int> Window = WindowFor(*MI)) {
1237 assert(*Window >= 0 && *Window <= MaxWindow && "window out of bounds");
1238 return *Window - WaitStates;
1239 }
1240 }
1241
1242 ++WaitStates;
1243
1244 if (WaitStates >= MaxWindow)
1245 break;
1246 }
1247 return 0;
1248}
1249
1250int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg,
1251 IsHazardFn IsHazardDef,
1252 int Limit) const {
1253 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1254
1255 auto IsHazardFn = [IsHazardDef, TRI, Reg](const MachineInstr &MI) {
1256 return IsHazardDef(MI) && MI.modifiesRegister(Reg, TRI);
1257 };
1258
1259 return getWaitStatesSince(IsHazard: IsHazardFn, Limit);
1260}
1261
1262int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard,
1263 int Limit) const {
1264 auto IsHazardFn = [IsHazard](const MachineInstr &MI) {
1265 return isSSetReg(Opcode: MI.getOpcode()) && IsHazard(MI);
1266 };
1267
1268 return getWaitStatesSince(IsHazard: IsHazardFn, Limit);
1269}
1270
1271//===----------------------------------------------------------------------===//
1272// No-op Hazard Detection
1273//===----------------------------------------------------------------------===//
1274
1275static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV,
1276 MCRegister Reg) {
1277 for (MCRegUnit Unit : TRI.regunits(Reg))
1278 BV.set(static_cast<unsigned>(Unit));
1279}
1280
1281static void addRegsToSet(const SIRegisterInfo &TRI,
1282 iterator_range<MachineInstr::const_mop_iterator> Ops,
1283 BitVector &DefSet, BitVector &UseSet) {
1284 for (const MachineOperand &Op : Ops) {
1285 if (Op.isReg())
1286 addRegUnits(TRI, BV&: Op.isDef() ? DefSet : UseSet, Reg: Op.getReg().asMCReg());
1287 }
1288}
1289
1290void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) const {
1291 addRegsToSet(TRI, Ops: MI.operands(), DefSet&: ClauseDefs, UseSet&: ClauseUses);
1292}
1293
1294static bool breaksSMEMSoftClause(MachineInstr *MI) {
1295 return !SIInstrInfo::isSMRD(MI: *MI);
1296}
1297
1298static bool breaksVMEMSoftClause(MachineInstr *MI) {
1299 return !SIInstrInfo::isVMEM(MI: *MI);
1300}
1301
1302int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) const {
1303 // SMEM soft clause are only present on VI+, and only matter if xnack is
1304 // enabled.
1305 if (!ST.isXNACKEnabled())
1306 return 0;
1307
1308 bool IsSMRD = TII.isSMRD(MI: *MEM);
1309
1310 resetClause();
1311
1312 // A soft-clause is any group of consecutive SMEM instructions. The
1313 // instructions in this group may return out of order and/or may be
1314 // replayed (i.e. the same instruction issued more than once).
1315 //
1316 // In order to handle these situations correctly we need to make sure that
1317 // when a clause has more than one instruction, no instruction in the clause
1318 // writes to a register that is read by another instruction in the clause
1319 // (including itself). If we encounter this situation, we need to break the
1320 // clause by inserting a non SMEM instruction.
1321
1322 for (MachineInstr *MI : EmittedInstrs) {
1323 // When we hit a non-SMEM instruction then we have passed the start of the
1324 // clause and we can stop.
1325 if (!MI)
1326 break;
1327
1328 if (IsSMRD ? breaksSMEMSoftClause(MI) : breaksVMEMSoftClause(MI))
1329 break;
1330
1331 addClauseInst(MI: *MI);
1332 }
1333
1334 if (ClauseDefs.none())
1335 return 0;
1336
1337 // We need to make sure not to put loads and stores in the same clause if they
1338 // use the same address. For now, just start a new clause whenever we see a
1339 // store.
1340 if (MEM->mayStore())
1341 return 1;
1342
1343 addClauseInst(MI: *MEM);
1344
1345 // If the set of defs and uses intersect then we cannot add this instruction
1346 // to the clause, so we have a hazard.
1347 return ClauseDefs.anyCommon(RHS: ClauseUses) ? 1 : 0;
1348}
1349
1350int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) const {
1351 int WaitStatesNeeded = 0;
1352
1353 WaitStatesNeeded = checkSoftClauseHazards(MEM: SMRD);
1354
1355 // This SMRD hazard only affects SI.
1356 if (!ST.hasSMRDReadVALUDefHazard())
1357 return WaitStatesNeeded;
1358
1359 // A read of an SGPR by SMRD instruction requires 4 wait states when the
1360 // SGPR was written by a VALU instruction.
1361 int SmrdSgprWaitStates = 4;
1362 auto IsHazardDefFn = [this](const MachineInstr &MI) {
1363 return TII.isVALU(MI, /*AllowLDSDMA=*/false);
1364 };
1365 auto IsBufferHazardDefFn = [this](const MachineInstr &MI) {
1366 return TII.isSALU(MI);
1367 };
1368
1369 bool IsBufferSMRD = TII.isBufferSMRD(MI: *SMRD);
1370
1371 for (const MachineOperand &Use : SMRD->uses()) {
1372 if (!Use.isReg())
1373 continue;
1374 int WaitStatesNeededForUse =
1375 SmrdSgprWaitStates - getWaitStatesSinceDef(Reg: Use.getReg(), IsHazardDef: IsHazardDefFn,
1376 Limit: SmrdSgprWaitStates);
1377 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
1378
1379 // This fixes what appears to be undocumented hardware behavior in SI where
1380 // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor
1381 // needs some number of nops in between. We don't know how many we need, but
1382 // let's use 4. This wasn't discovered before probably because the only
1383 // case when this happens is when we expand a 64-bit pointer into a full
1384 // descriptor and use s_buffer_load_dword instead of s_load_dword, which was
1385 // probably never encountered in the closed-source land.
1386 if (IsBufferSMRD) {
1387 int WaitStatesNeededForUse =
1388 SmrdSgprWaitStates - getWaitStatesSinceDef(Reg: Use.getReg(),
1389 IsHazardDef: IsBufferHazardDefFn,
1390 Limit: SmrdSgprWaitStates);
1391 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
1392 }
1393 }
1394
1395 return WaitStatesNeeded;
1396}
1397
1398int GCNHazardRecognizer::checkVMEMHazards(MachineInstr *VMEM) const {
1399 if (!ST.hasVMEMReadSGPRVALUDefHazard())
1400 return 0;
1401
1402 int WaitStatesNeeded = checkSoftClauseHazards(MEM: VMEM);
1403
1404 // A read of an SGPR by a VMEM instruction requires 5 wait states when the
1405 // SGPR was written by a VALU Instruction.
1406 const int VmemSgprWaitStates = 5;
1407 auto IsHazardDefFn = [this](const MachineInstr &MI) {
1408 return TII.isVALU(MI, /*AllowLDSDMA=*/false);
1409 };
1410 for (const MachineOperand &Use : VMEM->uses()) {
1411 if (!Use.isReg() || TRI.isVectorRegister(MRI: MF.getRegInfo(), Reg: Use.getReg()))
1412 continue;
1413
1414 int WaitStatesNeededForUse =
1415 VmemSgprWaitStates - getWaitStatesSinceDef(Reg: Use.getReg(), IsHazardDef: IsHazardDefFn,
1416 Limit: VmemSgprWaitStates);
1417 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
1418 }
1419 return WaitStatesNeeded;
1420}
1421
1422int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) const {
1423 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1424 const SIInstrInfo *TII = ST.getInstrInfo();
1425
1426 // Check for DPP VGPR read after VALU VGPR write and EXEC write.
1427 int DppVgprWaitStates = 2;
1428 int DppExecWaitStates = 5;
1429 int WaitStatesNeeded = 0;
1430 auto IsHazardDefFn = [TII](const MachineInstr &MI) {
1431 return TII->isVALU(MI, /*AllowLDSDMA=*/false);
1432 };
1433
1434 for (const MachineOperand &Use : DPP->uses()) {
1435 if (!Use.isReg() || !TRI->isVGPR(MRI: MF.getRegInfo(), Reg: Use.getReg()))
1436 continue;
1437 int WaitStatesNeededForUse =
1438 DppVgprWaitStates - getWaitStatesSinceDef(
1439 Reg: Use.getReg(),
1440 IsHazardDef: [](const MachineInstr &) { return true; },
1441 Limit: DppVgprWaitStates);
1442 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
1443 }
1444
1445 WaitStatesNeeded = std::max(
1446 a: WaitStatesNeeded,
1447 b: DppExecWaitStates - getWaitStatesSinceDef(Reg: AMDGPU::EXEC, IsHazardDef: IsHazardDefFn,
1448 Limit: DppExecWaitStates));
1449
1450 return WaitStatesNeeded;
1451}
1452
1453int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) const {
1454 const SIInstrInfo *TII = ST.getInstrInfo();
1455
1456 // v_div_fmas requires 4 wait states after a write to vcc from a VALU
1457 // instruction.
1458 const int DivFMasWaitStates = 4;
1459 auto IsHazardDefFn = [TII](const MachineInstr &MI) {
1460 return TII->isVALU(MI, /*AllowLDSDMA=*/false);
1461 };
1462 int WaitStatesNeeded = getWaitStatesSinceDef(Reg: AMDGPU::VCC, IsHazardDef: IsHazardDefFn,
1463 Limit: DivFMasWaitStates);
1464
1465 return DivFMasWaitStates - WaitStatesNeeded;
1466}
1467
1468int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) const {
1469 const SIInstrInfo *TII = ST.getInstrInfo();
1470 unsigned GetRegHWReg = getHWReg(TII, RegInstr: *GetRegInstr);
1471
1472 const int GetRegWaitStates = 2;
1473 auto IsHazardFn = [TII, GetRegHWReg](const MachineInstr &MI) {
1474 return GetRegHWReg == getHWReg(TII, RegInstr: MI);
1475 };
1476 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazard: IsHazardFn, Limit: GetRegWaitStates);
1477
1478 return GetRegWaitStates - WaitStatesNeeded;
1479}
1480
1481int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) const {
1482 const SIInstrInfo *TII = ST.getInstrInfo();
1483 unsigned HWReg = getHWReg(TII, RegInstr: *SetRegInstr);
1484
1485 const int SetRegWaitStates = ST.getSetRegWaitStates();
1486 auto IsHazardFn = [TII, HWReg](const MachineInstr &MI) {
1487 return HWReg == getHWReg(TII, RegInstr: MI);
1488 };
1489 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazard: IsHazardFn, Limit: SetRegWaitStates);
1490 return SetRegWaitStates - WaitStatesNeeded;
1491}
1492
1493int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) const {
1494 if (!MI.mayStore())
1495 return -1;
1496
1497 const SIInstrInfo *TII = ST.getInstrInfo();
1498 unsigned Opcode = MI.getOpcode();
1499 const MCInstrDesc &Desc = MI.getDesc();
1500
1501 int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::vdata);
1502 int VDataRCID = -1;
1503 if (VDataIdx != -1)
1504 VDataRCID = TII->getOpRegClassID(OpInfo: Desc.operands()[VDataIdx]);
1505
1506 if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) {
1507 // There is no hazard if the instruction does not use vector regs
1508 // (like wbinvl1)
1509 if (VDataIdx == -1)
1510 return -1;
1511 if (AMDGPU::getRegBitWidth(RCID: VDataRCID) > 64) {
1512 // When SOFFSET-dependent wide-store windows apply, the BUFFER_STORE
1513 // source-vgpr WAR hazard exists for every SOFFSET shape; the wait-state
1514 // count differs by SOFFSET and is computed in checkVALUHazardsHelper.
1515 // Otherwise the hazard only exists if soffset is not an SGPR.
1516 if (ST.hasVDecCoExecHazard())
1517 return VDataIdx;
1518 const MachineOperand *SOffset =
1519 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
1520 if (!SOffset || !SOffset->isReg())
1521 return VDataIdx;
1522 }
1523 }
1524
1525 // MIMG instructions create a hazard if they don't use a 256-bit T# and
1526 // the store size is greater than 8 bytes and they have more than two bits
1527 // of their dmask set.
1528 // All our MIMG definitions use a 256-bit T#, so we can skip checking for them.
1529 if (TII->isMIMG(MI)) {
1530 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::srsrc);
1531 assert(SRsrcIdx != -1 && AMDGPU::getRegBitWidth(TII->getOpRegClassID(
1532 Desc.operands()[SRsrcIdx])) == 256);
1533 (void)SRsrcIdx;
1534 }
1535
1536 if (TII->isFLAT(MI)) {
1537 // There is no hazard if the instruction does not use vector regs
1538 if (VDataIdx == -1)
1539 return -1;
1540
1541 if (AMDGPU::getRegBitWidth(RCID: VDataRCID) > 64)
1542 return VDataIdx;
1543 }
1544
1545 return -1;
1546}
1547
1548int GCNHazardRecognizer::checkUniformWindowVALUHazardsHelper(
1549 Register Reg) const {
1550 // Wide stores need a single wait-state bubble before a VALU that overwrites
1551 // store data. createsVALUHazard already excludes MUBUF/MTBUF stores with an
1552 // SGPR SOFFSET.
1553 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1554
1555 auto IsHazard = [&](const MachineInstr &MI) {
1556 int DataIdx = createsVALUHazard(MI);
1557 return DataIdx >= 0 &&
1558 TRI->regsOverlap(RegA: MI.getOperand(i: DataIdx).getReg(), RegB: Reg);
1559 };
1560
1561 return std::max(a: 0, b: 1 - getWaitStatesSince(IsHazard, /*Limit=*/1));
1562}
1563
1564int GCNHazardRecognizer::checkSOFFSETWindowVALUHazardsHelper(
1565 Register Reg) const {
1566 // The required wait-state window depends on the producer's SOFFSET shape:
1567 // - MUBUF/MTBUF wide store with sgpr SOFFSET: 1 wait state.
1568 // - MUBUF/MTBUF wide store with literal/absent SOFFSET, and FLAT wide
1569 // store: 2 wait states.
1570 // The 1-cycle sgpr-SOFFSET window was measured on gfx950.
1571 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1572 const SIInstrInfo *TII = ST.getInstrInfo();
1573
1574 int WaitStatesNeeded = 0;
1575
1576 // Scan each wait-state window separately and take the max padding needed.
1577 // getWaitStatesSince supplies the minimum distance to a producer over paths.
1578 for (int Window = 1; Window <= 2; ++Window) {
1579 auto IsHazard = [&](const MachineInstr &MI) {
1580 int DataIdx = createsVALUHazard(MI);
1581 if (DataIdx < 0 ||
1582 !TRI->regsOverlap(RegA: MI.getOperand(i: DataIdx).getReg(), RegB: Reg))
1583 return false;
1584
1585 // Window 1 matches every hazard producer. Window 2 excludes BUF stores
1586 // with an SGPR SOFFSET, which only require a single wait state.
1587 if (Window == 1 || !TII->isBUF(MI))
1588 return true;
1589
1590 const MachineOperand *SOffset =
1591 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
1592 return !SOffset || !SOffset->isReg();
1593 };
1594 WaitStatesNeeded = std::max(a: WaitStatesNeeded,
1595 b: Window - getWaitStatesSince(IsHazard, Limit: Window));
1596 }
1597
1598 return WaitStatesNeeded;
1599}
1600
1601int GCNHazardRecognizer::checkVALUHazardsHelper(
1602 const MachineOperand &Def, const MachineRegisterInfo &MRI) const {
1603 // Helper to check for the hazard where VMEM instructions that store more
1604 // than 8 bytes can have their store data overwritten by the next
1605 // instruction.
1606 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1607
1608 if (!TRI->isVectorRegister(MRI, Reg: Def.getReg()))
1609 return 0;
1610
1611 if (ST.hasVDecCoExecHazard())
1612 return checkSOFFSETWindowVALUHazardsHelper(Reg: Def.getReg());
1613
1614 return checkUniformWindowVALUHazardsHelper(Reg: Def.getReg());
1615}
1616
1617/// Dest sel forwarding issue occurs if additional logic is needed to swizzle /
1618/// pack the computed value into correct bit position of the dest register. This
1619/// occurs if we have SDWA with dst_sel != DWORD or if we have op_sel with
1620/// dst_sel that is not aligned to the register. This function analayzes the \p
1621/// MI and \returns an operand with dst forwarding issue, or nullptr if
1622/// none exists.
1623static const MachineOperand *
1624getDstSelForwardingOperand(const MachineInstr &MI, const GCNSubtarget &ST) {
1625 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false))
1626 return nullptr;
1627
1628 const SIInstrInfo *TII = ST.getInstrInfo();
1629
1630 unsigned Opcode = MI.getOpcode();
1631
1632 // There are three different types of instructions
1633 // which produce forwarded dest: 1. SDWA with dst_sel != DWORD, 2. VOP3
1634 // which write hi bits (e.g. op_sel[3] == 1), and 3. FP8DstSelInst
1635 // (instructions with dest byte sel, e.g. CVT_SR_BF8_F32) and
1636 // op_sel[3:2]
1637 // != 0
1638 if (SIInstrInfo::isSDWA(MI)) {
1639 // Type 1: SDWA with dst_sel != DWORD
1640 if (auto *DstSel = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::dst_sel))
1641 if (DstSel->getImm() != AMDGPU::SDWA::DWORD)
1642 return TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1643 }
1644
1645 AMDGPU::FPType IsFP4OrFP8ConvOpc = AMDGPU::getFPDstSelType(Opc: Opcode);
1646 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::op_sel)) {
1647 // Type 2: VOP3 which write the hi bits. DST_OP_SEL is a destination
1648 // selector in the VOP3 layout; VOP3P uses this bit for source op_sel_hi.
1649 if (SIInstrInfo::isVOP3(MI) && !SIInstrInfo::isVOP3P(MI) &&
1650 (TII->getNamedImmOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers) &
1651 SISrcMods::DST_OP_SEL))
1652 return TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1653
1654 // Type 2b: VOP3P mix instructions with a tied destination (MIXLO/MIXHI)
1655 // write only one half of vdst and preserve the other half, so treat them
1656 // like a VOP3 destination op_sel write regardless of their source
1657 // op_sel_hi bits.
1658 if (SIInstrInfo::isVOP3P(MI) &&
1659 AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::vdst_in))
1660 return TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1661
1662 // Type 3: FP8DstSelInst with op_sel[3:2] != 0)
1663 if (IsFP4OrFP8ConvOpc == AMDGPU::FPType::FP8 &&
1664 (TII->getNamedImmOperand(MI, OperandName: AMDGPU::OpName::src2_modifiers) &
1665 SISrcMods::OP_SEL_0))
1666 return TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1667 }
1668
1669 // Special case: nop is required for all the opsel values for fp4 sr variant
1670 // cvt scale instructions
1671 if (IsFP4OrFP8ConvOpc == AMDGPU::FPType::FP4)
1672 return TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1673
1674 return nullptr;
1675}
1676
1677/// Checks whether the provided \p MI "consumes" the operand with a Dest sel
1678/// fowarding issue \p Dst . We may "consume" the Dst via a standard explicit
1679/// RAW, or through irregular ways (e.g implicit RAW, certain types of WAW)
1680static bool consumesDstSelForwardingOperand(const MachineInstr *VALU,
1681 const MachineOperand *Dst,
1682 const SIRegisterInfo *TRI) {
1683 // We must consider implicit reads of the VALU. SDWA with dst_sel and
1684 // UNUSED_PRESERVE will implicitly read the result from forwarded dest,
1685 // and we must account for that hazard.
1686 // We also must account for WAW hazards. In particular, WAW with dest
1687 // preserve semantics (e.g. VOP3 with op_sel, VOP2 &&
1688 // !zeroesHigh16BitsOfDest) will read the forwarded dest for parity
1689 // check for ECC. Without accounting for this hazard, the ECC will be
1690 // wrong.
1691 // TODO: limit to RAW (including implicit reads) + problematic WAW (i.e.
1692 // complete zeroesHigh16BitsOfDest)
1693 for (auto &Operand : VALU->operands()) {
1694 if (Operand.isReg() && TRI->regsOverlap(RegA: Dst->getReg(), RegB: Operand.getReg())) {
1695 return true;
1696 }
1697 }
1698 return false;
1699}
1700
1701int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) const {
1702 int WaitStatesNeeded = 0;
1703
1704 if (ST.hasTransForwardingHazard() && !SIInstrInfo::isTRANS(MI: *VALU)) {
1705 const int TransDefWaitstates = 1;
1706
1707 auto IsTransDefFn = [this, VALU](const MachineInstr &MI) {
1708 if (!SIInstrInfo::isTRANS(MI))
1709 return false;
1710 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1711 const SIInstrInfo *TII = ST.getInstrInfo();
1712 Register Def = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst)->getReg();
1713
1714 for (const MachineOperand &Use : VALU->explicit_uses()) {
1715 if (Use.isReg() && TRI->regsOverlap(RegA: Def, RegB: Use.getReg()))
1716 return true;
1717 }
1718
1719 return false;
1720 };
1721
1722 int WaitStatesNeededForDef =
1723 TransDefWaitstates -
1724 getWaitStatesSince(IsHazard: IsTransDefFn, Limit: TransDefWaitstates);
1725 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1726 }
1727
1728 if (ST.hasDstSelForwardingHazard() || ST.hasCvtScaleForwardingHazard()) {
1729 const int Shift16DefWaitstates = 1;
1730
1731 auto IsShift16BitDefFn = [this, VALU](const MachineInstr &ProducerMI) {
1732 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1733 const MachineOperand *ForwardedDst =
1734 getDstSelForwardingOperand(MI: ProducerMI, ST);
1735 if (ForwardedDst) {
1736 return consumesDstSelForwardingOperand(VALU, Dst: ForwardedDst, TRI);
1737 }
1738
1739 if (ProducerMI.isInlineAsm()) {
1740 // Assume inline asm has dst forwarding hazard
1741 for (auto &Def : ProducerMI.all_defs()) {
1742 if (consumesDstSelForwardingOperand(VALU, Dst: &Def, TRI))
1743 return true;
1744 }
1745 }
1746
1747 return false;
1748 };
1749
1750 int WaitStatesNeededForDef =
1751 Shift16DefWaitstates -
1752 getWaitStatesSince(IsHazard: IsShift16BitDefFn, Limit: Shift16DefWaitstates);
1753 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1754 }
1755
1756 if (ST.hasVDecCoExecHazard()) {
1757 const int VALUWriteSGPRVALUReadWaitstates = 2;
1758 const int VALUWriteEXECRWLane = 4;
1759 const int VALUWriteVGPRReadlaneRead = 1;
1760
1761 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1762 const MachineRegisterInfo &MRI = MF.getRegInfo();
1763 Register UseReg;
1764 auto IsVALUDefSGPRFn = [&UseReg, TRI](const MachineInstr &MI) {
1765 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false))
1766 return false;
1767 return MI.modifiesRegister(Reg: UseReg, TRI);
1768 };
1769
1770 for (const MachineOperand &Use : VALU->explicit_uses()) {
1771 if (!Use.isReg())
1772 continue;
1773
1774 UseReg = Use.getReg();
1775 if (TRI->isSGPRReg(MRI, Reg: UseReg)) {
1776 int WaitStatesNeededForDef =
1777 VALUWriteSGPRVALUReadWaitstates -
1778 getWaitStatesSince(IsHazard: IsVALUDefSGPRFn,
1779 Limit: VALUWriteSGPRVALUReadWaitstates);
1780 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1781 }
1782 }
1783
1784 if (VALU->readsRegister(Reg: AMDGPU::VCC, TRI)) {
1785 UseReg = AMDGPU::VCC;
1786 int WaitStatesNeededForDef =
1787 VALUWriteSGPRVALUReadWaitstates -
1788 getWaitStatesSince(IsHazard: IsVALUDefSGPRFn, Limit: VALUWriteSGPRVALUReadWaitstates);
1789 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1790 }
1791
1792 switch (VALU->getOpcode()) {
1793 case AMDGPU::V_READLANE_B32:
1794 case AMDGPU::V_READFIRSTLANE_B32: {
1795 MachineOperand *Src = TII.getNamedOperand(MI&: *VALU, OperandName: AMDGPU::OpName::src0);
1796 UseReg = Src->getReg();
1797 int WaitStatesNeededForDef =
1798 VALUWriteVGPRReadlaneRead -
1799 getWaitStatesSince(IsHazard: IsVALUDefSGPRFn, Limit: VALUWriteVGPRReadlaneRead);
1800 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1801 }
1802 [[fallthrough]];
1803 case AMDGPU::V_WRITELANE_B32: {
1804 UseReg = AMDGPU::EXEC;
1805 int WaitStatesNeededForDef =
1806 VALUWriteEXECRWLane -
1807 getWaitStatesSince(IsHazard: IsVALUDefSGPRFn, Limit: VALUWriteEXECRWLane);
1808 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1809 break;
1810 }
1811 default:
1812 break;
1813 }
1814 }
1815
1816 // This checks for the hazard where VMEM instructions that store more than
1817 // 8 bytes can have there store data over written by the next instruction.
1818 if (!ST.has12DWordStoreHazard())
1819 return WaitStatesNeeded;
1820
1821 const MachineRegisterInfo &MRI = MF.getRegInfo();
1822
1823 for (const MachineOperand &Def : VALU->defs()) {
1824 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: checkVALUHazardsHelper(Def, MRI));
1825 }
1826
1827 return WaitStatesNeeded;
1828}
1829
1830int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) const {
1831 // This checks for hazards associated with inline asm statements.
1832 // Since inline asms can contain just about anything, we use this
1833 // to call/leverage other check*Hazard routines. Note that
1834 // this function doesn't attempt to address all possible inline asm
1835 // hazards (good luck), but is a collection of what has been
1836 // problematic thus far.
1837
1838 // see checkVALUHazards()
1839 if (!ST.has12DWordStoreHazard() && !ST.hasDstSelForwardingHazard() &&
1840 !ST.hasCvtScaleForwardingHazard())
1841 return 0;
1842
1843 const MachineRegisterInfo &MRI = MF.getRegInfo();
1844 int WaitStatesNeeded = 0;
1845
1846 for (const MachineOperand &Op :
1847 llvm::drop_begin(RangeOrContainer: IA->operands(), N: InlineAsm::MIOp_FirstOperand)) {
1848 if (Op.isReg() && Op.isDef()) {
1849 if (!TRI.isVectorRegister(MRI, Reg: Op.getReg()))
1850 continue;
1851
1852 if (ST.has12DWordStoreHazard()) {
1853 WaitStatesNeeded =
1854 std::max(a: WaitStatesNeeded, b: checkVALUHazardsHelper(Def: Op, MRI));
1855 }
1856 }
1857 }
1858
1859 if (ST.hasDstSelForwardingHazard()) {
1860 const int Shift16DefWaitstates = 1;
1861
1862 auto IsShift16BitDefFn = [this, &IA](const MachineInstr &ProducerMI) {
1863 const MachineOperand *Dst = getDstSelForwardingOperand(MI: ProducerMI, ST);
1864 // Assume inline asm reads the dst
1865 if (Dst)
1866 return IA->modifiesRegister(Reg: Dst->getReg(), TRI: &TRI) ||
1867 IA->readsRegister(Reg: Dst->getReg(), TRI: &TRI);
1868
1869 if (ProducerMI.isInlineAsm()) {
1870 // If MI is inline asm, assume it has dst forwarding hazard
1871 for (auto &Def : ProducerMI.all_defs()) {
1872 if (IA->modifiesRegister(Reg: Def.getReg(), TRI: &TRI) ||
1873 IA->readsRegister(Reg: Def.getReg(), TRI: &TRI)) {
1874 return true;
1875 }
1876 }
1877 }
1878
1879 return false;
1880 };
1881
1882 int WaitStatesNeededForDef =
1883 Shift16DefWaitstates -
1884 getWaitStatesSince(IsHazard: IsShift16BitDefFn, Limit: Shift16DefWaitstates);
1885 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1886 }
1887
1888 return WaitStatesNeeded;
1889}
1890
1891int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) const {
1892 const SIInstrInfo *TII = ST.getInstrInfo();
1893 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1894 const MachineRegisterInfo &MRI = MF.getRegInfo();
1895
1896 const MachineOperand *LaneSelectOp =
1897 TII->getNamedOperand(MI&: *RWLane, OperandName: AMDGPU::OpName::src1);
1898
1899 if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, Reg: LaneSelectOp->getReg()))
1900 return 0;
1901
1902 Register LaneSelectReg = LaneSelectOp->getReg();
1903 auto IsHazardFn = [TII](const MachineInstr &MI) {
1904 return TII->isVALU(MI, /*AllowLDSDMA=*/false);
1905 };
1906
1907 const int RWLaneWaitStates = 4;
1908 int WaitStatesSince = getWaitStatesSinceDef(Reg: LaneSelectReg, IsHazardDef: IsHazardFn,
1909 Limit: RWLaneWaitStates);
1910 return RWLaneWaitStates - WaitStatesSince;
1911}
1912
1913int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) const {
1914 if (!ST.hasRFEHazards())
1915 return 0;
1916
1917 const SIInstrInfo *TII = ST.getInstrInfo();
1918
1919 const int RFEWaitStates = 1;
1920
1921 auto IsHazardFn = [TII](const MachineInstr &MI) {
1922 return getHWReg(TII, RegInstr: MI) == AMDGPU::Hwreg::ID_TRAPSTS;
1923 };
1924 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazard: IsHazardFn, Limit: RFEWaitStates);
1925 return RFEWaitStates - WaitStatesNeeded;
1926}
1927
1928int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) const {
1929 const SIInstrInfo *TII = ST.getInstrInfo();
1930 const int ReadM0WaitStates = 1;
1931 auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isSALU(MI); };
1932 return ReadM0WaitStates -
1933 getWaitStatesSinceDef(Reg: AMDGPU::M0, IsHazardDef: IsHazardFn, Limit: ReadM0WaitStates);
1934}
1935
1936void GCNHazardRecognizer::emitVNops(MachineBasicBlock &MBB,
1937 MachineBasicBlock::iterator InsertPt,
1938 int WaitStatesNeeded, bool IsHoisting) {
1939 const DebugLoc &DL = IsHoisting ? DebugLoc() : InsertPt->getDebugLoc();
1940 for (int I = 0; I < WaitStatesNeeded; ++I)
1941 BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_NOP_e32));
1942}
1943
1944void GCNHazardRecognizer::fixHazards(MachineInstr *MI) {
1945 fixVMEMtoScalarWriteHazards(MI);
1946 fixVcmpxPermlaneHazards(MI);
1947 fixSMEMtoVectorWriteHazards(MI);
1948 fixVcmpxExecWARHazard(MI);
1949 fixLdsBranchVmemWARHazard(MI);
1950 if (ST.hasLdsDirect()) {
1951 fixLdsDirectVALUHazard(MI);
1952 fixLdsDirectVMEMHazard(MI);
1953 }
1954 fixVALUPartialForwardingHazard(MI);
1955 fixVALUTransUseHazard(MI);
1956 fixVALUTransCoexecutionHazards(MI);
1957 fixWMMAHazards(MI); // fall-through if co-execution is enabled.
1958 fixWMMACoexecutionHazards(MI);
1959 fixShift64HighRegBug(MI);
1960 fixVALUMaskWriteHazard(MI);
1961 fixRequiredExportPriority(MI);
1962 if (ST.hasVPermPk16Hazard())
1963 fixVPermPk16Hazard(MI);
1964 if (ST.requiresWaitIdleBeforeGetReg())
1965 fixGetRegWaitIdle(MI);
1966 if (ST.hasDsAtomicAsyncBarrierArriveB64PipeBug())
1967 fixDsAtomicAsyncBarrierArriveB64(MI);
1968 if (ST.hasScratchBaseForwardingHazard())
1969 fixScratchBaseForwardingHazard(MI);
1970 if (ST.setRegModeNeedsVNOPs())
1971 fixSetRegMode(MI);
1972 if (ST.hasNeedsTDMDrain())
1973 fixTDM(MI);
1974 if (ST.hasTransF16ToPartialVGPRWriteHazard())
1975 fixTransF16ToPartialVGPRWriteHazard(MI);
1976}
1977
1978static bool isVCmpXWritesExec(const SIInstrInfo &TII, const SIRegisterInfo &TRI,
1979 const MachineInstr &MI) {
1980 return (TII.isVOPC(MI) ||
1981 (MI.isCompare() && (TII.isVOP3(MI) || TII.isSDWA(MI)))) &&
1982 MI.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &TRI);
1983}
1984
1985bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) {
1986 if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(MI: *MI))
1987 return false;
1988
1989 const SIInstrInfo *TII = ST.getInstrInfo();
1990 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1991 auto IsHazardFn = [TII, TRI](const MachineInstr &MI) {
1992 return isVCmpXWritesExec(TII: *TII, TRI: *TRI, MI);
1993 };
1994
1995 auto IsExpiredFn = [](const MachineInstr &MI, int) {
1996 unsigned Opc = MI.getOpcode();
1997 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
1998 Opc != AMDGPU::V_NOP_e32 && Opc != AMDGPU::V_NOP_e64 &&
1999 Opc != AMDGPU::V_NOP_sdwa;
2000 };
2001
2002 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2003 std::numeric_limits<int>::max())
2004 return false;
2005
2006 // V_NOP will be discarded by SQ.
2007 // Use V_MOV_B32 v?, v?. Register must be alive so use src0 of V_PERMLANE*
2008 // which is always a VGPR and available.
2009 auto *Src0 = TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src0);
2010 Register Reg = Src0->getReg();
2011 bool IsUndef = Src0->isUndef();
2012 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2013 MCID: TII->get(Opcode: AMDGPU::V_MOV_B32_e32))
2014 .addReg(RegNo: Reg, Flags: RegState::Define | getDeadRegState(B: IsUndef))
2015 .addReg(RegNo: Reg, Flags: IsUndef ? RegState::Undef : RegState::Kill);
2016
2017 return true;
2018}
2019
2020bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) {
2021 if (!ST.hasVMEMtoScalarWriteHazard())
2022 return false;
2023 assert(!ST.hasExtendedWaitCounts());
2024
2025 if (!SIInstrInfo::isSALU(MI: *MI) && !SIInstrInfo::isSMRD(MI: *MI))
2026 return false;
2027
2028 if (MI->getNumDefs() == 0)
2029 return false;
2030
2031 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2032
2033 auto IsHazardFn = [TRI, MI](const MachineInstr &I) {
2034 if (!SIInstrInfo::isVMEM(MI: I) && !SIInstrInfo::isDS(MI: I))
2035 return false;
2036
2037 for (const MachineOperand &Def : MI->defs()) {
2038 const MachineOperand *Op =
2039 I.findRegisterUseOperand(Reg: Def.getReg(), TRI, isKill: false);
2040 if (!Op)
2041 continue;
2042 return true;
2043 }
2044 return false;
2045 };
2046
2047 auto IsExpiredFn = [](const MachineInstr &MI, int) {
2048 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ||
2049 (MI.getOpcode() == AMDGPU::S_WAITCNT &&
2050 !MI.getOperand(i: 0).getImm()) ||
2051 (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2052 AMDGPU::DepCtr::decodeFieldVmVsrc(
2053 Encoded: static_cast<unsigned>(MI.getOperand(i: 0).getImm())) == 0);
2054 };
2055
2056 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2057 std::numeric_limits<int>::max())
2058 return false;
2059
2060 const SIInstrInfo *TII = ST.getInstrInfo();
2061 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2062 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
2063 .addImm(Val: AMDGPU::DepCtr::encodeFieldVmVsrc(VmVsrc: 0, STI: ST));
2064 return true;
2065}
2066
2067bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) {
2068 if (!ST.hasSMEMtoVectorWriteHazard())
2069 return false;
2070 assert(!ST.hasExtendedWaitCounts());
2071
2072 if (!SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/false))
2073 return false;
2074
2075 AMDGPU::OpName SDSTName;
2076 switch (MI->getOpcode()) {
2077 case AMDGPU::V_READLANE_B32:
2078 case AMDGPU::V_READFIRSTLANE_B32:
2079 SDSTName = AMDGPU::OpName::vdst;
2080 break;
2081 default:
2082 SDSTName = AMDGPU::OpName::sdst;
2083 break;
2084 }
2085
2086 const SIInstrInfo *TII = ST.getInstrInfo();
2087 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2088 const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(GPU: ST.getCPU());
2089 const MachineOperand *SDST = TII->getNamedOperand(MI&: *MI, OperandName: SDSTName);
2090 if (!SDST) {
2091 for (const auto &MO : MI->implicit_operands()) {
2092 if (MO.isDef() && TRI->isSGPRClass(RC: TRI->getPhysRegBaseClass(Reg: MO.getReg()))) {
2093 SDST = &MO;
2094 break;
2095 }
2096 }
2097 }
2098
2099 if (!SDST)
2100 return false;
2101
2102 const Register SDSTReg = SDST->getReg();
2103 auto IsHazardFn = [SDSTReg, TRI](const MachineInstr &I) {
2104 return SIInstrInfo::isSMRD(MI: I) && I.readsRegister(Reg: SDSTReg, TRI);
2105 };
2106
2107 auto IsExpiredFn = [TII, IV](const MachineInstr &MI, int) {
2108 if (TII->isSALU(MI)) {
2109 switch (MI.getOpcode()) {
2110 case AMDGPU::S_SETVSKIP:
2111 case AMDGPU::S_VERSION:
2112 case AMDGPU::S_WAITCNT_VSCNT:
2113 case AMDGPU::S_WAITCNT_VMCNT:
2114 case AMDGPU::S_WAITCNT_EXPCNT:
2115 // These instructions cannot not mitigate the hazard.
2116 return false;
2117 case AMDGPU::S_WAITCNT_LGKMCNT:
2118 // Reducing lgkmcnt count to 0 always mitigates the hazard.
2119 return (MI.getOperand(i: 1).getImm() == 0) &&
2120 (MI.getOperand(i: 0).getReg() == AMDGPU::SGPR_NULL);
2121 case AMDGPU::S_WAITCNT: {
2122 const int64_t Imm = MI.getOperand(i: 0).getImm();
2123 AMDGPU::Waitcnt Decoded =
2124 AMDGPU::decodeWaitcnt(Version: IV, Encoded: static_cast<unsigned>(Imm));
2125 // DsCnt corresponds to LGKMCnt here.
2126 return Decoded.get(T: AMDGPU::DS_CNT) == 0;
2127 }
2128 default:
2129 assert((!SIInstrInfo::isWaitcnt(MI.getOpcode()) ||
2130 MI.getOpcode() == AMDGPU::S_WAIT_IDLE) &&
2131 "unexpected wait count instruction");
2132 // SOPP instructions cannot mitigate the hazard.
2133 if (TII->isSOPP(MI))
2134 return false;
2135 // At this point the SALU can be assumed to mitigate the hazard
2136 // because either:
2137 // (a) it is independent of the at risk SMEM (breaking chain),
2138 // or
2139 // (b) it is dependent on the SMEM, in which case an appropriate
2140 // s_waitcnt lgkmcnt _must_ exist between it and the at risk
2141 // SMEM instruction.
2142 return true;
2143 }
2144 }
2145 return false;
2146 };
2147
2148 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2149 std::numeric_limits<int>::max())
2150 return false;
2151
2152 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2153 MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::SGPR_NULL)
2154 .addImm(Val: 0);
2155 return true;
2156}
2157
2158bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) {
2159 if (!ST.hasVcmpxExecWARHazard())
2160 return false;
2161 assert(!ST.hasExtendedWaitCounts());
2162
2163 if (!SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/false))
2164 return false;
2165
2166 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2167 if (!MI->modifiesRegister(Reg: AMDGPU::EXEC, TRI))
2168 return false;
2169
2170 auto IsHazardFn = [TRI](const MachineInstr &I) {
2171 if (SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true))
2172 return false;
2173 return I.readsRegister(Reg: AMDGPU::EXEC, TRI);
2174 };
2175
2176 const SIInstrInfo *TII = ST.getInstrInfo();
2177 auto IsExpiredFn = [TII, TRI](const MachineInstr &MI, int) {
2178 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false)) {
2179 if (TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst))
2180 return true;
2181 for (auto MO : MI.implicit_operands())
2182 if (MO.isDef() && TRI->isSGPRClass(RC: TRI->getPhysRegBaseClass(Reg: MO.getReg())))
2183 return true;
2184 }
2185 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2186 AMDGPU::DepCtr::decodeFieldSaSdst(
2187 Encoded: static_cast<unsigned>(MI.getOperand(i: 0).getImm())) == 0)
2188 return true;
2189 return false;
2190 };
2191
2192 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2193 std::numeric_limits<int>::max())
2194 return false;
2195
2196 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2197 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
2198 .addImm(Val: AMDGPU::DepCtr::encodeFieldSaSdst(SaSdst: 0, STI: ST));
2199 return true;
2200}
2201
2202static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF,
2203 const GCNSubtarget &ST) {
2204 if (!ST.hasLdsBranchVmemWARHazard())
2205 return false;
2206
2207 // Check if the necessary condition for the hazard is met: both LDS and VMEM
2208 // instructions need to appear in the same function.
2209 bool HasLds = false;
2210 bool HasVmem = false;
2211 for (auto &MBB : MF) {
2212 for (auto &MI : MBB) {
2213 HasLds |= SIInstrInfo::isDS(MI) || SIInstrInfo::isLDSDMA(MI);
2214 HasVmem |= SIInstrInfo::isVMEM(MI);
2215 if (HasLds && HasVmem)
2216 return true;
2217 }
2218 }
2219 return false;
2220}
2221
2222static bool isStoreCountWaitZero(const MachineInstr &I) {
2223 return I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
2224 I.getOperand(i: 0).getReg() == AMDGPU::SGPR_NULL &&
2225 !I.getOperand(i: 1).getImm();
2226}
2227
2228bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) {
2229 if (!RunLdsBranchVmemWARHazardFixup)
2230 return false;
2231
2232 assert(ST.hasLdsBranchVmemWARHazard());
2233 assert(!ST.hasExtendedWaitCounts());
2234
2235 auto IsHazardInst = [](const MachineInstr &MI) {
2236 if (SIInstrInfo::isDS(MI) || SIInstrInfo::isLDSDMA(MI))
2237 return 1;
2238 if (SIInstrInfo::isVMEM(MI))
2239 return 2;
2240 return 0;
2241 };
2242
2243 auto InstType = IsHazardInst(*MI);
2244 if (!InstType)
2245 return false;
2246
2247 auto IsExpiredFn = [&IsHazardInst](const MachineInstr &I, int) {
2248 return IsHazardInst(I) || isStoreCountWaitZero(I);
2249 };
2250
2251 auto IsHazardFn = [InstType, &IsHazardInst](const MachineInstr &I) {
2252 if (!I.isBranch())
2253 return false;
2254
2255 auto IsHazardFn = [InstType, IsHazardInst](const MachineInstr &I) {
2256 auto InstType2 = IsHazardInst(I);
2257 return InstType2 && InstType != InstType2;
2258 };
2259
2260 auto IsExpiredFn = [InstType, &IsHazardInst](const MachineInstr &I, int) {
2261 auto InstType2 = IsHazardInst(I);
2262 if (InstType == InstType2)
2263 return true;
2264
2265 return isStoreCountWaitZero(I);
2266 };
2267
2268 return ::getWaitStatesSince(IsHazard: IsHazardFn, MI: &I, IsExpired: IsExpiredFn) !=
2269 std::numeric_limits<int>::max();
2270 };
2271
2272 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2273 std::numeric_limits<int>::max())
2274 return false;
2275
2276 const SIInstrInfo *TII = ST.getInstrInfo();
2277 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2278 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_VSCNT))
2279 .addReg(RegNo: AMDGPU::SGPR_NULL, Flags: RegState::Undef)
2280 .addImm(Val: 0);
2281
2282 return true;
2283}
2284
2285bool GCNHazardRecognizer::fixLdsDirectVALUHazard(MachineInstr *MI) {
2286 if (!SIInstrInfo::isLDSDIR(MI: *MI))
2287 return false;
2288
2289 const int NoHazardWaitStates = 15;
2290 const MachineOperand *VDST = TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::vdst);
2291 const Register VDSTReg = VDST->getReg();
2292
2293 bool VisitedTrans = false;
2294 auto IsHazardFn = [this, VDSTReg, &VisitedTrans](const MachineInstr &I) {
2295 if (!SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true))
2296 return false;
2297 VisitedTrans = VisitedTrans || SIInstrInfo::isTRANS(MI: I);
2298 // Cover both WAR and WAW
2299 return I.readsRegister(Reg: VDSTReg, TRI: &TRI) || I.modifiesRegister(Reg: VDSTReg, TRI: &TRI);
2300 };
2301 auto IsExpiredFn = [&](const MachineInstr &I, int WaitStates) {
2302 if (WaitStates >= NoHazardWaitStates)
2303 return true;
2304 // Instructions which cause va_vdst==0 expire hazard
2305 return SIInstrInfo::isVMEM(MI: I) || SIInstrInfo::isDS(MI: I) ||
2306 SIInstrInfo::isEXP(MI: I);
2307 };
2308 auto GetWaitStatesFn = [](const MachineInstr &MI) {
2309 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ? 1 : 0;
2310 };
2311
2312 DenseSet<const MachineBasicBlock *> Visited;
2313 auto Count = ::getWaitStatesSince(IsHazard: IsHazardFn, MBB: MI->getParent(),
2314 I: std::next(x: MI->getReverseIterator()), WaitStates: 0,
2315 IsExpired: IsExpiredFn, Visited, GetNumWaitStates: GetWaitStatesFn);
2316
2317 // Transcendentals can execute in parallel to other VALUs.
2318 // This makes va_vdst count unusable with a mixture of VALU and TRANS.
2319 if (VisitedTrans)
2320 Count = 0;
2321
2322 MachineOperand *WaitVdstOp =
2323 TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::waitvdst);
2324 WaitVdstOp->setImm(std::min(a: Count, b: NoHazardWaitStates));
2325
2326 return true;
2327}
2328
2329bool GCNHazardRecognizer::fixLdsDirectVMEMHazard(MachineInstr *MI) {
2330 if (!SIInstrInfo::isLDSDIR(MI: *MI))
2331 return false;
2332
2333 const MachineOperand *VDST = TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::vdst);
2334 const Register VDSTReg = VDST->getReg();
2335
2336 auto IsHazardFn = [this, VDSTReg](const MachineInstr &I) {
2337 if (!SIInstrInfo::isVMEM(MI: I) && !SIInstrInfo::isDS(MI: I))
2338 return false;
2339 return I.readsRegister(Reg: VDSTReg, TRI: &TRI) || I.modifiesRegister(Reg: VDSTReg, TRI: &TRI);
2340 };
2341 bool LdsdirCanWait = ST.hasLdsWaitVMSRC();
2342 // TODO: On GFX12 the hazard should expire on S_WAIT_LOADCNT/SAMPLECNT/BVHCNT
2343 // according to the type of VMEM instruction.
2344 auto IsExpiredFn = [this, LdsdirCanWait](const MachineInstr &I, int) {
2345 return SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true) ||
2346 SIInstrInfo::isEXP(MI: I) ||
2347 (I.getOpcode() == AMDGPU::S_WAITCNT && !I.getOperand(i: 0).getImm()) ||
2348 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2349 AMDGPU::DepCtr::decodeFieldVmVsrc(
2350 Encoded: static_cast<unsigned>(I.getOperand(i: 0).getImm())) == 0) ||
2351 (LdsdirCanWait && SIInstrInfo::isLDSDIR(MI: I) &&
2352 !TII.getNamedOperand(MI: I, OperandName: AMDGPU::OpName::waitvsrc)->getImm());
2353 };
2354
2355 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2356 std::numeric_limits<int>::max())
2357 return false;
2358
2359 if (LdsdirCanWait) {
2360 TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::waitvsrc)->setImm(0);
2361 } else {
2362 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2363 MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
2364 .addImm(Val: AMDGPU::DepCtr::encodeFieldVmVsrc(VmVsrc: 0, STI: ST));
2365 }
2366
2367 return true;
2368}
2369
2370bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(MachineInstr *MI) {
2371 if (!ST.hasVALUPartialForwardingHazard())
2372 return false;
2373 assert(!ST.hasExtendedWaitCounts());
2374
2375 if (!ST.isWave64() || !SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/false))
2376 return false;
2377
2378 SmallSetVector<Register, 4> SrcVGPRs;
2379
2380 for (const MachineOperand &Use : MI->explicit_uses()) {
2381 if (Use.isReg() && TRI.isVGPR(MRI: MF.getRegInfo(), Reg: Use.getReg()))
2382 SrcVGPRs.insert(X: Use.getReg());
2383 }
2384
2385 // Only applies with >= 2 unique VGPR sources
2386 if (SrcVGPRs.size() <= 1)
2387 return false;
2388
2389 // Look for the following pattern:
2390 // Va <- VALU [PreExecPos]
2391 // intv1
2392 // Exec <- SALU [ExecPos]
2393 // intv2
2394 // Vb <- VALU [PostExecPos]
2395 // intv3
2396 // MI Va, Vb (WaitState = 0)
2397 //
2398 // Where:
2399 // intv1 + intv2 <= 2 VALUs
2400 // intv3 <= 4 VALUs
2401 //
2402 // If found, insert an appropriate S_WAITCNT_DEPCTR before MI.
2403
2404 const int Intv1plus2MaxVALUs = 2;
2405 const int Intv3MaxVALUs = 4;
2406 const int IntvMaxVALUs = 6;
2407 const int NoHazardVALUWaitStates = IntvMaxVALUs + 2;
2408
2409 struct StateType {
2410 SmallDenseMap<Register, int, 4> DefPos;
2411 int ExecPos = std::numeric_limits<int>::max();
2412 int VALUs = 0;
2413
2414 static unsigned getHashValue(const StateType &State) {
2415 hash_code H = hash_combine(args: State.ExecPos, args: State.VALUs);
2416 for (const auto &[Reg, Pos] : State.DefPos)
2417 H = hash_combine(args: H, args: Reg, args: Pos);
2418 return static_cast<unsigned>(H);
2419 }
2420 static bool isEqual(const StateType &LHS, const StateType &RHS) {
2421 return LHS.DefPos == RHS.DefPos && LHS.ExecPos == RHS.ExecPos &&
2422 LHS.VALUs == RHS.VALUs;
2423 }
2424 };
2425
2426 StateType State;
2427
2428 // This overloads expiry testing with all the hazard detection
2429 auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) {
2430 // Too many VALU states have passed
2431 if (State.VALUs > NoHazardVALUWaitStates)
2432 return HazardExpired;
2433
2434 // Instructions which cause va_vdst==0 expire hazard
2435 if (SIInstrInfo::isVMEM(MI: I) || SIInstrInfo::isDS(MI: I) ||
2436 SIInstrInfo::isEXP(MI: I) ||
2437 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2438 AMDGPU::DepCtr::decodeFieldVaVdst(
2439 Encoded: static_cast<unsigned>(I.getOperand(i: 0).getImm())) == 0))
2440 return HazardExpired;
2441
2442 // Track registers writes
2443 bool Changed = false;
2444 if (SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/false)) {
2445 for (Register Src : SrcVGPRs) {
2446 if (!State.DefPos.count(Val: Src) && I.modifiesRegister(Reg: Src, TRI: &TRI)) {
2447 State.DefPos[Src] = State.VALUs;
2448 Changed = true;
2449 }
2450 }
2451 } else if (SIInstrInfo::isSALU(MI: I)) {
2452 if (State.ExecPos == std::numeric_limits<int>::max()) {
2453 if (!State.DefPos.empty() && I.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &TRI)) {
2454 State.ExecPos = State.VALUs;
2455 Changed = true;
2456 }
2457 }
2458 }
2459
2460 // Early expiration: too many VALUs in intv3
2461 if (State.VALUs > Intv3MaxVALUs && State.DefPos.empty())
2462 return HazardExpired;
2463
2464 // Only evaluate state if something changed
2465 if (!Changed)
2466 return NoHazardFound;
2467
2468 // Determine positions of VALUs pre/post exec change
2469 if (State.ExecPos == std::numeric_limits<int>::max())
2470 return NoHazardFound;
2471
2472 int PreExecPos = std::numeric_limits<int>::max();
2473 int PostExecPos = std::numeric_limits<int>::max();
2474
2475 for (auto Entry : State.DefPos) {
2476 int DefVALUs = Entry.second;
2477 if (DefVALUs != std::numeric_limits<int>::max()) {
2478 if (DefVALUs >= State.ExecPos)
2479 PreExecPos = std::min(a: PreExecPos, b: DefVALUs);
2480 else
2481 PostExecPos = std::min(a: PostExecPos, b: DefVALUs);
2482 }
2483 }
2484
2485 // Need a VALUs post exec change
2486 if (PostExecPos == std::numeric_limits<int>::max())
2487 return NoHazardFound;
2488
2489 // Too many VALUs in intv3?
2490 int Intv3VALUs = PostExecPos;
2491 if (Intv3VALUs > Intv3MaxVALUs)
2492 return HazardExpired;
2493
2494 // Too many VALUs in intv2?
2495 int Intv2VALUs = (State.ExecPos - PostExecPos) - 1;
2496 if (Intv2VALUs > Intv1plus2MaxVALUs)
2497 return HazardExpired;
2498
2499 // Need a VALUs pre exec change
2500 if (PreExecPos == std::numeric_limits<int>::max())
2501 return NoHazardFound;
2502
2503 // Too many VALUs in intv1?
2504 int Intv1VALUs = PreExecPos - State.ExecPos;
2505 if (Intv1VALUs > Intv1plus2MaxVALUs)
2506 return HazardExpired;
2507
2508 // Too many VALUs in intv1 + intv2
2509 if (Intv1VALUs + Intv2VALUs > Intv1plus2MaxVALUs)
2510 return HazardExpired;
2511
2512 return HazardFound;
2513 };
2514 auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) {
2515 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false))
2516 State.VALUs += 1;
2517 };
2518
2519 if (!hasHazard<StateType>(InitialState: State, IsHazard: IsHazardFn, UpdateState: UpdateStateFn, InitialMBB: MI->getParent(),
2520 InitialI: std::next(x: MI->getReverseIterator())))
2521 return false;
2522
2523 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2524 MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
2525 .addImm(Val: AMDGPU::DepCtr::encodeFieldVaVdst(VaVdst: 0, STI: ST));
2526
2527 return true;
2528}
2529
2530bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) {
2531 if (!ST.hasVALUTransUseHazard())
2532 return false;
2533 assert(!ST.hasExtendedWaitCounts());
2534
2535 if (!SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/false))
2536 return false;
2537
2538 SmallSet<Register, 4> SrcVGPRs;
2539
2540 for (const MachineOperand &Use : MI->explicit_uses()) {
2541 if (Use.isReg() && TRI.isVGPR(MRI: MF.getRegInfo(), Reg: Use.getReg()))
2542 SrcVGPRs.insert(V: Use.getReg());
2543 }
2544
2545 // Look for the following pattern:
2546 // Va <- TRANS VALU
2547 // intv
2548 // MI Va (WaitState = 0)
2549 //
2550 // Where:
2551 // intv <= 5 VALUs / 1 TRANS
2552 //
2553 // If found, insert an appropriate S_WAITCNT_DEPCTR before MI.
2554
2555 const int IntvMaxVALUs = 5;
2556 const int IntvMaxTRANS = 1;
2557
2558 struct StateType {
2559 int VALUs = 0;
2560 int TRANS = 0;
2561
2562 static unsigned getHashValue(const StateType &State) {
2563 return static_cast<unsigned>(hash_combine(args: State.VALUs, args: State.TRANS));
2564 }
2565 static bool isEqual(const StateType &LHS, const StateType &RHS) {
2566 return LHS.VALUs == RHS.VALUs && LHS.TRANS == RHS.TRANS;
2567 }
2568 };
2569
2570 StateType State;
2571
2572 // This overloads expiry testing with all the hazard detection
2573 auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) {
2574 // Too many VALU states have passed
2575 if (State.VALUs > IntvMaxVALUs || State.TRANS > IntvMaxTRANS)
2576 return HazardExpired;
2577
2578 // Instructions which cause va_vdst==0 expire hazard
2579 if (SIInstrInfo::isVMEM(MI: I) || SIInstrInfo::isDS(MI: I) ||
2580 SIInstrInfo::isEXP(MI: I) ||
2581 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2582 AMDGPU::DepCtr::decodeFieldVaVdst(
2583 Encoded: static_cast<unsigned>(I.getOperand(i: 0).getImm())) == 0))
2584 return HazardExpired;
2585
2586 // Track registers writes
2587 if (SIInstrInfo::isTRANS(MI: I)) {
2588 for (Register Src : SrcVGPRs) {
2589 if (I.modifiesRegister(Reg: Src, TRI: &TRI)) {
2590 return HazardFound;
2591 }
2592 }
2593 }
2594
2595 return NoHazardFound;
2596 };
2597 auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) {
2598 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false))
2599 State.VALUs += 1;
2600 if (SIInstrInfo::isTRANS(MI))
2601 State.TRANS += 1;
2602 };
2603
2604 if (!hasHazard<StateType>(InitialState: State, IsHazard: IsHazardFn, UpdateState: UpdateStateFn, InitialMBB: MI->getParent(),
2605 InitialI: std::next(x: MI->getReverseIterator())))
2606 return false;
2607
2608 // Hazard is observed - insert a wait on va_dst counter to ensure hazard is
2609 // avoided.
2610 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2611 MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
2612 .addImm(Val: AMDGPU::DepCtr::encodeFieldVaVdst(VaVdst: 0, STI: ST));
2613
2614 return true;
2615}
2616
2617bool GCNHazardRecognizer::fixVALUTransCoexecutionHazards(MachineInstr *MI) {
2618 if (!ST.hasTransCoexecutionHazard() || // Coexecution disabled.
2619 !SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true) ||
2620 SIInstrInfo::isTRANS(MI: *MI))
2621 return false;
2622
2623 const SIInstrInfo *TII = ST.getInstrInfo();
2624 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2625
2626 auto IsTransHazardFn = [MI, TII, TRI](const MachineInstr &I) {
2627 if (!SIInstrInfo::isTRANS(MI: I))
2628 return false;
2629
2630 // RAW: Trans(I) writes, VALU(MI) reads.
2631 Register TransDef = TII->getNamedOperand(MI: I, OperandName: AMDGPU::OpName::vdst)->getReg();
2632 for (const MachineOperand &ValuUse : MI->explicit_uses()) {
2633 if (ValuUse.isReg() && TRI->regsOverlap(RegA: TransDef, RegB: ValuUse.getReg()))
2634 return true;
2635 }
2636
2637 // WAR: Trans(I) reads, VALU(MI) writes. Use modifiesRegister for
2638 // multi-dest VALUs.
2639 for (const MachineOperand &TransUse : I.explicit_uses()) {
2640 if (TransUse.isReg() && MI->modifiesRegister(Reg: TransUse.getReg(), TRI))
2641 return true;
2642 }
2643
2644 return false;
2645 };
2646
2647 auto IsExpiredFn = [](const MachineInstr &I, int) {
2648 return SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true);
2649 };
2650
2651 const int HasVALU = std::numeric_limits<int>::max();
2652 if (::getWaitStatesSince(IsHazard: IsTransHazardFn, MI, IsExpired: IsExpiredFn) == HasVALU)
2653 return false;
2654
2655 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::V_NOP_e32));
2656 return true;
2657}
2658
2659bool GCNHazardRecognizer::fixWMMAHazards(MachineInstr *MI) {
2660 if (!SIInstrInfo::isWMMA(MI: *MI) && !SIInstrInfo::isSWMMAC(MI: *MI))
2661 return false;
2662
2663 const SIInstrInfo *TII = ST.getInstrInfo();
2664 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2665
2666 auto IsHazardFn = [MI, TII, TRI, this](const MachineInstr &I) {
2667 if (!SIInstrInfo::isWMMA(MI: I) && !SIInstrInfo::isSWMMAC(MI: I))
2668 return false;
2669
2670 // Src0(matrix A) or Src1(matrix B) of the current wmma instruction overlaps
2671 // with the dest(matrix D) of the previous wmma.
2672 const Register CurSrc0Reg =
2673 TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src0)->getReg();
2674 const Register CurSrc1Reg =
2675 TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src1)->getReg();
2676
2677 const Register PrevDstReg =
2678 TII->getNamedOperand(MI: I, OperandName: AMDGPU::OpName::vdst)->getReg();
2679
2680 if (TRI->regsOverlap(RegA: PrevDstReg, RegB: CurSrc0Reg) ||
2681 TRI->regsOverlap(RegA: PrevDstReg, RegB: CurSrc1Reg)) {
2682 return true;
2683 }
2684
2685 // GFX12+ allows overlap of matrix C with PrevDstReg (hardware will stall)
2686 // but Index can't overlap with PrevDstReg.
2687 if (AMDGPU::isGFX12Plus(STI: ST)) {
2688 if (SIInstrInfo::isSWMMAC(MI: *MI)) {
2689 const Register CurIndex =
2690 TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src2)->getReg();
2691 if (TRI->regsOverlap(RegA: PrevDstReg, RegB: CurIndex))
2692 return true;
2693 }
2694 return false;
2695 }
2696
2697 return false;
2698 };
2699
2700 auto IsExpiredFn = [](const MachineInstr &I, int) {
2701 return SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true);
2702 };
2703
2704 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2705 std::numeric_limits<int>::max())
2706 return false;
2707
2708 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::V_NOP_e32));
2709
2710 return true;
2711}
2712
2713static bool isCoexecutableVALUInst(const MachineInstr &MI) {
2714 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false) &&
2715 !SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI);
2716}
2717
2718// Classify XDL WMMA instructions into co-execution hazard categories
2719// (Refer to SPG 4.6.12.1), mainly based on instruction latency.
2720//
2721// Category 0: WMMA with Latency 8
2722// WMMA_*F16, WMMA_*BF16
2723// WMMA_*_16X16X128_{FP8,BF8}
2724// WMMA_*F8F6F4 if SRCA & SRCB are not both F4
2725//
2726// Category 1: WMMA Latency 16
2727// WMMA_IU8
2728//
2729// Category 2: SWMMAC with Latency 8
2730// SWMMAC_*F16, SWMMAC_*BF16,
2731// SWMMAC_*FP8FP8
2732// SWMMAC_*BF8FP8
2733// SWMMAC_*FP8BF8
2734// SWMMAC_*BF8BF8
2735//
2736// Category 3: SWMMAC with Latency 16
2737// SWMMAC_IU8
2738//
2739// Category 4: 16 Pass GFX1251 WMMA with latency 16
2740// V_WMMA_*_16X16X32_{F16,BF16}
2741// V_WMMA_{F32,F16}_16X16X64_{FP8,BF8}*
2742// V_WMMA_F32_16x16x128_F8F6F4 (F4 only)
2743// V_SWMMAC_*_16X16X64_{F16,BF16}
2744// V_SWMMAC_{F32,F16}_16X16X128_{FP8,BF8}*
2745//
2746// Category 5: 32 Pass GFX1251 WMMA with latency 32
2747// V_WMMA_F32_16x16x128_F8F6F4 (not all F4)
2748// V_WMMA_{F32,F16}_16X16X128_{FP8,BF8}*
2749// V_WMMA_F32_32X16X128_F4
2750// V_WMMA_I32_16X16X64_IU8
2751// V_WMMA_I32_16X16X64_IU8
2752//
2753// Category 6: gfx1250 WMMA with Latency 4 (one co-execution slot)
2754// WMMA_*_16X16X64_{FP8,BF8}
2755// WMMA_*F8F6F4 if SRCA & SRCB are both F4
2756static unsigned getWMMAHazardInstInCategory(const MachineInstr &MI,
2757 const SIInstrInfo *TII,
2758 const TargetSchedModel &SchedModel,
2759 const GCNSubtarget &ST) {
2760 assert(TII->isXDLWMMA(MI) && "must be xdl wmma");
2761 bool IsSWMMAC = SIInstrInfo::isSWMMAC(MI);
2762 bool IsLowestRateWMMA = ST.hasGFX125xLowestRateWMMA();
2763 unsigned Category = 0;
2764
2765 unsigned Latency = SchedModel.computeInstrLatency(MI: &MI);
2766 switch (Latency) {
2767 case 4:
2768 // Dense 4-cycle WMMA (gfx1250 16x16x64 FP8/BF8 and f8f6f4 with both
2769 // inputs F4). One co-execution slot; there is no 4-cycle SWMMAC.
2770 assert(!IsSWMMAC && "no 4-cycle SWMMAC expected");
2771 Category = 6;
2772 break;
2773 case 8:
2774 Category = IsSWMMAC ? 2 : 0;
2775 break;
2776 case 16:
2777 Category = IsLowestRateWMMA ? 4 : (IsSWMMAC ? 3 : 1);
2778 break;
2779 case 32:
2780 assert(IsLowestRateWMMA && "latency 32 is not expected");
2781 Category = 5;
2782 break;
2783 default:
2784 llvm_unreachable("unexpected xdl wmma latency");
2785 } // end switch.
2786
2787 return Category;
2788}
2789
2790int GCNHazardRecognizer::checkWMMACoexecutionHazards(MachineInstr *MI) const {
2791 if (!ST.hasWMMACoexecutionHazards())
2792 return 0;
2793
2794 const SIInstrInfo *TII = ST.getInstrInfo();
2795 if (!TII->isXDLWMMA(MI: *MI) && !isCoexecutableVALUInst(MI: *MI))
2796 return 0;
2797
2798 // WaitStates here is the number of V_NOPs or unrelated VALU instructions must
2799 // be in between the first WMMA and the second instruction to cover the hazard
2800 // (WMMAWaitStates if the second is also a WMMA, VALUWaitStates if the second
2801 // is a VALU). Refer to SPG 4.6.12.1. "Requirements for WMMA data hazards" for
2802 // numbers, which depends on the category of the first WMMA.
2803 const int WMMAWaitStates[] = {5, 9, 3, 5, 9, 17, 2};
2804 const int VALUWaitStates[] = {4, 8, 2, 4, 8, 16, 1};
2805 const bool IsWMMA = TII->isXDLWMMA(MI: *MI);
2806 const bool IsLowestRateWMMA = ST.hasGFX125xLowestRateWMMA();
2807 ArrayRef<int> RequiredWaitStates =
2808 IsWMMA ? ArrayRef<int>(WMMAWaitStates) : ArrayRef<int>(VALUWaitStates);
2809 const int MaxWaitStates =
2810 IsWMMA ? (IsLowestRateWMMA ? 17 : 9) : (IsLowestRateWMMA ? 16 : 8);
2811
2812 auto WindowFor = [=](const MachineInstr &I) -> std::optional<int> {
2813 if (!TII->isXDLWMMA(MI: I))
2814 return std::nullopt;
2815 if (IsWMMA ? !hasWMMAToWMMARegOverlap(WMMA: I, MI: *MI)
2816 : !hasWMMAToVALURegOverlap(WMMA: I, MI: *MI))
2817 return std::nullopt;
2818 return RequiredWaitStates[getWMMAHazardInstInCategory(MI: I, TII, SchedModel: TSchedModel,
2819 ST)];
2820 };
2821
2822 return getMaxVALUWindowDeficit(MaxWindow: MaxWaitStates, WindowFor);
2823}
2824
2825bool GCNHazardRecognizer::hasWMMAToWMMARegOverlap(
2826 const MachineInstr &WMMA, const MachineInstr &MI) const {
2827 Register D0 = TII.getNamedOperand(MI: WMMA, OperandName: AMDGPU::OpName::vdst)->getReg();
2828 Register A1 = TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src0)->getReg();
2829 Register B1 = TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src1)->getReg();
2830
2831 // WMMA0 writes (D0), WMMA1 reads (A1/B1/Idx1).
2832 if (TRI.regsOverlap(RegA: D0, RegB: A1) || TRI.regsOverlap(RegA: D0, RegB: B1))
2833 return true;
2834
2835 if (SIInstrInfo::isSWMMAC(MI)) {
2836 Register Idx1 = TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src2)->getReg();
2837 if (TRI.regsOverlap(RegA: D0, RegB: Idx1))
2838 return true;
2839 }
2840 return false;
2841}
2842
2843bool GCNHazardRecognizer::hasWMMAToVALURegOverlap(
2844 const MachineInstr &WMMA, const MachineInstr &MI) const {
2845 // WMMA writes, VALU reads.
2846 Register D0 = TII.getNamedOperand(MI: WMMA, OperandName: AMDGPU::OpName::vdst)->getReg();
2847 for (const MachineOperand &ValuUse : MI.explicit_uses()) {
2848 if (ValuUse.isReg() && TRI.regsOverlap(RegA: D0, RegB: ValuUse.getReg()))
2849 return true;
2850 }
2851
2852 // WMMA reads or writes, VALU writes.
2853 Register A0 = TII.getNamedOperand(MI: WMMA, OperandName: AMDGPU::OpName::src0)->getReg();
2854 Register B0 = TII.getNamedOperand(MI: WMMA, OperandName: AMDGPU::OpName::src1)->getReg();
2855 SmallVector<Register, 4> WMMARegs({D0, A0, B0});
2856
2857 const MachineOperand *Src2 = TII.getNamedOperand(MI: WMMA, OperandName: AMDGPU::OpName::src2);
2858 if (Src2->isReg())
2859 WMMARegs.push_back(Elt: Src2->getReg());
2860
2861 for (const MachineOperand &ValuDef : MI.defs()) {
2862 Register VDstReg = ValuDef.getReg();
2863 for (Register WMMAReg : WMMARegs) {
2864 if (TRI.regsOverlap(RegA: VDstReg, RegB: WMMAReg))
2865 return true;
2866 }
2867 }
2868 return false;
2869}
2870
2871bool GCNHazardRecognizer::isCoexecutionHazardFor(const MachineInstr &I,
2872 const MachineInstr &MI) const {
2873 // I is the potential WMMA hazard source, MI is the instruction being checked
2874 // for hazard.
2875 if (!TII.isXDLWMMA(MI: I))
2876 return false;
2877
2878 // Dispatch based on MI type
2879 if (TII.isXDLWMMA(MI))
2880 return hasWMMAToWMMARegOverlap(WMMA: I, MI);
2881 if (isCoexecutableVALUInst(MI))
2882 return hasWMMAToVALURegOverlap(WMMA: I, MI);
2883
2884 return false;
2885}
2886
2887bool GCNHazardRecognizer::hasWMMAHazardInLoop(MachineLoop *L, MachineInstr *MI,
2888 bool IncludeSubloops) {
2889 // Scan loop for any WMMA that hazards MI.
2890 // TODO: Avoid full loop scan when WMMA is beyond VALU distance.
2891 for (MachineBasicBlock *MBB : L->getBlocks()) {
2892 if (!IncludeSubloops && MLI->getLoopFor(BB: MBB) != L)
2893 continue;
2894 for (MachineInstr &I : *MBB) {
2895 if (&I == MI)
2896 continue;
2897 if (isCoexecutionHazardFor(I, MI: *MI))
2898 return true;
2899 }
2900 }
2901 return false;
2902}
2903
2904bool GCNHazardRecognizer::tryHoistWMMAVnopsFromLoop(MachineInstr *MI,
2905 int WaitStatesNeeded) {
2906 if (!MLI)
2907 return false;
2908
2909 MachineLoop *L = MLI->getLoopFor(BB: MI->getParent());
2910 if (!L) {
2911 ++NumWMMAHoistingBailed;
2912 return false;
2913 }
2914
2915 // If innermost loop has WMMA hazard, we can't hoist at all
2916 if (hasWMMAHazardInLoop(L, MI)) {
2917 ++NumWMMAHoistingBailed;
2918 return false;
2919 }
2920
2921 // Find outermost loop with no internal hazard
2922 MachineLoop *TargetLoop = L;
2923 while (MachineLoop *Parent = TargetLoop->getParentLoop()) {
2924 if (hasWMMAHazardInLoop(L: Parent, MI, IncludeSubloops: false))
2925 break; // Parent has hazard in its own blocks, stop here
2926 TargetLoop = Parent; // Safe to hoist further out
2927 }
2928
2929 // Need valid preheader to insert V_NOPs
2930 MachineBasicBlock *Preheader = TargetLoop->getLoopPreheader();
2931 if (!Preheader) {
2932 ++NumWMMAHoistingBailed;
2933 return false;
2934 }
2935
2936 LLVM_DEBUG(dbgs() << "WMMA V_NOP Hoisting: Moving " << WaitStatesNeeded
2937 << " V_NOPs from loop to " << printMBBReference(*Preheader)
2938 << "\n");
2939
2940 emitVNops(MBB&: *Preheader, InsertPt: Preheader->getFirstTerminator(), WaitStatesNeeded,
2941 /*IsHoisting=*/true);
2942 NumWMMANopsHoisted += WaitStatesNeeded;
2943 return true;
2944}
2945
2946bool GCNHazardRecognizer::fixWMMACoexecutionHazards(MachineInstr *MI) {
2947 int WaitStatesNeeded = checkWMMACoexecutionHazards(MI);
2948 if (WaitStatesNeeded <= 0)
2949 return false;
2950
2951 if (ST.getCLOpts().wmma_vnop_hoisting &&
2952 tryHoistWMMAVnopsFromLoop(MI, WaitStatesNeeded))
2953 return true;
2954
2955 emitVNops(MBB&: *MI->getParent(), InsertPt: MI->getIterator(), WaitStatesNeeded);
2956 return true;
2957}
2958
2959bool GCNHazardRecognizer::fixShift64HighRegBug(MachineInstr *MI) {
2960 if (!ST.hasShift64HighRegBug())
2961 return false;
2962 assert(!ST.hasExtendedWaitCounts());
2963
2964 switch (MI->getOpcode()) {
2965 default:
2966 return false;
2967 case AMDGPU::V_LSHLREV_B64_e64:
2968 case AMDGPU::V_LSHRREV_B64_e64:
2969 case AMDGPU::V_ASHRREV_I64_e64:
2970 break;
2971 }
2972
2973 MachineOperand *Amt = TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src0);
2974 if (!Amt->isReg())
2975 return false;
2976
2977 Register AmtReg = Amt->getReg();
2978 const MachineRegisterInfo &MRI = MF.getRegInfo();
2979 // Check if this is a last VGPR in the allocation block.
2980 if (!TRI.isVGPR(MRI, Reg: AmtReg) || ((AmtReg - AMDGPU::VGPR0) & 7) != 7)
2981 return false;
2982
2983 if (AmtReg != AMDGPU::VGPR255 && MRI.isPhysRegUsed(PhysReg: AmtReg + 1))
2984 return false;
2985
2986 assert(ST.needsAlignedVGPRs());
2987 static_assert(AMDGPU::VGPR0 + 1 == AMDGPU::VGPR1);
2988
2989 const DebugLoc &DL = MI->getDebugLoc();
2990 MachineBasicBlock *MBB = MI->getParent();
2991 MachineOperand *Src1 = TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src1);
2992
2993 // In:
2994 //
2995 // Dst = shiftrev64 Amt, Src1
2996 //
2997 // if Dst!=Src1 then avoid the bug with:
2998 //
2999 // Dst.sub0 = Amt
3000 // Dst = shift64 Dst.sub0, Src1
3001
3002 Register DstReg = MI->getOperand(i: 0).getReg();
3003 if (!Src1->isReg() || Src1->getReg() != DstReg) {
3004 Register DstLo = TRI.getSubReg(Reg: DstReg, Idx: AMDGPU::sub0);
3005 runOnInstruction(
3006 MI: BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstLo).add(MO: *Amt));
3007 Amt->setReg(DstLo);
3008 Amt->setIsKill(true);
3009 return true;
3010 }
3011
3012 bool Overlapped = MI->modifiesRegister(Reg: AmtReg, TRI: &TRI);
3013 Register NewReg;
3014 for (MCRegister Reg : Overlapped ? AMDGPU::VReg_64_Align2RegClass
3015 : AMDGPU::VGPR_32RegClass) {
3016 if (!MI->modifiesRegister(Reg, TRI: &TRI) && !MI->readsRegister(Reg, TRI: &TRI)) {
3017 NewReg = Reg;
3018 break;
3019 }
3020 }
3021
3022 Register NewAmt = Overlapped ? (Register)TRI.getSubReg(Reg: NewReg, Idx: AMDGPU::sub1)
3023 : NewReg;
3024 Register NewAmtLo;
3025
3026 if (Overlapped)
3027 NewAmtLo = TRI.getSubReg(Reg: NewReg, Idx: AMDGPU::sub0);
3028
3029 // Insert a full wait count because found register might be pending a wait.
3030 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT))
3031 .addImm(Val: 0);
3032
3033 // Insert V_SWAP_B32 instruction(s) and run hazard recognizer on them.
3034 if (Overlapped)
3035 runOnInstruction(
3036 MI: BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SWAP_B32), DestReg: NewAmtLo)
3037 .addDef(RegNo: AmtReg - 1)
3038 .addReg(RegNo: AmtReg - 1, Flags: RegState::Undef)
3039 .addReg(RegNo: NewAmtLo, Flags: RegState::Undef));
3040 runOnInstruction(MI: BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SWAP_B32), DestReg: NewAmt)
3041 .addDef(RegNo: AmtReg)
3042 .addReg(RegNo: AmtReg, Flags: RegState::Undef)
3043 .addReg(RegNo: NewAmt, Flags: RegState::Undef));
3044
3045 // Instructions emitted after the current instruction will be processed by the
3046 // parent loop of the hazard recognizer in a natural way.
3047 BuildMI(BB&: *MBB, I: std::next(x: MI->getIterator()), MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SWAP_B32),
3048 DestReg: AmtReg)
3049 .addDef(RegNo: NewAmt)
3050 .addReg(RegNo: NewAmt)
3051 .addReg(RegNo: AmtReg);
3052 if (Overlapped)
3053 BuildMI(BB&: *MBB, I: std::next(x: MI->getIterator()), MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SWAP_B32),
3054 DestReg: AmtReg - 1)
3055 .addDef(RegNo: NewAmtLo)
3056 .addReg(RegNo: NewAmtLo)
3057 .addReg(RegNo: AmtReg - 1);
3058
3059 // Re-running hazard recognizer on the modified instruction is not necessary,
3060 // inserted V_SWAP_B32 has already both read and write new registers so
3061 // hazards related to these register has already been handled.
3062 Amt->setReg(NewAmt);
3063 Amt->setIsKill(false);
3064 // We do not update liveness, so verifier may see it as undef.
3065 Amt->setIsUndef();
3066 if (Overlapped) {
3067 MI->getOperand(i: 0).setReg(NewReg);
3068 Src1->setReg(NewReg);
3069 Src1->setIsKill(false);
3070 Src1->setIsUndef();
3071 }
3072
3073 return true;
3074}
3075
3076int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) const {
3077 int NSAtoVMEMWaitStates = 1;
3078
3079 if (!ST.hasNSAtoVMEMBug())
3080 return 0;
3081
3082 if (!SIInstrInfo::isMUBUF(MI: *MI) && !SIInstrInfo::isMTBUF(MI: *MI))
3083 return 0;
3084
3085 const SIInstrInfo *TII = ST.getInstrInfo();
3086 const auto *Offset = TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::offset);
3087 if (!Offset || (Offset->getImm() & 6) == 0)
3088 return 0;
3089
3090 auto IsHazardFn = [TII](const MachineInstr &I) {
3091 if (!SIInstrInfo::isMIMG(MI: I))
3092 return false;
3093 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc: I.getOpcode());
3094 return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA &&
3095 TII->getInstSizeInBytes(MI: I) >= 16;
3096 };
3097
3098 return NSAtoVMEMWaitStates - getWaitStatesSince(IsHazard: IsHazardFn, Limit: 1);
3099}
3100
3101int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(
3102 MachineInstr *MI) const {
3103 int FPAtomicToDenormModeWaitStates = 3;
3104
3105 if (!ST.hasFPAtomicToDenormModeHazard())
3106 return 0;
3107 assert(!ST.hasExtendedWaitCounts());
3108
3109 if (MI->getOpcode() != AMDGPU::S_DENORM_MODE)
3110 return 0;
3111
3112 auto IsHazardFn = [](const MachineInstr &I) {
3113 if (!SIInstrInfo::isVMEM(MI: I))
3114 return false;
3115 return SIInstrInfo::isFPAtomic(MI: I);
3116 };
3117
3118 auto IsExpiredFn = [](const MachineInstr &MI, int WaitStates) {
3119 if (WaitStates >= 3 || SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
3120 return true;
3121
3122 return SIInstrInfo::isWaitcnt(Opcode: MI.getOpcode());
3123 };
3124
3125 return FPAtomicToDenormModeWaitStates -
3126 ::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn);
3127}
3128
3129int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) const {
3130 assert(SIInstrInfo::isMAI(*MI));
3131
3132 return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI);
3133}
3134
3135int GCNHazardRecognizer::checkMFMAPadding(MachineInstr *MI) const {
3136 // Early exit if no padding is requested.
3137 if (MFMAPaddingRatio == 0)
3138 return 0;
3139
3140 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
3141 if (!SIInstrInfo::isMFMA(MI: *MI) || MFI->getOccupancy() < 2)
3142 return 0;
3143
3144 int NeighborMFMALatency = 0;
3145 auto IsNeighboringMFMA = [&NeighborMFMALatency,
3146 this](const MachineInstr &MI) {
3147 if (!SIInstrInfo::isMFMA(MI))
3148 return false;
3149
3150 NeighborMFMALatency = this->getMFMAPipelineWaitStates(MI);
3151 return true;
3152 };
3153
3154 const int MaxMFMAPipelineWaitStates = 16;
3155 int WaitStatesSinceNeighborMFMA =
3156 getWaitStatesSince(IsHazard: IsNeighboringMFMA, Limit: MaxMFMAPipelineWaitStates);
3157
3158 int NeighborMFMAPaddingNeeded =
3159 (NeighborMFMALatency * MFMAPaddingRatio / 100) -
3160 WaitStatesSinceNeighborMFMA;
3161
3162 return std::max(a: 0, b: NeighborMFMAPaddingNeeded);
3163}
3164
3165int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) const {
3166 int WaitStatesNeeded = 0;
3167 unsigned Opc = MI->getOpcode();
3168
3169 auto IsVALUFn = [](const MachineInstr &MI) {
3170 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false) || MI.isInlineAsm();
3171 };
3172
3173 if (Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) { // MFMA or v_accvgpr_write
3174 const int LegacyVALUWritesVGPRWaitStates = 2;
3175 const int VALUWritesExecWaitStates = 4;
3176 const int MaxWaitStates = 4;
3177
3178 int WaitStatesNeededForUse = VALUWritesExecWaitStates -
3179 getWaitStatesSinceDef(Reg: AMDGPU::EXEC, IsHazardDef: IsVALUFn, Limit: MaxWaitStates);
3180 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3181
3182 if (WaitStatesNeeded < MaxWaitStates) {
3183 for (const MachineOperand &Use : MI->explicit_uses()) {
3184 const int MaxWaitStates = 2;
3185
3186 if (!Use.isReg() || !TRI.isVGPR(MRI: MF.getRegInfo(), Reg: Use.getReg()))
3187 continue;
3188
3189 int WaitStatesNeededForUse = LegacyVALUWritesVGPRWaitStates -
3190 getWaitStatesSinceDef(Reg: Use.getReg(), IsHazardDef: IsVALUFn, Limit: MaxWaitStates);
3191 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3192
3193 if (WaitStatesNeeded == MaxWaitStates)
3194 break;
3195 }
3196 }
3197 }
3198
3199 for (const MachineOperand &Op : MI->explicit_operands()) {
3200 if (!Op.isReg() || !TRI.isAGPR(MRI: MF.getRegInfo(), Reg: Op.getReg()))
3201 continue;
3202
3203 if (Op.isDef() && Opc != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
3204 continue;
3205
3206 const int MFMAWritesAGPROverlappedSrcABWaitStates = 4;
3207 const int MFMAWritesAGPROverlappedSrcCWaitStates = 2;
3208 const int MFMA4x4WritesAGPRAccVgprReadWaitStates = 4;
3209 const int MFMA16x16WritesAGPRAccVgprReadWaitStates = 10;
3210 const int MFMA32x32WritesAGPRAccVgprReadWaitStates = 18;
3211 const int MFMA4x4WritesAGPRAccVgprWriteWaitStates = 1;
3212 const int MFMA16x16WritesAGPRAccVgprWriteWaitStates = 7;
3213 const int MFMA32x32WritesAGPRAccVgprWriteWaitStates = 15;
3214 const int MaxWaitStates = 18;
3215 Register Reg = Op.getReg();
3216 unsigned HazardDefLatency = 0;
3217
3218 auto IsOverlappedMFMAFn = [Reg, &HazardDefLatency,
3219 this](const MachineInstr &MI) {
3220 if (!SIInstrInfo::isMFMA(MI))
3221 return false;
3222 Register DstReg = MI.getOperand(i: 0).getReg();
3223 if (DstReg == Reg)
3224 return false;
3225 HazardDefLatency =
3226 std::max(a: HazardDefLatency, b: TSchedModel.computeInstrLatency(MI: &MI));
3227 return TRI.regsOverlap(RegA: DstReg, RegB: Reg);
3228 };
3229
3230 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsHazardDef: IsOverlappedMFMAFn,
3231 Limit: MaxWaitStates);
3232 int NeedWaitStates = MFMAWritesAGPROverlappedSrcABWaitStates;
3233 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2);
3234 int OpNo = Op.getOperandNo();
3235 if (OpNo == SrcCIdx) {
3236 NeedWaitStates = MFMAWritesAGPROverlappedSrcCWaitStates;
3237 } else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) {
3238 switch (HazardDefLatency) {
3239 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprReadWaitStates;
3240 break;
3241 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprReadWaitStates;
3242 break;
3243 case 16: [[fallthrough]];
3244 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprReadWaitStates;
3245 break;
3246 }
3247 } else if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
3248 switch (HazardDefLatency) {
3249 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprWriteWaitStates;
3250 break;
3251 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprWriteWaitStates;
3252 break;
3253 case 16: [[fallthrough]];
3254 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprWriteWaitStates;
3255 break;
3256 }
3257 }
3258
3259 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
3260 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3261
3262 if (WaitStatesNeeded == MaxWaitStates)
3263 return WaitStatesNeeded; // Early exit.
3264
3265 auto IsAccVgprWriteFn = [Reg, this](const MachineInstr &MI) {
3266 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
3267 return false;
3268 Register DstReg = MI.getOperand(i: 0).getReg();
3269 return TRI.regsOverlap(RegA: Reg, RegB: DstReg);
3270 };
3271
3272 const int AccVGPRWriteMFMAReadSrcCWaitStates = 1;
3273 const int AccVGPRWriteMFMAReadSrcABWaitStates = 3;
3274 const int AccVGPRWriteAccVgprReadWaitStates = 3;
3275 NeedWaitStates = AccVGPRWriteMFMAReadSrcABWaitStates;
3276 if (OpNo == SrcCIdx)
3277 NeedWaitStates = AccVGPRWriteMFMAReadSrcCWaitStates;
3278 else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64)
3279 NeedWaitStates = AccVGPRWriteAccVgprReadWaitStates;
3280
3281 WaitStatesNeededForUse = NeedWaitStates -
3282 getWaitStatesSinceDef(Reg, IsHazardDef: IsAccVgprWriteFn, Limit: MaxWaitStates);
3283 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3284
3285 if (WaitStatesNeeded == MaxWaitStates)
3286 return WaitStatesNeeded; // Early exit.
3287 }
3288
3289 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
3290 const int MFMA4x4ReadSrcCAccVgprWriteWaitStates = 0;
3291 const int MFMA16x16ReadSrcCAccVgprWriteWaitStates = 5;
3292 const int MFMA32x32ReadSrcCAccVgprWriteWaitStates = 13;
3293 const int MaxWaitStates = 13;
3294 Register DstReg = MI->getOperand(i: 0).getReg();
3295 unsigned HazardDefLatency = 0;
3296
3297 auto IsSrcCMFMAFn = [DstReg, &HazardDefLatency,
3298 this](const MachineInstr &MI) {
3299 if (!SIInstrInfo::isMFMA(MI))
3300 return false;
3301 Register Reg = TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src2)->getReg();
3302 HazardDefLatency =
3303 std::max(a: HazardDefLatency, b: TSchedModel.computeInstrLatency(MI: &MI));
3304 return TRI.regsOverlap(RegA: Reg, RegB: DstReg);
3305 };
3306
3307 int WaitStatesSince = getWaitStatesSince(IsHazard: IsSrcCMFMAFn, Limit: MaxWaitStates);
3308 int NeedWaitStates;
3309 switch (HazardDefLatency) {
3310 case 2: NeedWaitStates = MFMA4x4ReadSrcCAccVgprWriteWaitStates;
3311 break;
3312 case 8: NeedWaitStates = MFMA16x16ReadSrcCAccVgprWriteWaitStates;
3313 break;
3314 case 16: [[fallthrough]];
3315 default: NeedWaitStates = MFMA32x32ReadSrcCAccVgprWriteWaitStates;
3316 break;
3317 }
3318
3319 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSince;
3320 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3321 }
3322
3323 // Pad neighboring MFMA with noops for better inter-wave performance.
3324 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: checkMFMAPadding(MI));
3325
3326 return WaitStatesNeeded;
3327}
3328
3329/// One MFMA can be written with up to four opcodes that differ only in how vdst
3330/// and src2 are encoded: both are either AGPRs or VGPRs, and the mac form ties
3331/// vdst to src2 instead of taking them as separate operands. \returns the AGPR,
3332/// non-mac opcode, so that every form of the same MFMA maps to one value.
3333static unsigned getMFMANonMacAGPRFormOp(unsigned Opc) {
3334 if (int NonMacOp = AMDGPU::getMFMAEarlyClobberOp(Opcode: Opc); NonMacOp != -1)
3335 Opc = NonMacOp;
3336 if (int AGPROp = AMDGPU::getAGPRFormOp(Opcode: Opc); AGPROp != -1)
3337 Opc = AGPROp;
3338 return Opc;
3339}
3340
3341/// \returns true if \p Opc0 and \p Opc1 are the same MFMA, ignoring the mac
3342/// form and whether vdst/src2 are AGPRs or VGPRs.
3343static bool isSameMFMA(unsigned Opc0, unsigned Opc1) {
3344 return getMFMANonMacAGPRFormOp(Opc: Opc0) == getMFMANonMacAGPRFormOp(Opc: Opc1);
3345}
3346
3347static int
3348GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates(int NumPasses,
3349 bool IsGFX950) {
3350 // xdl def cycles | gfx940 | gfx950
3351 // 2 pass | 3 4
3352 // 4 pass | 5 6
3353 // 8 pass | 9 10
3354 // 16 pass | 17 18
3355 return NumPasses + 1 + IsGFX950;
3356}
3357
3358static int
3359GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates(int NumPasses,
3360 bool IsGFX950) {
3361 // xdl def cycles | gfx940 | gfx950
3362 // 2 pass | 3 3
3363 // 4 pass | 5 6
3364 // 8 pass | 9 10
3365 // 16 pass | 17 18
3366 return NumPasses + 1 + (NumPasses != 2 && IsGFX950);
3367}
3368
3369static int
3370GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates(int NumPasses) {
3371 // 2 pass -> 2
3372 // 4 pass -> 4
3373 // 8 pass -> 8
3374 // 16 pass -> 16
3375 return NumPasses;
3376}
3377
3378static int
3379GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses) {
3380 // 2 pass -> 4
3381 // 4 pass -> 6
3382 // 8 pass -> 10
3383 // 16 pass -> 18
3384 return NumPasses + 2;
3385}
3386
3387static int GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses,
3388 bool IsGFX950) {
3389 // xdl def cycles | gfx942 | gfx950
3390 // 2 pass | 5 5
3391 // 4 pass | 7 8
3392 // 8 pass | 11 12
3393 // 16 pass | 19 20
3394 return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
3395}
3396
3397int GCNHazardRecognizer::getMFMAOverlappedSrcCWaitStates(
3398 const MachineInstr *Reader, const MachineInstr *Writer) const {
3399 constexpr int SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates = 2;
3400 constexpr int SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates = 8;
3401 constexpr int SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates = 16;
3402 constexpr int SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates = 3;
3403 constexpr int SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates = 9;
3404 constexpr int SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates = 17;
3405 constexpr int DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 9;
3406 constexpr int GFX950_DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 17;
3407 constexpr int DMFMA4x4WritesVGPROverlappedSrcCWaitStates = 4;
3408
3409 // An XDL read of a non-XDL result needs no wait states. DGEMM is never XDL,
3410 // so this also covers the f64 writers handled below.
3411 if (TII.isXDL(MI: *Reader) && !TII.isXDL(MI: *Writer))
3412 return 0;
3413
3414 switch (Writer->getOpcode()) {
3415 case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
3416 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
3417 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
3418 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
3419 return ST.hasGFX950Insts()
3420 ? GFX950_DMFMA16x16WritesVGPROverlappedSrcCWaitStates
3421 : DMFMA16x16WritesVGPROverlappedSrcCWaitStates;
3422 case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
3423 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
3424 return DMFMA4x4WritesVGPROverlappedSrcCWaitStates;
3425 default:
3426 break;
3427 }
3428
3429 int NumPasses = TSchedModel.computeInstrLatency(MI: Writer);
3430 if (ST.hasGFX940Insts()) {
3431 if (!TII.isXDL(MI: *Writer))
3432 return GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates(
3433 NumPasses);
3434 return TII.isXDL(MI: *Reader)
3435 ? GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates(
3436 NumPasses, IsGFX950: ST.hasGFX950Insts())
3437 : GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates(
3438 NumPasses, IsGFX950: ST.hasGFX950Insts());
3439 }
3440
3441 bool IsDGEMM = SIInstrInfo::isDGEMM(Opcode: Reader->getOpcode());
3442 switch (NumPasses) {
3443 case 2:
3444 return IsDGEMM ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates
3445 : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates;
3446 case 8:
3447 return IsDGEMM ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates
3448 : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates;
3449 case 16:
3450 return IsDGEMM ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates
3451 : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates;
3452 default:
3453 llvm_unreachable("unexpected number of passes");
3454 }
3455}
3456
3457int GCNHazardRecognizer::getMFMAReadWaitStates(const MachineInstr &Consumer,
3458 const MachineInstr &Producer,
3459 Register Reg,
3460 bool IsSrcC) const {
3461 unsigned Opc = Consumer.getOpcode();
3462 const int SMFMA4x4WritesVGPROverlappedSrcABWaitStates = 5;
3463 const int SMFMA16x16WritesVGPROverlappedSrcABWaitStates = 11;
3464 const int SMFMA32x32WritesVGPROverlappedSrcABWaitStates = 19;
3465 const int DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates = 6;
3466 const int DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 11;
3467 const int GFX950_DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 19;
3468 const int DMFMA4x4WritesVGPRFullSrcCWaitStates = 4;
3469 const int GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates = 2;
3470
3471 bool FullReg = Producer.getOperand(i: 0).getReg() == Reg;
3472 unsigned Opc1 = Producer.getOpcode();
3473 int NeedWaitStates = 0;
3474 if (IsSrcC) {
3475 if (!SIInstrInfo::isDGEMM(Opcode: Opc) &&
3476 (!ST.hasGFX940Insts() && SIInstrInfo::isDGEMM(Opcode: Opc1))) {
3477 NeedWaitStates = 0;
3478 } else if (FullReg) {
3479 if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
3480 Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) &&
3481 (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
3482 Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64))
3483 NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates;
3484 else if (ST.hasGFX940Insts() &&
3485 TSchedModel.computeInstrLatency(MI: &Producer) == 2)
3486 NeedWaitStates = GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates;
3487
3488 // The accumulator forwarding path that allows zero wait states is only
3489 // available while the chain stays on a single MFMA. Two different MFMAs
3490 // sharing an accumulator need the wait states of a partial overlap.
3491 if (ST.hasGFX940Insts() && !isSameMFMA(Opc0: Opc, Opc1)) {
3492 NeedWaitStates =
3493 std::max(a: NeedWaitStates,
3494 b: getMFMAOverlappedSrcCWaitStates(Reader: &Consumer, Writer: &Producer));
3495 }
3496 } else {
3497 NeedWaitStates = getMFMAOverlappedSrcCWaitStates(Reader: &Consumer, Writer: &Producer);
3498 }
3499 } else {
3500 switch (Opc1) {
3501 case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
3502 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
3503 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
3504 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
3505 NeedWaitStates =
3506 ST.hasGFX950Insts()
3507 ? GFX950_DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates
3508 : DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates;
3509 break;
3510 case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
3511 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
3512 NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates;
3513 break;
3514 default:
3515 int NumPasses = TSchedModel.computeInstrLatency(MI: &Producer);
3516
3517 if (ST.hasGFX940Insts()) {
3518 NeedWaitStates =
3519 TII.isXDL(MI: Producer)
3520 ? GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(
3521 NumPasses, IsGFX950: ST.hasGFX950Insts())
3522 : GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates(
3523 NumPasses);
3524 break;
3525 }
3526
3527 switch (NumPasses) {
3528 case 2:
3529 NeedWaitStates = SMFMA4x4WritesVGPROverlappedSrcABWaitStates;
3530 break;
3531 case 4:
3532 llvm_unreachable("unexpected number of passes for mfma");
3533 case 8:
3534 NeedWaitStates = SMFMA16x16WritesVGPROverlappedSrcABWaitStates;
3535 break;
3536 case 16:
3537 default:
3538 NeedWaitStates = SMFMA32x32WritesVGPROverlappedSrcABWaitStates;
3539 }
3540 }
3541 }
3542 return NeedWaitStates;
3543}
3544
3545int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) const {
3546 int WaitStatesNeeded = 0;
3547 unsigned Opc = MI->getOpcode();
3548
3549 auto IsLegacyVALUFn = [](const MachineInstr &MI) {
3550 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false) &&
3551 !SIInstrInfo::isMFMA(MI);
3552 };
3553
3554 auto IsLegacyVALUNotDotFn = [](const MachineInstr &MI) {
3555 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false) &&
3556 !SIInstrInfo::isMFMA(MI) && !SIInstrInfo::isDOT(MI);
3557 };
3558
3559 if (!SIInstrInfo::isMFMA(MI: *MI))
3560 return WaitStatesNeeded;
3561
3562 const int VALUWritesExecWaitStates = 4;
3563 int WaitStatesNeededForUse = VALUWritesExecWaitStates -
3564 getWaitStatesSinceDef(Reg: AMDGPU::EXEC, IsHazardDef: IsLegacyVALUFn,
3565 Limit: VALUWritesExecWaitStates);
3566 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3567
3568 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2);
3569
3570 // Loop for both DGEMM and S/HGEMM 2nd instruction.
3571 for (const MachineOperand &Use : MI->explicit_uses()) {
3572 const int LegacyVALUNotDotWritesVGPRWaitStates = 2;
3573 const int MaxWaitStates =
3574 GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(
3575 NumPasses: 16, IsGFX950: ST.hasGFX950Insts());
3576
3577 if (!Use.isReg())
3578 continue;
3579 Register Reg = Use.getReg();
3580
3581 WaitStatesNeededForUse = LegacyVALUNotDotWritesVGPRWaitStates -
3582 getWaitStatesSinceDef(Reg, IsHazardDef: IsLegacyVALUNotDotFn, Limit: MaxWaitStates);
3583 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3584
3585 int OpNo = Use.getOperandNo();
3586 WaitStatesNeeded = std::max(
3587 a: WaitStatesNeeded,
3588 b: getMaxWindowDeficit(
3589 MaxWindow: MaxWaitStates, WindowFor: [&](const MachineInstr &P) -> std::optional<int> {
3590 if (!SIInstrInfo::isMFMA(MI: P) ||
3591 !TRI.regsOverlap(RegA: P.getOperand(i: 0).getReg(), RegB: Reg))
3592 return std::nullopt;
3593 return getMFMAReadWaitStates(Consumer: *MI, Producer: P, Reg, IsSrcC: OpNo == SrcCIdx);
3594 }));
3595
3596 if (WaitStatesNeeded == MaxWaitStates)
3597 break;
3598 }
3599
3600 // Pad neighboring MFMA with noops for better inter-wave performance.
3601 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: checkMFMAPadding(MI));
3602
3603 return WaitStatesNeeded;
3604}
3605
3606int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) const {
3607 // On gfx90a+ relevant hazards are checked in checkMAIVALUHazards()
3608 if (!ST.hasMAIInsts() || ST.hasGFX90AInsts())
3609 return 0;
3610
3611 int WaitStatesNeeded = 0;
3612
3613 auto IsAccVgprReadFn = [](const MachineInstr &MI) {
3614 return MI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64;
3615 };
3616
3617 for (const MachineOperand &Op : MI->explicit_uses()) {
3618 if (!Op.isReg() || !TRI.isVGPR(MRI: MF.getRegInfo(), Reg: Op.getReg()))
3619 continue;
3620
3621 Register Reg = Op.getReg();
3622
3623 const int AccVgprReadLdStWaitStates = 2;
3624 const int VALUWriteAccVgprRdWrLdStDepVALUWaitStates = 1;
3625 const int MaxWaitStates = 2;
3626
3627 int WaitStatesNeededForUse = AccVgprReadLdStWaitStates -
3628 getWaitStatesSinceDef(Reg, IsHazardDef: IsAccVgprReadFn, Limit: MaxWaitStates);
3629 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3630
3631 if (WaitStatesNeeded == MaxWaitStates)
3632 return WaitStatesNeeded; // Early exit.
3633
3634 auto IsVALUAccVgprRdWrCheckFn = [Reg, this](const MachineInstr &MI) {
3635 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 &&
3636 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
3637 return false;
3638 auto IsVALUFn = [](const MachineInstr &MI) {
3639 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false) &&
3640 !SIInstrInfo::isMAI(MI);
3641 };
3642 return getWaitStatesSinceDef(Reg, IsHazardDef: IsVALUFn, Limit: 2 /*MaxWaitStates*/) <
3643 std::numeric_limits<int>::max();
3644 };
3645
3646 WaitStatesNeededForUse = VALUWriteAccVgprRdWrLdStDepVALUWaitStates -
3647 getWaitStatesSince(IsHazard: IsVALUAccVgprRdWrCheckFn, Limit: MaxWaitStates);
3648 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3649 }
3650
3651 return WaitStatesNeeded;
3652}
3653
3654int GCNHazardRecognizer::checkPermlaneHazards(MachineInstr *MI) const {
3655 assert(!ST.hasVcmpxPermlaneHazard() &&
3656 "this is a different vcmpx+permlane hazard");
3657 const SIRegisterInfo *TRI = ST.getRegisterInfo();
3658 const SIInstrInfo *TII = ST.getInstrInfo();
3659
3660 auto IsVCmpXWritesExecFn = [TII, TRI](const MachineInstr &MI) {
3661 return isVCmpXWritesExec(TII: *TII, TRI: *TRI, MI);
3662 };
3663
3664 auto IsVALUFn = [](const MachineInstr &MI) {
3665 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false);
3666 };
3667
3668 const int VCmpXWritesExecWaitStates = 4;
3669 const int VALUWritesVDstWaitStates = 2;
3670 int WaitStatesNeeded = 0;
3671
3672 for (const MachineOperand &Op : MI->explicit_uses()) {
3673 if (!Op.isReg() || !TRI->isVGPR(MRI: MF.getRegInfo(), Reg: Op.getReg()))
3674 continue;
3675 Register Reg = Op.getReg();
3676
3677 int WaitStatesSinceDef =
3678 VALUWritesVDstWaitStates -
3679 getWaitStatesSinceDef(Reg, IsHazardDef: IsVALUFn,
3680 /*MaxWaitStates=*/Limit: VALUWritesVDstWaitStates);
3681 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesSinceDef);
3682 if (WaitStatesNeeded >= VALUWritesVDstWaitStates)
3683 break;
3684 }
3685
3686 int VCmpXHazardWaits =
3687 VCmpXWritesExecWaitStates -
3688 getWaitStatesSince(IsHazard: IsVCmpXWritesExecFn, Limit: VCmpXWritesExecWaitStates);
3689
3690 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: VCmpXHazardWaits);
3691 return WaitStatesNeeded;
3692}
3693
3694static int GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(int NumPasses) {
3695 // 2 pass -> 4
3696 // 4 pass -> 6
3697 // 8 pass -> 10
3698 // 16 pass -> 18
3699 return NumPasses + 2;
3700}
3701
3702static int GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(int NumPasses,
3703 bool IsGFX950) {
3704 // xdl def cycles | gfx942 | gfx950
3705 // 2 pass | 5 5
3706 // 4 pass | 7 8
3707 // 8 pass | 11 12
3708 // 16 pass | 19 20
3709 return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
3710}
3711
3712static int GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses,
3713 bool IsGFX950) {
3714 // xdl def cycles | gfx942 | gfx950
3715 // 2 pass | 5 5
3716 // 4 pass | 7 8
3717 // 8 pass | 11 12
3718 // 16 pass | 19 20
3719 return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
3720}
3721
3722static int GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses) {
3723 // 2 pass -> 4
3724 // 4 pass -> 6
3725 // 8 pass -> 10
3726 // 16 pass -> 18
3727 return NumPasses + 2;
3728}
3729
3730int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const {
3731 if (!ST.hasGFX90AInsts())
3732 return 0;
3733
3734 auto IsDGEMMFn = [](const MachineInstr &MI) -> bool {
3735 return SIInstrInfo::isDGEMM(Opcode: MI.getOpcode());
3736 };
3737
3738 // This is checked in checkMAIHazards90A()
3739 if (SIInstrInfo::isMFMA(MI: *MI))
3740 return 0;
3741
3742 const MachineRegisterInfo &MRI = MF.getRegInfo();
3743
3744 int WaitStatesNeeded = 0;
3745
3746 bool IsMem = SIInstrInfo::isVMEM(MI: *MI) || SIInstrInfo::isDS(MI: *MI);
3747 bool IsMemOrExport = IsMem || SIInstrInfo::isEXP(MI: *MI);
3748 bool IsVALU = SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/false);
3749
3750 unsigned Reg;
3751 auto IsMFMAWriteFn = [&Reg, this](const MachineInstr &MI) {
3752 if (!SIInstrInfo::isMFMA(MI) ||
3753 !TRI.regsOverlap(RegA: MI.getOperand(i: 0).getReg(), RegB: Reg))
3754 return false;
3755 return true;
3756 };
3757
3758 const MachineInstr *DOT = nullptr;
3759 auto IsDotWriteFn = [&Reg, &DOT, this](const MachineInstr &MI) {
3760 if (!SIInstrInfo::isDOT(MI) ||
3761 !TRI.regsOverlap(RegA: MI.getOperand(i: 0).getReg(), RegB: Reg))
3762 return false;
3763 DOT = &MI;
3764 return true;
3765 };
3766
3767 bool DGEMMAfterVALUWrite = false;
3768 auto IsDGEMMHazard = [&DGEMMAfterVALUWrite, this](const MachineInstr &MI) {
3769 // Found DGEMM on reverse traversal to def.
3770 if (SIInstrInfo::isDGEMM(Opcode: MI.getOpcode()))
3771 DGEMMAfterVALUWrite = true;
3772
3773 // Only hazard if register is defined by a VALU and a DGEMM is found after
3774 // after the def.
3775 if (!TII.isVALU(MI, /*AllowLDSDMA=*/false) || !DGEMMAfterVALUWrite)
3776 return false;
3777
3778 return true;
3779 };
3780
3781 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opcode: MI->getOpcode(),
3782 Name: AMDGPU::OpName::src2);
3783
3784 if (IsMemOrExport || IsVALU) {
3785 const int SMFMA4x4WriteVgprVALUMemExpReadWaitStates = 5;
3786 const int SMFMA16x16WriteVgprVALUMemExpReadWaitStates = 11;
3787 const int SMFMA32x32WriteVgprVALUMemExpReadWaitStates = 19;
3788 const int DMFMA4x4WriteVgprMemExpReadWaitStates = 9;
3789 const int DMFMA16x16WriteVgprMemExpReadWaitStates = 18;
3790 const int DMFMA4x4WriteVgprVALUReadWaitStates = 6;
3791 const int DMFMA16x16WriteVgprVALUReadWaitStates = 11;
3792 const int GFX950_DMFMA16x16WriteVgprVALUReadWaitStates = 19;
3793 const int DotWriteSameDotReadSrcAB = 3;
3794 const int DotWriteDifferentVALURead = 3;
3795 const int DMFMABetweenVALUWriteVMEMRead = 2;
3796 const int MaxWaitStates =
3797 GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(NumPasses: 16,
3798 IsGFX950: ST.hasGFX950Insts());
3799
3800 for (const MachineOperand &Use : MI->explicit_uses()) {
3801 if (!Use.isReg())
3802 continue;
3803 Reg = Use.getReg();
3804
3805 DOT = nullptr;
3806 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsHazardDef: IsDotWriteFn,
3807 Limit: MaxWaitStates);
3808 if (DOT) {
3809 int NeedWaitStates = 0;
3810 if (DOT->getOpcode() == MI->getOpcode()) {
3811 if (&Use - &MI->getOperand(i: 0) != SrcCIdx)
3812 NeedWaitStates = DotWriteSameDotReadSrcAB;
3813 } else {
3814 NeedWaitStates = DotWriteDifferentVALURead;
3815 }
3816
3817 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
3818 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3819 }
3820
3821 // Workaround for HW data hazard bug observed only in GFX90A. When there
3822 // is a DGEMM instruction in-between a VALU and a VMEM instruction it
3823 // causes the SQ to incorrectly not insert two wait states between the two
3824 // instructions needed to avoid data hazard.
3825 if (IsMem && ST.hasGFX90AInsts() && !ST.hasGFX940Insts()) {
3826 DGEMMAfterVALUWrite = false;
3827 if (TRI.isVectorRegister(MRI, Reg)) {
3828 int WaitStatesNeededForUse =
3829 DMFMABetweenVALUWriteVMEMRead -
3830 getWaitStatesSinceDef(Reg, IsHazardDef: IsDGEMMHazard,
3831 Limit: DMFMABetweenVALUWriteVMEMRead);
3832
3833 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3834 }
3835 }
3836
3837 auto MFMAWindow = [&](const MachineInstr &Producer) {
3838 int NumPasses = TSchedModel.computeInstrLatency(MI: &Producer);
3839 int NeedWaitStates = MaxWaitStates;
3840
3841 if (SIInstrInfo::isDGEMM(Opcode: Producer.getOpcode())) {
3842 switch (NumPasses) {
3843 case 4:
3844 NeedWaitStates = IsMemOrExport
3845 ? DMFMA4x4WriteVgprMemExpReadWaitStates
3846 : DMFMA4x4WriteVgprVALUReadWaitStates;
3847 break;
3848 case 8:
3849 case 16:
3850 NeedWaitStates =
3851 IsMemOrExport
3852 ? DMFMA16x16WriteVgprMemExpReadWaitStates
3853 : (ST.hasGFX950Insts()
3854 ? GFX950_DMFMA16x16WriteVgprVALUReadWaitStates
3855 : DMFMA16x16WriteVgprVALUReadWaitStates);
3856 break;
3857 default:
3858 llvm_unreachable("unexpected dgemm");
3859 }
3860 } else if (ST.hasGFX940Insts()) {
3861 NeedWaitStates =
3862 TII.isXDL(MI: Producer)
3863 ? GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(
3864 NumPasses, IsGFX950: ST.hasGFX950Insts())
3865 : GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates(
3866 NumPasses);
3867 } else {
3868 switch (NumPasses) {
3869 case 2:
3870 NeedWaitStates = SMFMA4x4WriteVgprVALUMemExpReadWaitStates;
3871 break;
3872 case 8:
3873 NeedWaitStates = SMFMA16x16WriteVgprVALUMemExpReadWaitStates;
3874 break;
3875 case 16:
3876 NeedWaitStates = SMFMA32x32WriteVgprVALUMemExpReadWaitStates;
3877 break;
3878 default:
3879 llvm_unreachable("unexpected number of passes for mfma");
3880 }
3881 }
3882
3883 assert(NeedWaitStates <= MaxWaitStates &&
3884 "hazard requirement exceeds the scan window");
3885 return NeedWaitStates;
3886 };
3887
3888 WaitStatesNeeded = std::max(
3889 a: WaitStatesNeeded,
3890 b: getMaxWindowDeficit(MaxWindow: MaxWaitStates,
3891 WindowFor: [&](const MachineInstr &P) -> std::optional<int> {
3892 if (!IsMFMAWriteFn(P))
3893 return std::nullopt;
3894 return MFMAWindow(P);
3895 }));
3896
3897 if (WaitStatesNeeded >= MaxWaitStates)
3898 break;
3899 }
3900 }
3901
3902 unsigned Opc = MI->getOpcode();
3903 const int DMFMAToFMA64WaitStates = 2;
3904 if ((Opc == AMDGPU::V_FMA_F64_e64 ||
3905 Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64 ||
3906 Opc == AMDGPU::V_FMAC_F64_dpp) &&
3907 WaitStatesNeeded < DMFMAToFMA64WaitStates) {
3908 int WaitStatesNeededForUse = DMFMAToFMA64WaitStates -
3909 getWaitStatesSince(IsHazard: IsDGEMMFn, Limit: DMFMAToFMA64WaitStates);
3910 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3911 }
3912
3913 if (!IsVALU && !IsMemOrExport)
3914 return WaitStatesNeeded;
3915
3916 for (const MachineOperand &Def : MI->defs()) {
3917 const int SMFMA4x4WriteVgprVALUWawWaitStates = 5;
3918 const int SMFMA16x16WriteVgprVALUWawWaitStates = 11;
3919 const int SMFMA32x32WriteVgprVALUWawWaitStates = 19;
3920 const int SMFMA4x4ReadVgprVALUWarWaitStates = 1;
3921 const int GFX940_XDL4PassReadVgprVALUWarWaitStates = 3;
3922 const int SMFMA16x16ReadVgprVALUWarWaitStates = 7;
3923 const int SMFMA32x32ReadVgprVALUWarWaitStates = 15;
3924 const int DMFMA4x4WriteVgprVALUWriteWaitStates = 6;
3925 const int DMFMA16x16WriteVgprVALUWriteWaitStates = 11;
3926 const int DotWriteDifferentVALUWrite = 3;
3927 const int MaxWaitStates =
3928 GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(NumPasses: 16, IsGFX950: ST.hasGFX950Insts());
3929 const int MaxWarWaitStates = 15;
3930
3931 Reg = Def.getReg();
3932
3933 DOT = nullptr;
3934 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsHazardDef: IsDotWriteFn,
3935 Limit: MaxWaitStates);
3936 if (DOT && DOT->getOpcode() != MI->getOpcode())
3937 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: DotWriteDifferentVALUWrite -
3938 WaitStatesSinceDef);
3939
3940 auto MFMAWawWindow = [&](const MachineInstr &Producer) {
3941 int NeedWaitStates = MaxWaitStates;
3942 int NumPasses = TSchedModel.computeInstrLatency(MI: &Producer);
3943
3944 if (SIInstrInfo::isDGEMM(Opcode: Producer.getOpcode())) {
3945 switch (NumPasses) {
3946 case 4:
3947 NeedWaitStates = DMFMA4x4WriteVgprVALUWriteWaitStates;
3948 break;
3949 case 8:
3950 case 16:
3951 NeedWaitStates = DMFMA16x16WriteVgprVALUWriteWaitStates;
3952 break;
3953 default:
3954 llvm_unreachable("unexpected number of cycles for dgemm");
3955 }
3956 } else if (ST.hasGFX940Insts()) {
3957 NeedWaitStates =
3958 TII.isXDL(MI: Producer)
3959 ? GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(
3960 NumPasses, IsGFX950: ST.hasGFX950Insts())
3961 : GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(NumPasses);
3962 } else {
3963 switch (NumPasses) {
3964 case 2:
3965 NeedWaitStates = SMFMA4x4WriteVgprVALUWawWaitStates;
3966 break;
3967 case 8:
3968 NeedWaitStates = SMFMA16x16WriteVgprVALUWawWaitStates;
3969 break;
3970 case 16:
3971 NeedWaitStates = SMFMA32x32WriteVgprVALUWawWaitStates;
3972 break;
3973 default:
3974 llvm_unreachable("Unexpected number of passes for mfma");
3975 }
3976 }
3977
3978 assert(NeedWaitStates <= MaxWaitStates &&
3979 "hazard requirement exceeds the scan window");
3980 return NeedWaitStates;
3981 };
3982
3983 WaitStatesNeeded = std::max(
3984 a: WaitStatesNeeded,
3985 b: getMaxWindowDeficit(MaxWindow: MaxWaitStates,
3986 WindowFor: [&](const MachineInstr &P) -> std::optional<int> {
3987 if (!IsMFMAWriteFn(P))
3988 return std::nullopt;
3989 return MFMAWawWindow(P);
3990 }));
3991
3992 if (WaitStatesNeeded >= MaxWaitStates)
3993 break;
3994
3995 // Wait states this write needs after an MFMA that read Reg as srcC.
3996 auto SMFMAReadAsCWindow =
3997 [&](const MachineInstr &Reader) -> std::optional<int> {
3998 if (!SIInstrInfo::isMFMA(MI: Reader) ||
3999 SIInstrInfo::isDGEMM(Opcode: Reader.getOpcode()) ||
4000 !Reader.readsRegister(Reg, TRI: &TRI))
4001 return std::nullopt;
4002
4003 if (ST.hasGFX940Insts() && !TII.isXDL(MI: Reader))
4004 return std::nullopt;
4005
4006 const MachineOperand *SrcC =
4007 TII.getNamedOperand(MI: Reader, OperandName: AMDGPU::OpName::src2);
4008 assert(SrcC && "an MFMA must have a src2 operand");
4009 if (!SrcC->isReg() || !TRI.regsOverlap(RegA: SrcC->getReg(), RegB: Reg))
4010 return std::nullopt;
4011
4012 switch (TSchedModel.computeInstrLatency(MI: &Reader)) {
4013 case 2:
4014 return SMFMA4x4ReadVgprVALUWarWaitStates;
4015 case 4:
4016 assert(ST.hasGFX940Insts() && "4-pass XDL is gfx940 or later");
4017 return GFX940_XDL4PassReadVgprVALUWarWaitStates;
4018 case 8:
4019 return SMFMA16x16ReadVgprVALUWarWaitStates;
4020 case 16:
4021 [[fallthrough]];
4022 default:
4023 return SMFMA32x32ReadVgprVALUWarWaitStates;
4024 }
4025 };
4026
4027 WaitStatesNeeded =
4028 std::max(a: WaitStatesNeeded,
4029 b: getMaxWindowDeficit(MaxWindow: MaxWarWaitStates, WindowFor: SMFMAReadAsCWindow));
4030 }
4031
4032 return WaitStatesNeeded;
4033}
4034
4035bool GCNHazardRecognizer::ShouldPreferAnother(SUnit *SU) const {
4036 if (!SU->isInstr())
4037 return false;
4038
4039 const MachineInstr *MAI = nullptr;
4040
4041 auto IsMFMAFn = [&MAI](const MachineInstr &MI) {
4042 MAI = nullptr;
4043 if (SIInstrInfo::isMFMA(MI))
4044 MAI = &MI;
4045 return MAI != nullptr;
4046 };
4047
4048 MachineInstr *MI = SU->getInstr();
4049 if (IsMFMAFn(*MI)) {
4050 int W = getWaitStatesSince(IsHazard: IsMFMAFn, Limit: 16);
4051 if (MAI)
4052 return W < (int)TSchedModel.computeInstrLatency(MI: MAI);
4053 }
4054
4055 return false;
4056}
4057
4058// Adjust global offsets for instructions bundled with S_GETPC_B64 after
4059// insertion of a new instruction.
4060static void updateGetPCBundle(MachineInstr *NewMI) {
4061 if (!NewMI->isBundled())
4062 return;
4063
4064 // Find start of bundle.
4065 auto I = NewMI->getIterator();
4066 while (I->isBundledWithPred())
4067 I--;
4068 if (I->isBundle())
4069 I++;
4070
4071 // Bail if this is not an S_GETPC bundle.
4072 if (I->getOpcode() != AMDGPU::S_GETPC_B64)
4073 return;
4074
4075 // Update offsets of any references in the bundle.
4076 const unsigned NewBytes = 4;
4077 assert(NewMI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
4078 "Unexpected instruction insertion in bundle");
4079 auto NextMI = std::next(x: NewMI->getIterator());
4080 auto End = NewMI->getParent()->end();
4081 while (NextMI != End && NextMI->isBundledWithPred()) {
4082 for (auto &Operand : NextMI->operands()) {
4083 if (Operand.isGlobal())
4084 Operand.setOffset(Operand.getOffset() + NewBytes);
4085 }
4086 NextMI++;
4087 }
4088}
4089
4090bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) {
4091 if (!ST.hasVALUMaskWriteHazard())
4092 return false;
4093 assert(!ST.hasExtendedWaitCounts());
4094
4095 if (!ST.isWave64())
4096 return false;
4097
4098 const bool IsSALU = SIInstrInfo::isSALU(MI: *MI);
4099 const bool IsVALU = SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/false);
4100 if (!IsSALU && !IsVALU)
4101 return false;
4102
4103 // The hazard sequence is three instructions:
4104 // 1. VALU reads SGPR as mask
4105 // 2. VALU/SALU writes SGPR
4106 // 3. VALU/SALU reads SGPR
4107 // The hazard can expire if the distance between 2 and 3 is sufficient,
4108 // or (2) is VALU and (3) is SALU.
4109 // In practice this happens <10% of the time, hence always assume the hazard
4110 // exists if (1) and (2) are present to avoid searching all SGPR reads.
4111
4112 const SIRegisterInfo *TRI = ST.getRegisterInfo();
4113 const MachineRegisterInfo &MRI = MF.getRegInfo();
4114
4115 auto IgnoreableSGPR = [](const Register Reg) {
4116 switch (Reg) {
4117 case AMDGPU::EXEC:
4118 case AMDGPU::EXEC_LO:
4119 case AMDGPU::EXEC_HI:
4120 case AMDGPU::M0:
4121 case AMDGPU::SGPR_NULL:
4122 case AMDGPU::SGPR_NULL64:
4123 case AMDGPU::SCC:
4124 return true;
4125 default:
4126 return false;
4127 }
4128 };
4129 auto IsVCC = [](const Register Reg) {
4130 return Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::VCC_HI;
4131 };
4132
4133 struct StateType {
4134 SmallSet<Register, 2> HazardSGPRs;
4135
4136 static unsigned getHashValue(const StateType &State) {
4137 return static_cast<unsigned>(hash_combine_range(R: State.HazardSGPRs));
4138 }
4139 static bool isEqual(const StateType &LHS, const StateType &RHS) {
4140 return LHS.HazardSGPRs == RHS.HazardSGPRs;
4141 }
4142 };
4143
4144 StateType InitialState;
4145
4146 // Look for SGPR write.
4147 MachineOperand *HazardDef = nullptr;
4148 for (MachineOperand &Op : MI->all_defs()) {
4149 Register Reg = Op.getReg();
4150 if (IgnoreableSGPR(Reg))
4151 continue;
4152 if (!IsVCC(Reg)) {
4153 if (Op.isImplicit())
4154 continue;
4155 if (!TRI->isSGPRReg(MRI, Reg))
4156 continue;
4157 }
4158
4159 HazardDef = &Op;
4160 break;
4161 }
4162
4163 if (!HazardDef)
4164 return false;
4165
4166 // Setup to track writes to individual SGPRs
4167 const Register HazardReg = HazardDef->getReg();
4168 if (AMDGPU::SReg_32RegClass.contains(Reg: HazardReg)) {
4169 InitialState.HazardSGPRs.insert(V: HazardReg);
4170 } else {
4171 assert(AMDGPU::SReg_64RegClass.contains(HazardReg));
4172 InitialState.HazardSGPRs.insert(V: TRI->getSubReg(Reg: HazardReg, Idx: AMDGPU::sub0));
4173 InitialState.HazardSGPRs.insert(V: TRI->getSubReg(Reg: HazardReg, Idx: AMDGPU::sub1));
4174 }
4175
4176 auto IsHazardFn = [&](StateType &State, const MachineInstr &I) {
4177 if (State.HazardSGPRs.empty())
4178 return HazardExpired;
4179
4180 switch (I.getOpcode()) {
4181 case AMDGPU::V_ADDC_U32_e32:
4182 case AMDGPU::V_ADDC_U32_dpp:
4183 case AMDGPU::V_CNDMASK_B16_t16_e32:
4184 case AMDGPU::V_CNDMASK_B16_fake16_e32:
4185 case AMDGPU::V_CNDMASK_B16_t16_dpp:
4186 case AMDGPU::V_CNDMASK_B16_fake16_dpp:
4187 case AMDGPU::V_CNDMASK_B32_e32:
4188 case AMDGPU::V_CNDMASK_B32_dpp:
4189 case AMDGPU::V_DIV_FMAS_F32_e64:
4190 case AMDGPU::V_DIV_FMAS_F64_e64:
4191 case AMDGPU::V_SUBB_U32_e32:
4192 case AMDGPU::V_SUBB_U32_dpp:
4193 case AMDGPU::V_SUBBREV_U32_e32:
4194 case AMDGPU::V_SUBBREV_U32_dpp: {
4195 // These implicitly read VCC as mask source.
4196 return IsVCC(HazardReg) ? HazardFound : NoHazardFound;
4197 }
4198 case AMDGPU::V_ADDC_U32_e64:
4199 case AMDGPU::V_ADDC_U32_e64_dpp:
4200 case AMDGPU::V_CNDMASK_B16_t16_e64:
4201 case AMDGPU::V_CNDMASK_B16_fake16_e64:
4202 case AMDGPU::V_CNDMASK_B16_t16_e64_dpp:
4203 case AMDGPU::V_CNDMASK_B16_fake16_e64_dpp:
4204 case AMDGPU::V_CNDMASK_B32_e64:
4205 case AMDGPU::V_CNDMASK_B32_e64_dpp:
4206 case AMDGPU::V_SUBB_U32_e64:
4207 case AMDGPU::V_SUBB_U32_e64_dpp:
4208 case AMDGPU::V_SUBBREV_U32_e64:
4209 case AMDGPU::V_SUBBREV_U32_e64_dpp: {
4210 // Only check mask register overlaps.
4211 const MachineOperand *SSRCOp = TII.getNamedOperand(MI: I, OperandName: AMDGPU::OpName::src2);
4212 assert(SSRCOp);
4213 bool Result = TRI->regsOverlap(RegA: SSRCOp->getReg(), RegB: HazardReg);
4214 return Result ? HazardFound : NoHazardFound;
4215 }
4216 default:
4217 return NoHazardFound;
4218 }
4219 };
4220
4221 auto UpdateStateFn = [&](StateType &State, const MachineInstr &I) {
4222 // Update tracking of SGPR writes.
4223 for (auto &Op : I.all_defs()) {
4224 Register Reg = Op.getReg();
4225 if (IgnoreableSGPR(Reg))
4226 continue;
4227 if (!IsVCC(Reg)) {
4228 if (Op.isImplicit())
4229 continue;
4230 if (!TRI->isSGPRReg(MRI, Reg))
4231 continue;
4232 }
4233
4234 // Stop tracking any SGPRs with writes on the basis that they will
4235 // already have an appropriate wait inserted afterwards.
4236 SmallVector<Register, 2> Found;
4237 for (Register SGPR : State.HazardSGPRs) {
4238 if (Reg == SGPR || TRI->regsOverlap(RegA: Reg, RegB: SGPR))
4239 Found.push_back(Elt: SGPR);
4240 }
4241 for (Register SGPR : Found)
4242 State.HazardSGPRs.erase(V: SGPR);
4243 }
4244 };
4245
4246 // Check for hazard
4247 if (!hasHazard<StateType>(InitialState, IsHazard: IsHazardFn, UpdateState: UpdateStateFn,
4248 InitialMBB: MI->getParent(),
4249 InitialI: std::next(x: MI->getReverseIterator())))
4250 return false;
4251
4252 // Compute counter mask
4253 unsigned DepCtr =
4254 IsVALU ? (IsVCC(HazardReg) ? AMDGPU::DepCtr::encodeFieldVaVcc(VaVcc: 0, STI: ST)
4255 : AMDGPU::DepCtr::encodeFieldVaSdst(VaSdst: 0, STI: ST))
4256 : AMDGPU::DepCtr::encodeFieldSaSdst(SaSdst: 0, STI: ST);
4257
4258 // Add s_waitcnt_depctr after SGPR write.
4259 auto NextMI = std::next(x: MI->getIterator());
4260 auto NewMI = BuildMI(BB&: *MI->getParent(), I: NextMI, MIMD: MI->getDebugLoc(),
4261 MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4262 .addImm(Val: DepCtr);
4263
4264 // SALU write may be s_getpc in a bundle.
4265 updateGetPCBundle(NewMI);
4266
4267 return true;
4268}
4269
4270static bool ensureEntrySetPrio(MachineFunction *MF, int Priority,
4271 const SIInstrInfo &TII) {
4272 MachineBasicBlock &EntryMBB = MF->front();
4273 if (EntryMBB.begin() != EntryMBB.end()) {
4274 auto &EntryMI = *EntryMBB.begin();
4275 if (EntryMI.getOpcode() == AMDGPU::S_SETPRIO &&
4276 EntryMI.getOperand(i: 0).getImm() >= Priority)
4277 return false;
4278 }
4279
4280 BuildMI(BB&: EntryMBB, I: EntryMBB.begin(), MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_SETPRIO))
4281 .addImm(Val: Priority);
4282 return true;
4283}
4284
4285bool GCNHazardRecognizer::fixRequiredExportPriority(MachineInstr *MI) {
4286 if (!ST.hasRequiredExportPriority())
4287 return false;
4288
4289 // Assume the following shader types will never have exports,
4290 // and avoid adding or adjusting S_SETPRIO.
4291 MachineBasicBlock *MBB = MI->getParent();
4292 MachineFunction *MF = MBB->getParent();
4293 auto CC = MF->getFunction().getCallingConv();
4294 switch (CC) {
4295 case CallingConv::AMDGPU_CS:
4296 case CallingConv::AMDGPU_CS_Chain:
4297 case CallingConv::AMDGPU_CS_ChainPreserve:
4298 case CallingConv::AMDGPU_KERNEL:
4299 return false;
4300 default:
4301 break;
4302 }
4303
4304 const int MaxPriority = 3;
4305 const int NormalPriority = 2;
4306 const int PostExportPriority = 0;
4307
4308 auto It = MI->getIterator();
4309 switch (MI->getOpcode()) {
4310 case AMDGPU::S_ENDPGM:
4311 case AMDGPU::S_ENDPGM_SAVED:
4312 case AMDGPU::S_ENDPGM_ORDERED_PS_DONE:
4313 case AMDGPU::SI_RETURN_TO_EPILOG:
4314 // Ensure shader with calls raises priority at entry.
4315 // This ensures correct priority if exports exist in callee.
4316 if (MF->getFrameInfo().hasCalls())
4317 return ensureEntrySetPrio(MF, Priority: NormalPriority, TII);
4318 return false;
4319 case AMDGPU::S_SETPRIO: {
4320 // Raise minimum priority unless in workaround.
4321 auto &PrioOp = MI->getOperand(i: 0);
4322 int Prio = static_cast<int>(PrioOp.getImm());
4323 bool InWA = (Prio == PostExportPriority) &&
4324 (It != MBB->begin() && TII.isEXP(MI: *std::prev(x: It)));
4325 if (InWA || Prio >= NormalPriority)
4326 return false;
4327 PrioOp.setImm(std::min(a: Prio + NormalPriority, b: MaxPriority));
4328 return true;
4329 }
4330 default:
4331 if (!TII.isEXP(MI: *MI))
4332 return false;
4333 break;
4334 }
4335
4336 // Check entry priority at each export (as there will only be a few).
4337 // Note: amdgpu_gfx can only be a callee, so defer to caller setprio.
4338 bool Changed = false;
4339 if (CC != CallingConv::AMDGPU_Gfx && CC != CallingConv::AMDGPU_Gfx_WholeWave)
4340 Changed = ensureEntrySetPrio(MF, Priority: NormalPriority, TII);
4341
4342 auto NextMI = std::next(x: It);
4343 bool EndOfShader = false;
4344 if (NextMI != MBB->end()) {
4345 // Only need WA at end of sequence of exports.
4346 if (TII.isEXP(MI: *NextMI))
4347 return Changed;
4348 // Assume appropriate S_SETPRIO after export means WA already applied.
4349 if (NextMI->getOpcode() == AMDGPU::S_SETPRIO &&
4350 NextMI->getOperand(i: 0).getImm() == PostExportPriority)
4351 return Changed;
4352 EndOfShader = NextMI->getOpcode() == AMDGPU::S_ENDPGM;
4353 }
4354
4355 const DebugLoc &DL = MI->getDebugLoc();
4356
4357 // Lower priority.
4358 BuildMI(BB&: *MBB, I: NextMI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_SETPRIO))
4359 .addImm(Val: PostExportPriority);
4360
4361 if (!EndOfShader) {
4362 // Wait for exports to complete.
4363 BuildMI(BB&: *MBB, I: NextMI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_EXPCNT))
4364 .addReg(RegNo: AMDGPU::SGPR_NULL)
4365 .addImm(Val: 0);
4366 }
4367
4368 BuildMI(BB&: *MBB, I: NextMI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_NOP)).addImm(Val: 0);
4369 BuildMI(BB&: *MBB, I: NextMI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_NOP)).addImm(Val: 0);
4370
4371 if (!EndOfShader) {
4372 // Return to normal (higher) priority.
4373 BuildMI(BB&: *MBB, I: NextMI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_SETPRIO))
4374 .addImm(Val: NormalPriority);
4375 }
4376
4377 return true;
4378}
4379
4380// Advance past meta instructions (debug values, labels, CFI, KILL, etc.) to the
4381// next instruction that actually issues. Unlike skipDebugInstructionsForward /
4382// next_nodbg, this skips the full isMetaInstruction() set.
4383static MachineBasicBlock::iterator
4384skipMetaInstructionsForward(MachineBasicBlock::iterator I,
4385 MachineBasicBlock::iterator End) {
4386 while (I != End && I->isMetaInstruction())
4387 ++I;
4388 return I;
4389}
4390
4391bool GCNHazardRecognizer::fixVPermPk16Hazard(MachineInstr *MI) {
4392 // Requirement #1 of 2:
4393 // The cross-wave entry-block mitigation is delegated to the mandatory
4394 // unclaused-VMEM entry prologue (GLOBAL_PREFETCH_B8 + V_NOP).
4395 assert(ST.hasRequiresInitialUnclausedVmem() &&
4396 "V_PERM_PK16-hazard subtarget must provide the unclaused-VMEM entry "
4397 "prologue to satisfy the cross-wave entry mitigation");
4398
4399 if (!SIInstrInfo::isVPermPk16(Opcode: MI->getOpcode()))
4400 return false;
4401
4402 MachineBasicBlock *MBB = MI->getParent();
4403
4404 // Requirement #2 of 2:
4405 // V_PERM_PK16 must be immediately followed by a safe instruction.
4406 MachineBasicBlock::iterator NextI =
4407 skipMetaInstructionsForward(I: std::next(x: MI->getIterator()), End: MBB->end());
4408 if (NextI != MBB->end() && TII.isVPermPk16SafeInstr(MI: *NextI))
4409 return false;
4410
4411 // EXEC is guaranteed non-zero here: V_PERM_PK16 reports unwanted effects
4412 // when EXEC is empty, so s_cbranch_execz over this region is retained.
4413 // A plain V_NOP is therefore a real VALU nop and clears the hazard.
4414 BuildMI(BB&: *MBB, I: NextI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_NOP_e32));
4415 return true;
4416}
4417
4418bool GCNHazardRecognizer::fixGetRegWaitIdle(MachineInstr *MI) {
4419 if (!isSGetReg(Opcode: MI->getOpcode()))
4420 return false;
4421
4422 const SIInstrInfo *TII = ST.getInstrInfo();
4423 switch (getHWReg(TII, RegInstr: *MI)) {
4424 default:
4425 return false;
4426 case AMDGPU::Hwreg::ID_STATUS:
4427 case AMDGPU::Hwreg::ID_STATE_PRIV:
4428 case AMDGPU::Hwreg::ID_EXCP_FLAG_PRIV:
4429 case AMDGPU::Hwreg::ID_EXCP_FLAG_USER:
4430 break;
4431 }
4432
4433 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
4434 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4435 .addImm(Val: 0);
4436 return true;
4437}
4438
4439bool GCNHazardRecognizer::fixDsAtomicAsyncBarrierArriveB64(MachineInstr *MI) {
4440 if (MI->getOpcode() != AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
4441 return false;
4442
4443 const SIInstrInfo *TII = ST.getInstrInfo();
4444 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
4445 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4446 .addImm(Val: AMDGPU::DepCtr::encodeFieldVmVsrc(VmVsrc: 0, STI: ST));
4447 BuildMI(BB&: *MI->getParent(), I: std::next(x: MI->getIterator()), MIMD: MI->getDebugLoc(),
4448 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4449 .addImm(Val: AMDGPU::DepCtr::encodeFieldVmVsrc(VmVsrc: 0, STI: ST));
4450
4451 return true;
4452}
4453
4454bool GCNHazardRecognizer::fixScratchBaseForwardingHazard(MachineInstr *MI) {
4455 // No reason to check this in pre-RA scheduling, SGPRs have to be allocated
4456 // for hazard to trigger.
4457 if (!isHazardRecognizerMode())
4458 return false;
4459
4460 const SIRegisterInfo *TRI = ST.getRegisterInfo();
4461 const SIInstrInfo *TII = ST.getInstrInfo();
4462 // Hazard expires after 10 SGPR writes by SALU or 8 SGPR writes by VALU.
4463 const int FlatScrBaseWaitStates = 10;
4464
4465 bool ReadsFlatScrLo =
4466 MI->readsRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_LO, TRI);
4467 bool ReadsFlatScrHi =
4468 MI->readsRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_HI, TRI);
4469 if (isSGetReg(Opcode: MI->getOpcode())) {
4470 switch (getHWReg(TII, RegInstr: *MI)) {
4471 default:
4472 break;
4473 case AMDGPU::Hwreg::ID_FLAT_SCR_LO:
4474 ReadsFlatScrLo = true;
4475 break;
4476 case AMDGPU::Hwreg::ID_FLAT_SCR_HI:
4477 ReadsFlatScrHi = true;
4478 break;
4479 }
4480 }
4481
4482 const MachineRegisterInfo &MRI = MF.getRegInfo();
4483
4484 auto IsRegDefHazard = [&](Register Reg) -> bool {
4485 DenseSet<const MachineBasicBlock *> Visited;
4486 auto IsHazardFn = [TRI, Reg](const MachineInstr &MI) {
4487 return MI.modifiesRegister(Reg, TRI);
4488 };
4489
4490 // This literally abuses the idea of waitstates. Instead of waitstates it
4491 // returns 1 for SGPR written and 0 otherwise.
4492 auto IsSGPRDef = [TII, TRI, &MRI](const MachineInstr &MI) -> unsigned {
4493 if (!TII->isSALU(MI) && !TII->isVALU(MI, /*AllowLDSDMA=*/false))
4494 return 0;
4495 for (const MachineOperand &MO : MI.all_defs()) {
4496 if (TRI->isSGPRReg(MRI, Reg: MO.getReg()))
4497 return 1;
4498 }
4499 return 0;
4500 };
4501
4502 auto IsExpiredFn = [=](const MachineInstr &MI, int SgprWrites) {
4503 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR) {
4504 unsigned Wait = static_cast<unsigned>(MI.getOperand(i: 0).getImm());
4505 if (AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: Wait) == 0 &&
4506 AMDGPU::DepCtr::decodeFieldVaSdst(Encoded: Wait) == 0)
4507 return true;
4508 }
4509 return SgprWrites >= FlatScrBaseWaitStates;
4510 };
4511
4512 return ::getWaitStatesSince(
4513 IsHazard: IsHazardFn, MBB: MI->getParent(), I: std::next(x: MI->getReverseIterator()),
4514 WaitStates: 0, IsExpired: IsExpiredFn, Visited, GetNumWaitStates: IsSGPRDef) < FlatScrBaseWaitStates;
4515 };
4516
4517 if ((!ReadsFlatScrLo || MRI.isConstantPhysReg(PhysReg: AMDGPU::SGPR102) ||
4518 !IsRegDefHazard(AMDGPU::SGPR102)) &&
4519 (!ReadsFlatScrHi || MRI.isConstantPhysReg(PhysReg: AMDGPU::SGPR103) ||
4520 !IsRegDefHazard(AMDGPU::SGPR103)))
4521 return false;
4522
4523 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
4524 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4525 .addImm(Val: AMDGPU::DepCtr::encodeFieldVaSdst(
4526 Encoded: AMDGPU::DepCtr::encodeFieldSaSdst(SaSdst: 0, STI: ST), VaSdst: 0));
4527 return true;
4528}
4529
4530bool GCNHazardRecognizer::fixSetRegMode(MachineInstr *MI) {
4531 if (!isSSetReg(Opcode: MI->getOpcode()) ||
4532 MI->getOperand(i: 1).getImm() != AMDGPU::Hwreg::ID_MODE)
4533 return false;
4534
4535 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_NOP_e32));
4536 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_NOP_e32));
4537 return true;
4538}
4539
4540bool GCNHazardRecognizer::fixTDM(MachineInstr *MI) {
4541 auto IsTDM = [&](const MachineInstr &MI) -> bool {
4542 return SIInstrInfo::usesTENSOR_CNT(MI) &&
4543 MI.getOpcode() != AMDGPU::S_WAIT_TENSORCNT;
4544 };
4545
4546 if (!IsTDM(*MI))
4547 return false;
4548
4549 auto IsExpiredFn = [](const MachineInstr &MI, int) {
4550 if (MI.getOpcode() != AMDGPU::S_WAIT_TENSORCNT)
4551 return false;
4552 return MI.getOperand(i: 0).getImm() <= 10;
4553 };
4554
4555 if (::getWaitStatesSince(IsHazard: IsTDM, MI, IsExpired: IsExpiredFn) ==
4556 std::numeric_limits<int>::max())
4557 return false;
4558
4559 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
4560 MCID: TII.get(Opcode: AMDGPU::S_WAIT_TENSORCNT))
4561 .addImm(Val: 10);
4562 return true;
4563}
4564
4565bool GCNHazardRecognizer::fixTransF16ToPartialVGPRWriteHazard(
4566 MachineInstr *MI) {
4567 if (!SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/false))
4568 return false;
4569
4570 const int NumVALUNeeded = 4;
4571 const MachineRegisterInfo &MRI = MF.getRegInfo();
4572
4573 struct StateType {
4574 int VALUs = 0;
4575
4576 static unsigned getHashValue(const StateType &State) { return State.VALUs; }
4577 static bool isEqual(const StateType &LHS, const StateType &RHS) {
4578 return LHS.VALUs == RHS.VALUs;
4579 }
4580 };
4581
4582 StateType State;
4583
4584 auto IsHazardFn = [&](StateType &State, const MachineInstr &I) {
4585 // Too many VALU states have passed
4586 if (State.VALUs >= NumVALUNeeded)
4587 return HazardExpired;
4588
4589 // Instructions which cause va_vdst==0 expire hazard
4590 if (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
4591 AMDGPU::DepCtr::decodeFieldVaVdst(
4592 Encoded: static_cast<unsigned>(I.getOperand(i: 0).getImm())) == 0)
4593 return HazardExpired;
4594
4595 // Track registers writes
4596 if (!SIInstrInfo::isTRANS(MI: I))
4597 return NoHazardFound;
4598
4599 MCRegister Def = I.getOperand(i: 0).getReg();
4600 if (!TRI.isVGPR(MRI, Reg: Def))
4601 return NoHazardFound;
4602
4603 unsigned Opc = I.getOpcode();
4604 const MCInstrDesc &Desc = TII.get(Opcode: Opc);
4605 // Note: checking vdst does not work because it is a generic
4606 // OPERAND_REGISTER.
4607 switch (
4608 Desc.operands()[AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0)]
4609 .OperandType) {
4610 case AMDGPU::OPERAND_REG_IMM_BF16:
4611 case AMDGPU::OPERAND_REG_IMM_FP16:
4612 case AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16:
4613 case AMDGPU::OPERAND_REG_INLINE_C_BF16:
4614 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
4615 break;
4616 default:
4617 // DPP instructions have VGPR-only src0 with generic OPERAND_REGISTER
4618 // type. The isDPP() check will catch any TRANS opcodes including f32.
4619 // This is suboptimal but shall only affect fake16 compilation because
4620 // of the Def size check below.
4621 if (SIInstrInfo::isDPP(MI: I))
4622 break;
4623 return NoHazardFound;
4624 }
4625
4626 if (TRI.getRegSizeInBits(Reg: Def, MRI) == 16) {
4627 // Real True16 case.
4628 const auto Sub = AMDGPU::isHi16Reg(Reg: Def, MRI: TRI)
4629 ? std::make_pair(x: AMDGPU::hi16, y: AMDGPU::lo16)
4630 : std::make_pair(x: AMDGPU::lo16, y: AMDGPU::hi16);
4631 MCRegister Def32 =
4632 TRI.getMatchingSuperReg(Reg: Def, SubIdx: Sub.first, RC: &AMDGPU::VGPR_32RegClass);
4633 MCRegister OffendingDef = TRI.getSubReg(Reg: Def32, Idx: Sub.second);
4634 // Only interested in partial 16-bit write, not any wider register.
4635 return MI->modifiesRegister(Reg: OffendingDef, TRI: nullptr) ? HazardFound
4636 : NoHazardFound;
4637 }
4638
4639 // Bail for any non-16 bit DPP instructions catched in true16 mode.
4640 if (ST.useRealTrue16Insts())
4641 return NoHazardFound;
4642
4643 // Fake16 case is suboptimal as it checks the whole 32-bit def overlap w/o
4644 // analyzing OPSEL bits.
4645 return MI->modifiesRegister(Reg: Def, TRI: nullptr) ? HazardFound : NoHazardFound;
4646 };
4647
4648 auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) {
4649 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
4650 ++State.VALUs;
4651 };
4652
4653 if (!hasHazard<StateType>(InitialState: State, IsHazard: IsHazardFn, UpdateState: UpdateStateFn, InitialMBB: MI->getParent(),
4654 InitialI: std::next(x: MI->getReverseIterator())))
4655 return false;
4656
4657 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
4658 MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4659 .addImm(Val: AMDGPU::DepCtr::encodeFieldVaVdst(VaVdst: 0, STI: ST));
4660
4661 return true;
4662}
4663