1//===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements hazard recognizers for scheduling on GCN processors.
10//
11//===----------------------------------------------------------------------===//
12
13#include "GCNHazardRecognizer.h"
14#include "AMDGPUWaitcntUtils.h"
15#include "GCNSubtarget.h"
16#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
17#include "SIMachineFunctionInfo.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/CodeGen/MachineFrameInfo.h"
20#include "llvm/CodeGen/MachineFunction.h"
21#include "llvm/CodeGen/MachineInstrBuilder.h"
22#include "llvm/CodeGen/ScheduleDAG.h"
23#include "llvm/Support/Debug.h"
24#include "llvm/TargetParser/AMDGPUTargetParser.h"
25
26using namespace llvm;
27
28#define DEBUG_TYPE "gcn-hazard-recognizer"
29// Opt-in debug type for the per-candidate co-execution slot traces, which are
30// far too noisy for the normal debug output. Pass both types to get everything.
31#define DEBUG_TYPE_VERBOSE "gcn-hazard-recognizer-verbose"
32
33STATISTIC(NumWMMANopsHoisted,
34 "Number of WMMA hazard V_NOPs hoisted from loops");
35STATISTIC(NumWMMAHoistingBailed,
36 "Number of WMMA hazards where V_NOP hoisting was not possible");
37
38namespace {
39
40struct MFMAPaddingRatioParser : public cl::parser<unsigned> {
41 MFMAPaddingRatioParser(cl::Option &O) : cl::parser<unsigned>(O) {}
42
43 bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) {
44 if (Arg.getAsInteger(Radix: 0, Result&: Value))
45 return O.error(Message: "'" + Arg + "' value invalid for uint argument!");
46
47 if (Value > 100)
48 return O.error(Message: "'" + Arg + "' value must be in the range [0, 100]!");
49
50 return false;
51 }
52};
53
54} // end anonymous namespace
55
56static cl::opt<unsigned, false, MFMAPaddingRatioParser>
57 MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(Val: 0), cl::Hidden,
58 cl::desc("Fill a percentage of the latency between "
59 "neighboring MFMA with s_nops."));
60
61// This is intended for debugging purposes only.
62static cl::opt<unsigned>
63 NopPadding("amdgpu-snop-padding", cl::init(Val: 0), cl::Hidden,
64 cl::desc("Insert a s_nop x before every instruction"));
65
66static cl::opt<bool> EnableWMMAVnopHoisting(
67 "amdgpu-wmma-vnop-hoisting", cl::init(Val: true), cl::Hidden,
68 cl::desc("Hoist WMMA hazard V_NOPs from loops to preheaders"));
69
70//===----------------------------------------------------------------------===//
71// Hazard Recognizer Implementation
72//===----------------------------------------------------------------------===//
73
74static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF,
75 const GCNSubtarget &ST);
76
77GCNHazardRecognizer::GCNHazardRecognizer(
78 const MachineFunction &MF, GCNHazardRecognizer::OperatingMode Mode,
79 MachineLoopInfo *MLI)
80 : Mode(Mode), CurrCycleInstr(nullptr), MF(MF),
81 ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()),
82 TRI(TII.getRegisterInfo()), TSchedModel(TII.getSchedModel()), MLI(MLI),
83 ClauseUses(TRI.getNumRegUnits()), ClauseDefs(TRI.getNumRegUnits()) {
84 MaxLookAhead = MF.getRegInfo().isPhysRegUsed(PhysReg: AMDGPU::AGPR0) ? 19 : 5;
85 RunLdsBranchVmemWARHazardFixup = shouldRunLdsBranchVmemWARHazardFixup(MF, ST);
86 LLVM_DEBUG({
87 if (isPreRA())
88 dbgs() << " PreRA hazard recognizer: " << MF.getName() << "\n";
89 });
90}
91
92GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF,
93 MachineLoopInfo *MLI)
94 : GCNHazardRecognizer(MF, OperatingMode::PostRA, MLI) {}
95
96GCNHazardRecognizer::~GCNHazardRecognizer() {
97 // Dump any active co-execution window that did not complete naturally
98 // (e.g. region ended before the window expired).
99 LLVM_DEBUG({
100 if (CurrentCoExecStage.has_value()) {
101 unsigned Stage = *CurrentCoExecStage;
102 if (Stage < AMDGPU::MaxCoExecStages)
103 CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
104 dbgs() << " CoExec window ended at stage " << Stage << ":\n";
105 dumpCoExecWindow();
106 }
107 });
108}
109
110void GCNHazardRecognizer::Reset() {
111 EmittedInstrs.clear();
112 EmittedVALUInstrs.clear();
113 HasPendingWMMACoexecHazard = false;
114 if (isSchedulerMode())
115 schedulerReset();
116}
117
118void GCNHazardRecognizer::schedulerReset() {
119 LLVM_DEBUG({
120 if (CurrentCoExecStage.has_value() || CyclesUntilTRANS > 0 ||
121 CyclesUntilVALU > 0)
122 dbgs() << " Scheduler Reset: clearing co-exec window, TRANS="
123 << CyclesUntilTRANS << ", VALU=" << CyclesUntilVALU << "\n";
124 });
125 CurrentCoExecStage = std::nullopt;
126 CoExecWindowStartCycle = 0;
127 CyclesUntilTRANS = 0;
128 CyclesUntilVALU = 0;
129 ActiveCoExecInfo = AMDGPU::CoExecInfo();
130 CoExecWindowLog.fill(u: '.');
131}
132
133void GCNHazardRecognizer::dumpCoExecWindow() const {
134 unsigned W = ActiveCoExecInfo.TotalWindow;
135 if (W == 0)
136 return;
137
138 // Print the stage numbers row.
139 dbgs() << " Stages: ";
140 for (unsigned I = 0; I < W; ++I)
141 dbgs() << I % 10 << ' ';
142 dbgs() << '\n';
143
144 // Print the pattern row.
145 dbgs() << " Slots: ";
146 for (unsigned I = 0; I < W; ++I)
147 dbgs() << ActiveCoExecInfo.Pattern[I] << ' ';
148 dbgs() << '\n';
149
150 // Print the scheduled row.
151 dbgs() << " Scheduled: ";
152 for (unsigned I = 0; I < W; ++I)
153 dbgs() << CoExecWindowLog[I] << ' ';
154 dbgs() << '\n';
155}
156
157void GCNHazardRecognizer::schedulerAdvanceCycle() {
158 // Record what happened at the current stage of the co-exec window.
159 if (CurrentCoExecStage.has_value()) {
160 unsigned Stage = *CurrentCoExecStage;
161 if (Stage < AMDGPU::MaxCoExecStages) {
162 if (CurrCycleInstr)
163 CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
164 else
165 CoExecWindowLog[Stage] = '-';
166 }
167 }
168
169 LLVM_DEBUG({
170 bool HasState = CurrentCoExecStage.has_value() || CyclesUntilTRANS > 0 ||
171 CyclesUntilVALU > 0;
172 if (HasState) {
173 dbgs() << " Scheduler AdvanceCycle:";
174 if (CurrentCoExecStage.has_value()) {
175 unsigned Stage = *CurrentCoExecStage;
176 unsigned Next = Stage + 1;
177 if (Next >= ActiveCoExecInfo.TotalWindow)
178 dbgs() << " stage " << Stage << "->expired";
179 else
180 dbgs() << " stage " << Stage << "->" << Next;
181 }
182 if (CyclesUntilTRANS > 0)
183 dbgs() << " TRANS=" << CyclesUntilTRANS << "->"
184 << (CyclesUntilTRANS - 1);
185 if (CyclesUntilVALU > 0)
186 dbgs() << " VALU=" << CyclesUntilVALU << "->" << (CyclesUntilVALU - 1);
187 dbgs() << "\n";
188 }
189 });
190
191 // Decrement hazard counters.
192 if (CyclesUntilTRANS > 0)
193 --CyclesUntilTRANS;
194 if (CyclesUntilVALU > 0)
195 --CyclesUntilVALU;
196
197 // Advance WMMA co-execution window.
198 if (CurrentCoExecStage.has_value()) {
199 unsigned Stage = *CurrentCoExecStage + 1;
200 if (Stage >= ActiveCoExecInfo.TotalWindow) {
201 // Window expired.
202 LLVM_DEBUG({
203 dbgs() << " CoExec window complete:\n";
204 dumpCoExecWindow();
205 });
206 CurrentCoExecStage = std::nullopt;
207 } else {
208 CurrentCoExecStage = Stage;
209 }
210 }
211}
212
213bool GCNHazardRecognizer::hasCoExecWindowModel() const {
214 // The co-execution slot patterns returned by getCoExecInfo() are derived from
215 // gfx1250 timings, so the window model is restricted to gfx1250 for now.
216 // gfx1251 and gfx12.5-generic report the same co-execution hazard features
217 // but have different WMMA latencies, so they need their own slot patterns
218 // before they can be modeled here.
219 return ST.hasWMMACoexecutionHazards() && ST.hasTransCoexecutionHazard() &&
220 AMDGPU::isGFX1250(STI: ST);
221}
222
223void GCNHazardRecognizer::updateWMMAWindowState(const MachineInstr &MI) {
224 if (!hasCoExecWindowModel())
225 return;
226
227 // Check if this is a WMMA instruction.
228 if (!SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI))
229 return;
230
231 // If a previous window was still active, dump it before starting a new one.
232 // Record the current stage (filled by this new WMMA) before dumping.
233 LLVM_DEBUG({
234 if (CurrentCoExecStage.has_value()) {
235 unsigned Stage = *CurrentCoExecStage;
236 if (Stage < AMDGPU::MaxCoExecStages)
237 CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
238 dbgs() << " CoExec window interrupted at stage " << Stage << ":\n";
239 dumpCoExecWindow();
240 }
241 });
242
243 // Start a new co-execution window.
244 ActiveCoExecInfo = AMDGPU::getCoExecInfo(MI, TII);
245 CurrentCoExecStage = 0;
246 CoExecWindowLog.fill(u: '.');
247
248 LLVM_DEBUG(dbgs() << " WMMA window started: " << ActiveCoExecInfo.Pattern
249 << " (window=" << ActiveCoExecInfo.TotalWindow << ")\n"
250 << " " << MI);
251}
252
253void GCNHazardRecognizer::updateTRANSState(const MachineInstr &MI) {
254 if (!hasCoExecWindowModel())
255 return;
256 if (!SIInstrInfo::isTRANS(MI))
257 return;
258
259 // Back-to-back TRANS instructions have a 1-cycle hazard.
260 // This is checked via checkTRANSHazard() and does not create a co-exec
261 // window. The TRANS shadow slot allows anything except TRANS and
262 // multi-cycle VALU.
263 // Set to 2: bumpCycle advances to the next pick's cycle (decrementing
264 // by 1 via AdvanceCycle) before the next instruction's hazard check, so
265 // the counter is observed at 1 there. That 1-cycle stall lets the
266 // strategy pick a non-TRANS, non-multi-cycle-VALU candidate to fill the
267 // shadow slot.
268 CyclesUntilTRANS = 2;
269 LLVM_DEBUG(dbgs() << " TRANS hazard set: CyclesUntilTRANS=2\n");
270}
271
272void GCNHazardRecognizer::updateMultiCycleVALUState(const MachineInstr &MI) {
273 if (!hasCoExecWindowModel())
274 return;
275 // Multi-cycle VALU (CVT, etc.) blocks subsequent VALU for repeat rate cycles.
276 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
277 return;
278
279 // Skip WMMA and TRANS - they have their own tracking.
280 if (SIInstrInfo::isWMMA(MI) || SIInstrInfo::isSWMMAC(MI) ||
281 SIInstrInfo::isTRANS(MI))
282 return;
283
284 unsigned RepeatRate = TII.getRepeatRate(MI);
285 if (RepeatRate > 1) {
286 // bumpCycle's AdvanceCycle decrements once before the next pick's
287 // hazard check (same convention as CyclesUntilTRANS), so to expose
288 // RepeatRate-1 cycles of shadow we must seed with RepeatRate.
289 CyclesUntilVALU = RepeatRate;
290 LLVM_DEBUG(dbgs() << " Multi-cycle VALU: repeat=" << RepeatRate
291 << ", CyclesUntilVALU=" << CyclesUntilVALU << "\n");
292 }
293}
294
295AMDGPU::CoExecMaskT
296GCNHazardRecognizer::getCoExecMaskForMI(const MachineInstr &MI,
297 const SIInstrInfo &TII) {
298 return AMDGPU::getCoExecMask(F: AMDGPU::classifyFlavor(MI, SII: TII));
299}
300
301unsigned GCNHazardRecognizer::checkTRANSHazard(const MachineInstr &MI) const {
302 if (!CyclesUntilTRANS)
303 return 0;
304
305 // Only TRANS and multi-cycle VALU are blocked by the TRANS shadow.
306 if (SIInstrInfo::isTRANS(MI))
307 return CyclesUntilTRANS;
308
309 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
310 !SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI) &&
311 TII.getRepeatRate(MI) > 1)
312 return CyclesUntilTRANS;
313
314 return 0;
315}
316
317unsigned
318GCNHazardRecognizer::checkMultiCycleVALUHazard(const MachineInstr &MI) const {
319 if (!CyclesUntilVALU)
320 return 0;
321
322 // Multi-cycle VALU blocks anything on the VALU pipe - VALU, WMMA, SWMMAC,
323 // and TRANS - for RepeatRate-1 cycles. Only off-pipe instructions (MEM,
324 // SALU, control) can fill the shadow.
325 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
326 !SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI) &&
327 !SIInstrInfo::isTRANS(MI))
328 return 0;
329
330 return CyclesUntilVALU;
331}
332
333unsigned
334GCNHazardRecognizer::checkWMMACoexecSlot(const MachineInstr &MI) const {
335 // No hazard if not in a WMMA window.
336 if (!CurrentCoExecStage.has_value())
337 return 0;
338
339 unsigned Stage = *CurrentCoExecStage;
340 AMDGPU::CoExecMaskT InstMask = getCoExecMaskForMI(MI, TII);
341 // Check if the instruction can co-execute at the current stage.
342 if (ActiveCoExecInfo.canCoExec(InstMask, Stage))
343 return 0;
344
345 // Find next allowed stage and return stall cycles.
346 auto NextStage = ActiveCoExecInfo.findNextAllowedStage(InstMask, FromStage: Stage);
347 if (NextStage.has_value()) {
348 unsigned StallCycles = *NextStage - Stage;
349 DEBUG_WITH_TYPE(
350 DEBUG_TYPE_VERBOSE,
351 dbgs() << " CoExec stall: stage=" << Stage << "("
352 << AMDGPU::getStageTypeName(ActiveCoExecInfo.getType(Stage))
353 << ") mask=" << AMDGPU::getCoExecMaskName(InstMask)
354 << " -> stall " << StallCycles << " (next allowed=" << *NextStage
355 << ")\n"
356 << " " << MI);
357 return StallCycles;
358 }
359
360 // No compatible slot in window - stall until window ends.
361 unsigned StallCycles = ActiveCoExecInfo.TotalWindow - Stage;
362 DEBUG_WITH_TYPE(
363 DEBUG_TYPE_VERBOSE,
364 dbgs() << " CoExec stall: stage=" << Stage << "("
365 << AMDGPU::getStageTypeName(ActiveCoExecInfo.getType(Stage))
366 << ") mask=" << AMDGPU::getCoExecMaskName(InstMask) << " -> stall "
367 << StallCycles << " (window ends)\n"
368 << " " << MI);
369 return StallCycles;
370}
371
372unsigned
373GCNHazardRecognizer::checkMultiShadowHazard(const MachineInstr &MI) const {
374 // This models a VALU caught in both a WMMA and a TRANS shadow.
375 if (!hasCoExecWindowModel())
376 return 0;
377
378 // No hazard if not in a WMMA window.
379 if (!CurrentCoExecStage.has_value())
380 return 0;
381
382 if (!CyclesUntilTRANS)
383 return 0;
384
385 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ||
386 SIInstrInfo::isLDSDMA(MI))
387 return 0;
388
389 // We have a VALU instruction that is under both a TRANS and WMMA shadow.
390 // We need to wait for at least one to clear.
391
392 unsigned LookAheadStage = *CurrentCoExecStage + CyclesUntilTRANS;
393 AMDGPU::CoExecMaskT InstMask = getCoExecMaskForMI(MI, TII);
394 // Check if the instruction can co-execute at the current stage.
395 if (ActiveCoExecInfo.canCoExec(InstMask, Stage: LookAheadStage))
396 return CyclesUntilTRANS;
397
398 // Find next allowed stage and return stall cycles.
399 auto NextStage =
400 ActiveCoExecInfo.findNextAllowedStage(InstMask, FromStage: LookAheadStage);
401 if (NextStage.has_value()) {
402 unsigned StallCycles = *NextStage - *CurrentCoExecStage;
403 return StallCycles;
404 }
405
406 // No compatible slot in window - stall until window ends.
407 unsigned StallCycles = ActiveCoExecInfo.TotalWindow - *CurrentCoExecStage;
408 return StallCycles;
409}
410
411void GCNHazardRecognizer::schedulerEmitInstruction(MachineInstr *MI) {
412 LLVM_DEBUG({
413 bool InWindow = CurrentCoExecStage.has_value();
414 bool HasActiveState =
415 InWindow || CyclesUntilTRANS > 0 || CyclesUntilVALU > 0;
416 if (HasActiveState) {
417 if (InWindow) {
418 unsigned Stage = *CurrentCoExecStage;
419 dbgs() << " Stage " << Stage << "("
420 << AMDGPU::getStageTypeName(ActiveCoExecInfo.getType(Stage))
421 << ") Emit ["
422 << AMDGPU::getCoExecMaskName(getCoExecMaskForMI(*MI, TII))
423 << "]: " << *MI;
424 } else {
425 dbgs() << " Emit ["
426 << AMDGPU::getCoExecMaskName(getCoExecMaskForMI(*MI, TII))
427 << "]: " << *MI;
428 }
429 }
430 });
431 DEBUG_WITH_TYPE(DEBUG_TYPE_VERBOSE, {
432 bool HasActiveState = CurrentCoExecStage.has_value() ||
433 CyclesUntilTRANS > 0 || CyclesUntilVALU > 0;
434 if (!HasActiveState)
435 dbgs() << " Emit ["
436 << AMDGPU::getCoExecMaskName(getCoExecMaskForMI(*MI, TII))
437 << "]: " << *MI;
438 });
439 updateWMMAWindowState(MI: *MI);
440 updateTRANSState(MI: *MI);
441 updateMultiCycleVALUState(MI: *MI);
442}
443
444void GCNHazardRecognizer::EmitInstruction(SUnit *SU) {
445 EmitInstruction(MI: SU->getInstr());
446}
447
448void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) {
449 CurrCycleInstr = MI;
450 if (isSchedulerMode())
451 schedulerEmitInstruction(MI);
452}
453
454static bool isDivFMas(unsigned Opcode) {
455 return Opcode == AMDGPU::V_DIV_FMAS_F32_e64 || Opcode == AMDGPU::V_DIV_FMAS_F64_e64;
456}
457
458static bool isSGetReg(unsigned Opcode) {
459 return Opcode == AMDGPU::S_GETREG_B32 || Opcode == AMDGPU::S_GETREG_B32_const;
460}
461
462static bool isSSetReg(unsigned Opcode) {
463 switch (Opcode) {
464 case AMDGPU::S_SETREG_B32:
465 case AMDGPU::S_SETREG_B32_mode:
466 case AMDGPU::S_SETREG_IMM32_B32:
467 case AMDGPU::S_SETREG_IMM32_B32_mode:
468 return true;
469 }
470 return false;
471}
472
473static bool isRWLane(unsigned Opcode) {
474 return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32;
475}
476
477static bool isRFE(unsigned Opcode) {
478 return Opcode == AMDGPU::S_RFE_B64;
479}
480
481static bool isSMovRel(unsigned Opcode) {
482 switch (Opcode) {
483 case AMDGPU::S_MOVRELS_B32:
484 case AMDGPU::S_MOVRELS_B64:
485 case AMDGPU::S_MOVRELD_B32:
486 case AMDGPU::S_MOVRELD_B64:
487 return true;
488 default:
489 return false;
490 }
491}
492
493static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII,
494 const MachineInstr &MI) {
495 if (TII.isAlwaysGDS(Opcode: MI.getOpcode()))
496 return true;
497
498 switch (MI.getOpcode()) {
499 case AMDGPU::S_SENDMSG:
500 case AMDGPU::S_SENDMSGHALT:
501 case AMDGPU::S_TTRACEDATA:
502 return true;
503 // These DS opcodes don't support GDS.
504 case AMDGPU::DS_NOP:
505 case AMDGPU::DS_PERMUTE_B32:
506 case AMDGPU::DS_BPERMUTE_B32:
507 return false;
508 default:
509 if (TII.isDS(Opcode: MI.getOpcode())) {
510 int GDS = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
511 Name: AMDGPU::OpName::gds);
512 if (MI.getOperand(i: GDS).getImm())
513 return true;
514 }
515 return false;
516 }
517}
518
519static bool isPermlane(const MachineInstr &MI) {
520 unsigned Opcode = MI.getOpcode();
521 return Opcode == AMDGPU::V_PERMLANE16_B32_e64 ||
522 Opcode == AMDGPU::V_PERMLANE64_B32 ||
523 Opcode == AMDGPU::V_PERMLANEX16_B32_e64 ||
524 Opcode == AMDGPU::V_PERMLANE16_VAR_B32_e64 ||
525 Opcode == AMDGPU::V_PERMLANEX16_VAR_B32_e64 ||
526 Opcode == AMDGPU::V_PERMLANE16_SWAP_B32_e32 ||
527 Opcode == AMDGPU::V_PERMLANE16_SWAP_B32_e64 ||
528 Opcode == AMDGPU::V_PERMLANE32_SWAP_B32_e32 ||
529 Opcode == AMDGPU::V_PERMLANE32_SWAP_B32_e64 ||
530 Opcode == AMDGPU::V_PERMLANE_BCAST_B32_e64 ||
531 Opcode == AMDGPU::V_PERMLANE_UP_B32_e64 ||
532 Opcode == AMDGPU::V_PERMLANE_DOWN_B32_e64 ||
533 Opcode == AMDGPU::V_PERMLANE_XOR_B32_e64 ||
534 Opcode == AMDGPU::V_PERMLANE_IDX_GEN_B32_e64;
535}
536
537static bool isLdsDma(const MachineInstr &MI) {
538 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
539 (SIInstrInfo::isMUBUF(MI) || SIInstrInfo::isFLAT(MI));
540}
541
542static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) {
543 const MachineOperand *RegOp = TII->getNamedOperand(MI: RegInstr,
544 OperandName: AMDGPU::OpName::simm16);
545 return std::get<0>(t: AMDGPU::Hwreg::HwregEncoding::decode(Encoded: RegOp->getImm()));
546}
547
548ScheduleHazardRecognizer::HazardType
549GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
550 MachineInstr *MI = SU->getInstr();
551 // If we are not in "HazardRecognizerMode" and therefore not being run from
552 // the scheduler, track possible stalls from hazards but don't insert noops.
553 auto HazardType = isHazardRecognizerMode() ? NoopHazard : Hazard;
554
555 if (MI->isBundle())
556 return NoHazard;
557
558 // Check co-execution slot hazards and pipeline stalls in scheduler modes.
559 if (isSchedulerMode()) {
560 if (checkMultiShadowHazard(MI: *MI) > 0)
561 return Hazard;
562 if (checkWMMACoexecSlot(MI: *MI) > 0)
563 return Hazard;
564 if (checkTRANSHazard(MI: *MI) > 0)
565 return Hazard;
566 if (checkMultiCycleVALUHazard(MI: *MI) > 0)
567 return Hazard;
568 // The remaining checks are all defined by register dependences.
569 if (!hasPhysRegs())
570 return NoHazard;
571 }
572
573 if (SIInstrInfo::isSMRD(MI: *MI) && checkSMRDHazards(SMRD: MI) > 0)
574 return HazardType;
575
576 if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0)
577 return HazardType;
578
579 if (checkFPAtomicToDenormModeHazard(MI) > 0)
580 return HazardType;
581
582 // Hazards which cannot be mitigated with S_NOPs.
583 if (!isHazardRecognizerMode()) {
584 if (checkWMMACoexecutionHazards(MI) > 0) {
585 HasPendingWMMACoexecHazard = true;
586 return Hazard;
587 }
588 }
589
590 if (ST.hasNoDataDepHazard())
591 return NoHazard;
592
593 if (SIInstrInfo::isVMEM(MI: *MI) && checkVMEMHazards(VMEM: MI) > 0)
594 return HazardType;
595
596 if (SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true) &&
597 checkVALUHazards(VALU: MI) > 0)
598 return HazardType;
599
600 if (SIInstrInfo::isDPP(MI: *MI) && checkDPPHazards(DPP: MI) > 0)
601 return HazardType;
602
603 if (isDivFMas(Opcode: MI->getOpcode()) && checkDivFMasHazards(DivFMas: MI) > 0)
604 return HazardType;
605
606 if (isRWLane(Opcode: MI->getOpcode()) && checkRWLaneHazards(RWLane: MI) > 0)
607 return HazardType;
608
609 if ((SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true) ||
610 SIInstrInfo::isVMEM(MI: *MI) || SIInstrInfo::isDS(MI: *MI) ||
611 SIInstrInfo::isEXP(MI: *MI)) &&
612 checkMAIVALUHazards(MI) > 0)
613 return HazardType;
614
615 if (isSGetReg(Opcode: MI->getOpcode()) && checkGetRegHazards(GetRegInstr: MI) > 0)
616 return HazardType;
617
618 if (isSSetReg(Opcode: MI->getOpcode()) && checkSetRegHazards(SetRegInstr: MI) > 0)
619 return HazardType;
620
621 if (isRFE(Opcode: MI->getOpcode()) && checkRFEHazards(RFE: MI) > 0)
622 return HazardType;
623
624 if (((ST.hasReadM0MovRelInterpHazard() &&
625 (TII.isVINTRP(MI: *MI) || isSMovRel(Opcode: MI->getOpcode()) ||
626 MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 ||
627 MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) ||
628 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, MI: *MI)) ||
629 (ST.hasReadM0LdsDmaHazard() && isLdsDma(MI: *MI)) ||
630 (ST.hasReadM0LdsDirectHazard() &&
631 MI->readsRegister(Reg: AMDGPU::LDS_DIRECT, /*TRI=*/nullptr))) &&
632 checkReadM0Hazards(SMovRel: MI) > 0)
633 return HazardType;
634
635 if (SIInstrInfo::isMAI(MI: *MI) && checkMAIHazards(MI) > 0)
636 return HazardType;
637
638 if ((SIInstrInfo::isVMEM(MI: *MI) || SIInstrInfo::isDS(MI: *MI)) &&
639 checkMAILdStHazards(MI) > 0)
640 return HazardType;
641
642 if (MI->isInlineAsm() && checkInlineAsmHazards(IA: MI) > 0)
643 return HazardType;
644
645 return NoHazard;
646}
647
648static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII,
649 unsigned Quantity) {
650 while (Quantity > 0) {
651 unsigned Arg = std::min(a: Quantity, b: 8u);
652 Quantity -= Arg;
653 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_NOP))
654 .addImm(Val: Arg - 1);
655 }
656}
657
658unsigned
659GCNHazardRecognizer::getMFMAPipelineWaitStates(const MachineInstr &MI) const {
660 const MCSchedClassDesc *SC = TSchedModel.resolveSchedClass(MI: &MI);
661 assert(TSchedModel.getWriteProcResBegin(SC) !=
662 TSchedModel.getWriteProcResEnd(SC));
663 return TSchedModel.getWriteProcResBegin(SC)->ReleaseAtCycle;
664}
665
666void GCNHazardRecognizer::processBundle() {
667 MachineBasicBlock::instr_iterator MI = std::next(x: CurrCycleInstr->getIterator());
668 MachineBasicBlock::instr_iterator E = CurrCycleInstr->getParent()->instr_end();
669 // Check bundled MachineInstr's for hazards.
670 for (; MI != E && MI->isInsideBundle(); ++MI) {
671 CurrCycleInstr = &*MI;
672 unsigned WaitStates = PreEmitNoopsCommon(CurrCycleInstr);
673
674 if (isHazardRecognizerMode()) {
675 fixHazards(MI: CurrCycleInstr);
676
677 insertNoopsInBundle(MI: CurrCycleInstr, TII, Quantity: WaitStates);
678 }
679
680 // It’s unnecessary to track more than MaxLookAhead instructions. Since we
681 // include the bundled MI directly after, only add a maximum of
682 // (MaxLookAhead - 1) noops to EmittedInstrs.
683 for (unsigned i = 0, e = std::min(a: WaitStates, b: MaxLookAhead - 1); i < e; ++i)
684 EmittedInstrs.push_front(x: nullptr);
685
686 EmittedInstrs.push_front(x: CurrCycleInstr);
687 EmittedInstrs.resize(new_size: MaxLookAhead);
688 }
689 CurrCycleInstr = nullptr;
690}
691
692void GCNHazardRecognizer::runOnInstruction(MachineInstr *MI) {
693 assert(isHazardRecognizerMode());
694
695 unsigned NumPreNoops = PreEmitNoops(MI);
696 EmitNoops(Quantity: NumPreNoops);
697 if (MI->isInsideBundle())
698 insertNoopsInBundle(MI, TII, Quantity: NumPreNoops);
699 else
700 TII.insertNoops(MBB&: *MI->getParent(), MI: MachineBasicBlock::iterator(MI),
701 Quantity: NumPreNoops);
702 EmitInstruction(MI);
703 AdvanceCycle();
704}
705
706unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
707 assert(isHazardRecognizerMode());
708 CurrCycleInstr = MI;
709 unsigned W = PreEmitNoopsCommon(MI);
710 fixHazards(MI);
711 CurrCycleInstr = nullptr;
712 return std::max(a: W, b: NopPadding.getValue());
713}
714
715unsigned GCNHazardRecognizer::getHazardWaitStates(MachineInstr *MI) const {
716 unsigned W = 0;
717
718 // Check co-execution slot hazards and pipeline stalls in scheduler modes.
719 if (isSchedulerMode()) {
720 W = checkWMMACoexecSlot(MI: *MI);
721 W = std::max(a: W, b: checkTRANSHazard(MI: *MI));
722 W = std::max(a: W, b: checkMultiCycleVALUHazard(MI: *MI));
723 W = std::max(a: W, b: checkMultiShadowHazard(MI: *MI));
724 // The remaining checks are all defined by register dependences.
725 if (!hasPhysRegs())
726 return W;
727 }
728
729 return std::max(a: W, b: PreEmitNoopsCommon(MI));
730}
731
732unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) const {
733 if (MI->isBundle())
734 return 0;
735
736 int WaitStates = 0;
737
738 if (SIInstrInfo::isSMRD(MI: *MI))
739 return std::max(a: WaitStates, b: checkSMRDHazards(SMRD: MI));
740
741 if (ST.hasNSAtoVMEMBug())
742 WaitStates = std::max(a: WaitStates, b: checkNSAtoVMEMHazard(MI));
743
744 WaitStates = std::max(a: WaitStates, b: checkFPAtomicToDenormModeHazard(MI));
745
746 if (ST.hasNoDataDepHazard())
747 return WaitStates;
748
749 if (SIInstrInfo::isVMEM(MI: *MI))
750 WaitStates = std::max(a: WaitStates, b: checkVMEMHazards(VMEM: MI));
751
752 if (SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true))
753 WaitStates = std::max(a: WaitStates, b: checkVALUHazards(VALU: MI));
754
755 if (SIInstrInfo::isDPP(MI: *MI))
756 WaitStates = std::max(a: WaitStates, b: checkDPPHazards(DPP: MI));
757
758 if (isDivFMas(Opcode: MI->getOpcode()))
759 WaitStates = std::max(a: WaitStates, b: checkDivFMasHazards(DivFMas: MI));
760
761 if (isRWLane(Opcode: MI->getOpcode()))
762 WaitStates = std::max(a: WaitStates, b: checkRWLaneHazards(RWLane: MI));
763
764 if ((SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true) ||
765 SIInstrInfo::isVMEM(MI: *MI) || SIInstrInfo::isDS(MI: *MI) ||
766 SIInstrInfo::isEXP(MI: *MI)) &&
767 checkMAIVALUHazards(MI) > 0)
768 WaitStates = std::max(a: WaitStates, b: checkMAIVALUHazards(MI));
769
770 if (MI->isInlineAsm())
771 return std::max(a: WaitStates, b: checkInlineAsmHazards(IA: MI));
772
773 if (isSGetReg(Opcode: MI->getOpcode()))
774 return std::max(a: WaitStates, b: checkGetRegHazards(GetRegInstr: MI));
775
776 if (isSSetReg(Opcode: MI->getOpcode()))
777 return std::max(a: WaitStates, b: checkSetRegHazards(SetRegInstr: MI));
778
779 if (isRFE(Opcode: MI->getOpcode()))
780 return std::max(a: WaitStates, b: checkRFEHazards(RFE: MI));
781
782 if ((ST.hasReadM0MovRelInterpHazard() &&
783 (TII.isVINTRP(MI: *MI) || isSMovRel(Opcode: MI->getOpcode()) ||
784 MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 ||
785 MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) ||
786 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, MI: *MI)) ||
787 (ST.hasReadM0LdsDmaHazard() && isLdsDma(MI: *MI)) ||
788 (ST.hasReadM0LdsDirectHazard() &&
789 MI->readsRegister(Reg: AMDGPU::LDS_DIRECT, /*TRI=*/nullptr)))
790 return std::max(a: WaitStates, b: checkReadM0Hazards(SMovRel: MI));
791
792 if (SIInstrInfo::isMAI(MI: *MI))
793 return std::max(a: WaitStates, b: checkMAIHazards(MI));
794
795 if (SIInstrInfo::isVMEM(MI: *MI) || SIInstrInfo::isDS(MI: *MI))
796 return std::max(a: WaitStates, b: checkMAILdStHazards(MI));
797
798 if (ST.hasGFX950Insts() && isPermlane(MI: *MI))
799 return std::max(a: WaitStates, b: checkPermlaneHazards(MI));
800
801 return WaitStates;
802}
803
804void GCNHazardRecognizer::EmitNoop() {
805 EmittedInstrs.push_front(x: nullptr);
806}
807
808void GCNHazardRecognizer::AdvanceCycle() {
809 if (isSchedulerMode())
810 schedulerAdvanceCycle();
811
812 // When the scheduler detects a stall, it will call AdvanceCycle() without
813 // emitting any instructions.
814 if (!CurrCycleInstr) {
815 EmittedInstrs.push_front(x: nullptr);
816
817 if (HasPendingWMMACoexecHazard)
818 EmittedVALUInstrs.push_front(x: nullptr);
819 return;
820 }
821
822 HasPendingWMMACoexecHazard = false;
823
824 if (CurrCycleInstr->isBundle()) {
825 processBundle();
826 return;
827 }
828
829 unsigned NumWaitStates = TII.getNumWaitStates(MI: *CurrCycleInstr);
830 if (!NumWaitStates) {
831 CurrCycleInstr = nullptr;
832 return;
833 }
834
835 // Keep track of emitted instructions
836 EmittedInstrs.push_front(x: CurrCycleInstr);
837
838 bool IsVALUOrWMMA =
839 SIInstrInfo::isVALU(MI: *CurrCycleInstr, /*AllowLDSDMA=*/true) ||
840 SIInstrInfo::isWMMA(MI: *CurrCycleInstr) ||
841 SIInstrInfo::isSWMMAC(MI: *CurrCycleInstr);
842 if (IsVALUOrWMMA) {
843 EmittedVALUInstrs.push_front(x: CurrCycleInstr);
844 } else {
845 // A pending WMMA co-execution hazard optimistically records stall cycles as
846 // future V_NOPs. If the scheduler instead stalls for a different
847 // (S_NOP-resolvable) hazard and schedules a non-VALU into those cycles,
848 // they will not resolve the VALU-pipe hazard, so drop them here.
849 while (!EmittedVALUInstrs.empty() && EmittedVALUInstrs.front() == nullptr)
850 EmittedVALUInstrs.pop_front();
851 }
852
853 // Add a nullptr for each additional wait state after the first. Make sure
854 // not to add more than getMaxLookAhead() items to the list, since we
855 // truncate the list to that size right after this loop.
856 for (unsigned i = 1, e = std::min(a: NumWaitStates, b: getMaxLookAhead());
857 i < e; ++i) {
858 EmittedInstrs.push_front(x: nullptr);
859 }
860
861 // getMaxLookahead() is the largest number of wait states we will ever need
862 // to insert, so there is no point in keeping track of more than that many
863 // wait states.
864 EmittedInstrs.resize(new_size: getMaxLookAhead());
865 if (EmittedVALUInstrs.size() > MaxVALULookAhead)
866 EmittedVALUInstrs.resize(new_size: MaxVALULookAhead);
867
868 CurrCycleInstr = nullptr;
869}
870
871void GCNHazardRecognizer::RecedeCycle() {
872 assert(!isHazardRecognizerMode() &&
873 "Bottom-up scheduling shouldn't run in hazard recognizer mode");
874}
875
876//===----------------------------------------------------------------------===//
877// Helper Functions
878//===----------------------------------------------------------------------===//
879
880enum HazardFnResult { HazardFound, HazardExpired, NoHazardFound };
881
882// Search for a hazard in a block and its predecessors.
883template <typename StateT>
884static bool
885hasHazard(StateT InitialState,
886 function_ref<HazardFnResult(StateT &, const MachineInstr &)> IsHazard,
887 function_ref<void(StateT &, const MachineInstr &)> UpdateState,
888 const MachineBasicBlock *InitialMBB,
889 MachineBasicBlock::const_reverse_instr_iterator InitialI) {
890 struct StateMapKey {
891 SmallVectorImpl<StateT> *States;
892 unsigned Idx;
893 static bool isEqual(const StateMapKey &LHS, const StateMapKey &RHS) {
894 return LHS.States == RHS.States && LHS.Idx == RHS.Idx;
895 }
896 };
897 struct StateMapKeyTraits : DenseMapInfo<StateMapKey> {
898 static unsigned getHashValue(const StateMapKey &Key) {
899 return StateT::getHashValue((*Key.States)[Key.Idx]);
900 }
901 static unsigned getHashValue(const StateT &State) {
902 return StateT::getHashValue(State);
903 }
904 static bool isEqual(const StateMapKey &LHS, const StateMapKey &RHS) {
905 return StateT::isEqual((*LHS.States)[LHS.Idx], (*RHS.States)[RHS.Idx]);
906 }
907 static bool isEqual(const StateT &LHS, const StateMapKey &RHS) {
908 return StateT::isEqual(LHS, (*RHS.States)[RHS.Idx]);
909 }
910 };
911
912 SmallDenseMap<StateMapKey, unsigned, 8, StateMapKeyTraits> StateMap;
913 SmallVector<StateT, 8> States;
914
915 MachineBasicBlock::const_reverse_instr_iterator I = InitialI;
916 const MachineBasicBlock *MBB = InitialMBB;
917 StateT State = InitialState;
918
919 SmallSetVector<std::pair<const MachineBasicBlock *, unsigned>, 16> Worklist;
920 unsigned WorkIdx = 0;
921 for (;;) {
922 bool Expired = false;
923 for (auto E = MBB->instr_rend(); I != E; ++I) {
924 // No need to look at parent BUNDLE instructions.
925 if (I->isBundle())
926 continue;
927
928 auto Result = IsHazard(State, *I);
929 if (Result == HazardFound)
930 return true;
931 if (Result == HazardExpired) {
932 Expired = true;
933 break;
934 }
935
936 if (I->isInlineAsm() || I->isMetaInstruction())
937 continue;
938
939 UpdateState(State, *I);
940 }
941
942 if (!Expired) {
943 unsigned StateIdx = States.size();
944 StateMapKey Key = {&States, StateIdx};
945 auto Insertion = StateMap.insert_as(std::pair(Key, StateIdx), State);
946 if (Insertion.second) {
947 States.emplace_back(State);
948 } else {
949 StateIdx = Insertion.first->second;
950 }
951 for (MachineBasicBlock *Pred : MBB->predecessors())
952 Worklist.insert(X: std::pair(Pred, StateIdx));
953 }
954
955 if (WorkIdx == Worklist.size())
956 break;
957
958 unsigned StateIdx;
959 std::tie(args&: MBB, args&: StateIdx) = Worklist[WorkIdx++];
960 State = States[StateIdx];
961 I = MBB->instr_rbegin();
962 }
963
964 return false;
965}
966
967// Returns a minimum wait states since \p I walking all predecessors.
968// Only scans until \p IsExpired does not return true.
969// Can only be run in a hazard recognizer mode.
970static int
971getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
972 const MachineBasicBlock *MBB,
973 MachineBasicBlock::const_reverse_instr_iterator I,
974 int WaitStates, GCNHazardRecognizer::IsExpiredFn IsExpired,
975 DenseSet<const MachineBasicBlock *> &Visited,
976 GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates =
977 SIInstrInfo::getNumWaitStates) {
978 for (auto E = MBB->instr_rend(); I != E; ++I) {
979 // Don't add WaitStates for parent BUNDLE instructions.
980 if (I->isBundle())
981 continue;
982
983 if (IsHazard(*I))
984 return WaitStates;
985
986 if (I->isInlineAsm())
987 continue;
988
989 WaitStates += GetNumWaitStates(*I);
990
991 if (IsExpired(*I, WaitStates))
992 return std::numeric_limits<int>::max();
993 }
994
995 int MinWaitStates = std::numeric_limits<int>::max();
996 for (MachineBasicBlock *Pred : MBB->predecessors()) {
997 if (!Visited.insert(V: Pred).second)
998 continue;
999
1000 int W = getWaitStatesSince(IsHazard, MBB: Pred, I: Pred->instr_rbegin(), WaitStates,
1001 IsExpired, Visited, GetNumWaitStates);
1002
1003 MinWaitStates = std::min(a: MinWaitStates, b: W);
1004 }
1005
1006 return MinWaitStates;
1007}
1008
1009static int
1010getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
1011 const MachineInstr *MI,
1012 GCNHazardRecognizer::IsExpiredFn IsExpired,
1013 GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates =
1014 SIInstrInfo::getNumWaitStates) {
1015 DenseSet<const MachineBasicBlock *> Visited;
1016 return getWaitStatesSince(IsHazard, MBB: MI->getParent(),
1017 I: std::next(x: MI->getReverseIterator()), WaitStates: 0, IsExpired,
1018 Visited, GetNumWaitStates);
1019}
1020
1021int GCNHazardRecognizer::getWaitStatesSince(
1022 IsHazardFn IsHazard, int Limit, GetNumWaitStatesFn GetNumWaitStates) const {
1023 if (isHazardRecognizerMode()) {
1024 auto IsExpiredFn = [Limit](const MachineInstr &, int WaitStates) {
1025 return WaitStates >= Limit;
1026 };
1027 return ::getWaitStatesSince(IsHazard, MI: CurrCycleInstr, IsExpired: IsExpiredFn,
1028 GetNumWaitStates);
1029 }
1030
1031 int WaitStates = 0;
1032 for (MachineInstr *MI : EmittedInstrs) {
1033 if (MI) {
1034 if (IsHazard(*MI))
1035 return WaitStates;
1036
1037 if (MI->isInlineAsm())
1038 continue;
1039 }
1040 WaitStates += MI ? GetNumWaitStates(*MI) : 1;
1041
1042 if (WaitStates >= Limit)
1043 break;
1044 }
1045 return std::numeric_limits<int>::max();
1046}
1047
1048int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard,
1049 int Limit) const {
1050 return getWaitStatesSince(IsHazard, Limit, GetNumWaitStates: SIInstrInfo::getNumWaitStates);
1051}
1052
1053int GCNHazardRecognizer::getWaitStatesSinceVALU(IsHazardFn IsHazard,
1054 int Limit) const {
1055 if (isHazardRecognizerMode()) {
1056 auto GetVALUWaitStates = [](const MachineInstr &MI) -> unsigned {
1057 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ? 1 : 0;
1058 };
1059 return getWaitStatesSince(IsHazard, Limit, GetNumWaitStates: GetVALUWaitStates);
1060 }
1061
1062 // EmittedVALUInstrs is capped at MaxVALULookAhead, so a Limit beyond that
1063 // window could miss a hazard. Keep the cap in sync with the wait-state
1064 // tables.
1065 assert(Limit <= (int)MaxVALULookAhead &&
1066 "Limit exceeds the EmittedVALUInstrs lookahead window");
1067 int WaitStates = 0;
1068 for (MachineInstr *MI : EmittedVALUInstrs) {
1069 if (MI) {
1070 if (IsHazard(*MI))
1071 return WaitStates;
1072 }
1073
1074 ++WaitStates;
1075
1076 if (WaitStates >= Limit)
1077 break;
1078 }
1079 return std::numeric_limits<int>::max();
1080}
1081
1082int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg,
1083 IsHazardFn IsHazardDef,
1084 int Limit) const {
1085 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1086
1087 auto IsHazardFn = [IsHazardDef, TRI, Reg](const MachineInstr &MI) {
1088 return IsHazardDef(MI) && MI.modifiesRegister(Reg, TRI);
1089 };
1090
1091 return getWaitStatesSince(IsHazard: IsHazardFn, Limit);
1092}
1093
1094int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard,
1095 int Limit) const {
1096 auto IsHazardFn = [IsHazard](const MachineInstr &MI) {
1097 return isSSetReg(Opcode: MI.getOpcode()) && IsHazard(MI);
1098 };
1099
1100 return getWaitStatesSince(IsHazard: IsHazardFn, Limit);
1101}
1102
1103//===----------------------------------------------------------------------===//
1104// No-op Hazard Detection
1105//===----------------------------------------------------------------------===//
1106
1107static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV,
1108 MCRegister Reg) {
1109 for (MCRegUnit Unit : TRI.regunits(Reg))
1110 BV.set(static_cast<unsigned>(Unit));
1111}
1112
1113static void addRegsToSet(const SIRegisterInfo &TRI,
1114 iterator_range<MachineInstr::const_mop_iterator> Ops,
1115 BitVector &DefSet, BitVector &UseSet) {
1116 for (const MachineOperand &Op : Ops) {
1117 if (Op.isReg())
1118 addRegUnits(TRI, BV&: Op.isDef() ? DefSet : UseSet, Reg: Op.getReg().asMCReg());
1119 }
1120}
1121
1122void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) const {
1123 addRegsToSet(TRI, Ops: MI.operands(), DefSet&: ClauseDefs, UseSet&: ClauseUses);
1124}
1125
1126static bool breaksSMEMSoftClause(MachineInstr *MI) {
1127 return !SIInstrInfo::isSMRD(MI: *MI);
1128}
1129
1130static bool breaksVMEMSoftClause(MachineInstr *MI) {
1131 return !SIInstrInfo::isVMEM(MI: *MI);
1132}
1133
1134int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) const {
1135 // SMEM soft clause are only present on VI+, and only matter if xnack is
1136 // enabled.
1137 if (!ST.isXNACKEnabled())
1138 return 0;
1139
1140 bool IsSMRD = TII.isSMRD(MI: *MEM);
1141
1142 resetClause();
1143
1144 // A soft-clause is any group of consecutive SMEM instructions. The
1145 // instructions in this group may return out of order and/or may be
1146 // replayed (i.e. the same instruction issued more than once).
1147 //
1148 // In order to handle these situations correctly we need to make sure that
1149 // when a clause has more than one instruction, no instruction in the clause
1150 // writes to a register that is read by another instruction in the clause
1151 // (including itself). If we encounter this situation, we need to break the
1152 // clause by inserting a non SMEM instruction.
1153
1154 for (MachineInstr *MI : EmittedInstrs) {
1155 // When we hit a non-SMEM instruction then we have passed the start of the
1156 // clause and we can stop.
1157 if (!MI)
1158 break;
1159
1160 if (IsSMRD ? breaksSMEMSoftClause(MI) : breaksVMEMSoftClause(MI))
1161 break;
1162
1163 addClauseInst(MI: *MI);
1164 }
1165
1166 if (ClauseDefs.none())
1167 return 0;
1168
1169 // We need to make sure not to put loads and stores in the same clause if they
1170 // use the same address. For now, just start a new clause whenever we see a
1171 // store.
1172 if (MEM->mayStore())
1173 return 1;
1174
1175 addClauseInst(MI: *MEM);
1176
1177 // If the set of defs and uses intersect then we cannot add this instruction
1178 // to the clause, so we have a hazard.
1179 return ClauseDefs.anyCommon(RHS: ClauseUses) ? 1 : 0;
1180}
1181
1182int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) const {
1183 int WaitStatesNeeded = 0;
1184
1185 WaitStatesNeeded = checkSoftClauseHazards(MEM: SMRD);
1186
1187 // This SMRD hazard only affects SI.
1188 if (!ST.hasSMRDReadVALUDefHazard())
1189 return WaitStatesNeeded;
1190
1191 // A read of an SGPR by SMRD instruction requires 4 wait states when the
1192 // SGPR was written by a VALU instruction.
1193 int SmrdSgprWaitStates = 4;
1194 auto IsHazardDefFn = [this](const MachineInstr &MI) {
1195 return TII.isVALU(MI, /*AllowLDSDMA=*/true);
1196 };
1197 auto IsBufferHazardDefFn = [this](const MachineInstr &MI) {
1198 return TII.isSALU(MI);
1199 };
1200
1201 bool IsBufferSMRD = TII.isBufferSMRD(MI: *SMRD);
1202
1203 for (const MachineOperand &Use : SMRD->uses()) {
1204 if (!Use.isReg())
1205 continue;
1206 int WaitStatesNeededForUse =
1207 SmrdSgprWaitStates - getWaitStatesSinceDef(Reg: Use.getReg(), IsHazardDef: IsHazardDefFn,
1208 Limit: SmrdSgprWaitStates);
1209 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
1210
1211 // This fixes what appears to be undocumented hardware behavior in SI where
1212 // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor
1213 // needs some number of nops in between. We don't know how many we need, but
1214 // let's use 4. This wasn't discovered before probably because the only
1215 // case when this happens is when we expand a 64-bit pointer into a full
1216 // descriptor and use s_buffer_load_dword instead of s_load_dword, which was
1217 // probably never encountered in the closed-source land.
1218 if (IsBufferSMRD) {
1219 int WaitStatesNeededForUse =
1220 SmrdSgprWaitStates - getWaitStatesSinceDef(Reg: Use.getReg(),
1221 IsHazardDef: IsBufferHazardDefFn,
1222 Limit: SmrdSgprWaitStates);
1223 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
1224 }
1225 }
1226
1227 return WaitStatesNeeded;
1228}
1229
1230int GCNHazardRecognizer::checkVMEMHazards(MachineInstr *VMEM) const {
1231 if (!ST.hasVMEMReadSGPRVALUDefHazard())
1232 return 0;
1233
1234 int WaitStatesNeeded = checkSoftClauseHazards(MEM: VMEM);
1235
1236 // A read of an SGPR by a VMEM instruction requires 5 wait states when the
1237 // SGPR was written by a VALU Instruction.
1238 const int VmemSgprWaitStates = 5;
1239 auto IsHazardDefFn = [this](const MachineInstr &MI) {
1240 return TII.isVALU(MI, /*AllowLDSDMA=*/true);
1241 };
1242 for (const MachineOperand &Use : VMEM->uses()) {
1243 if (!Use.isReg() || TRI.isVectorRegister(MRI: MF.getRegInfo(), Reg: Use.getReg()))
1244 continue;
1245
1246 int WaitStatesNeededForUse =
1247 VmemSgprWaitStates - getWaitStatesSinceDef(Reg: Use.getReg(), IsHazardDef: IsHazardDefFn,
1248 Limit: VmemSgprWaitStates);
1249 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
1250 }
1251 return WaitStatesNeeded;
1252}
1253
1254int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) const {
1255 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1256 const SIInstrInfo *TII = ST.getInstrInfo();
1257
1258 // Check for DPP VGPR read after VALU VGPR write and EXEC write.
1259 int DppVgprWaitStates = 2;
1260 int DppExecWaitStates = 5;
1261 int WaitStatesNeeded = 0;
1262 auto IsHazardDefFn = [TII](const MachineInstr &MI) {
1263 return TII->isVALU(MI, /*AllowLDSDMA=*/true);
1264 };
1265
1266 for (const MachineOperand &Use : DPP->uses()) {
1267 if (!Use.isReg() || !TRI->isVGPR(MRI: MF.getRegInfo(), Reg: Use.getReg()))
1268 continue;
1269 int WaitStatesNeededForUse =
1270 DppVgprWaitStates - getWaitStatesSinceDef(
1271 Reg: Use.getReg(),
1272 IsHazardDef: [](const MachineInstr &) { return true; },
1273 Limit: DppVgprWaitStates);
1274 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
1275 }
1276
1277 WaitStatesNeeded = std::max(
1278 a: WaitStatesNeeded,
1279 b: DppExecWaitStates - getWaitStatesSinceDef(Reg: AMDGPU::EXEC, IsHazardDef: IsHazardDefFn,
1280 Limit: DppExecWaitStates));
1281
1282 return WaitStatesNeeded;
1283}
1284
1285int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) const {
1286 const SIInstrInfo *TII = ST.getInstrInfo();
1287
1288 // v_div_fmas requires 4 wait states after a write to vcc from a VALU
1289 // instruction.
1290 const int DivFMasWaitStates = 4;
1291 auto IsHazardDefFn = [TII](const MachineInstr &MI) {
1292 return TII->isVALU(MI, /*AllowLDSDMA=*/true);
1293 };
1294 int WaitStatesNeeded = getWaitStatesSinceDef(Reg: AMDGPU::VCC, IsHazardDef: IsHazardDefFn,
1295 Limit: DivFMasWaitStates);
1296
1297 return DivFMasWaitStates - WaitStatesNeeded;
1298}
1299
1300int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) const {
1301 const SIInstrInfo *TII = ST.getInstrInfo();
1302 unsigned GetRegHWReg = getHWReg(TII, RegInstr: *GetRegInstr);
1303
1304 const int GetRegWaitStates = 2;
1305 auto IsHazardFn = [TII, GetRegHWReg](const MachineInstr &MI) {
1306 return GetRegHWReg == getHWReg(TII, RegInstr: MI);
1307 };
1308 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazard: IsHazardFn, Limit: GetRegWaitStates);
1309
1310 return GetRegWaitStates - WaitStatesNeeded;
1311}
1312
1313int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) const {
1314 const SIInstrInfo *TII = ST.getInstrInfo();
1315 unsigned HWReg = getHWReg(TII, RegInstr: *SetRegInstr);
1316
1317 const int SetRegWaitStates = ST.getSetRegWaitStates();
1318 auto IsHazardFn = [TII, HWReg](const MachineInstr &MI) {
1319 return HWReg == getHWReg(TII, RegInstr: MI);
1320 };
1321 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazard: IsHazardFn, Limit: SetRegWaitStates);
1322 return SetRegWaitStates - WaitStatesNeeded;
1323}
1324
1325int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) const {
1326 if (!MI.mayStore())
1327 return -1;
1328
1329 const SIInstrInfo *TII = ST.getInstrInfo();
1330 unsigned Opcode = MI.getOpcode();
1331 const MCInstrDesc &Desc = MI.getDesc();
1332
1333 int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::vdata);
1334 int VDataRCID = -1;
1335 if (VDataIdx != -1)
1336 VDataRCID = TII->getOpRegClassID(OpInfo: Desc.operands()[VDataIdx]);
1337
1338 if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) {
1339 // There is no hazard if the instruction does not use vector regs
1340 // (like wbinvl1)
1341 if (VDataIdx == -1)
1342 return -1;
1343 if (AMDGPU::getRegBitWidth(RCID: VDataRCID) > 64) {
1344 // When SOFFSET-dependent wide-store windows apply, the BUFFER_STORE
1345 // source-vgpr WAR hazard exists for every SOFFSET shape; the wait-state
1346 // count differs by SOFFSET and is computed in checkVALUHazardsHelper.
1347 // Otherwise the hazard only exists if soffset is not an SGPR.
1348 if (ST.hasVDecCoExecHazard())
1349 return VDataIdx;
1350 const MachineOperand *SOffset =
1351 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
1352 if (!SOffset || !SOffset->isReg())
1353 return VDataIdx;
1354 }
1355 }
1356
1357 // MIMG instructions create a hazard if they don't use a 256-bit T# and
1358 // the store size is greater than 8 bytes and they have more than two bits
1359 // of their dmask set.
1360 // All our MIMG definitions use a 256-bit T#, so we can skip checking for them.
1361 if (TII->isMIMG(MI)) {
1362 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::srsrc);
1363 assert(SRsrcIdx != -1 && AMDGPU::getRegBitWidth(TII->getOpRegClassID(
1364 Desc.operands()[SRsrcIdx])) == 256);
1365 (void)SRsrcIdx;
1366 }
1367
1368 if (TII->isFLAT(MI)) {
1369 // There is no hazard if the instruction does not use vector regs
1370 if (VDataIdx == -1)
1371 return -1;
1372
1373 if (AMDGPU::getRegBitWidth(RCID: VDataRCID) > 64)
1374 return VDataIdx;
1375 }
1376
1377 return -1;
1378}
1379
1380int GCNHazardRecognizer::checkUniformWindowVALUHazardsHelper(
1381 Register Reg) const {
1382 // Wide stores need a single wait-state bubble before a VALU that overwrites
1383 // store data. createsVALUHazard already excludes MUBUF/MTBUF stores with an
1384 // SGPR SOFFSET.
1385 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1386
1387 auto IsHazard = [&](const MachineInstr &MI) {
1388 int DataIdx = createsVALUHazard(MI);
1389 return DataIdx >= 0 &&
1390 TRI->regsOverlap(RegA: MI.getOperand(i: DataIdx).getReg(), RegB: Reg);
1391 };
1392
1393 return std::max(a: 0, b: 1 - getWaitStatesSince(IsHazard, /*Limit=*/1));
1394}
1395
1396int GCNHazardRecognizer::checkSOFFSETWindowVALUHazardsHelper(
1397 Register Reg) const {
1398 // The required wait-state window depends on the producer's SOFFSET shape:
1399 // - MUBUF/MTBUF wide store with sgpr SOFFSET: 1 wait state.
1400 // - MUBUF/MTBUF wide store with literal/absent SOFFSET, and FLAT wide
1401 // store: 2 wait states.
1402 // The 1-cycle sgpr-SOFFSET window was measured on gfx950.
1403 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1404 const SIInstrInfo *TII = ST.getInstrInfo();
1405
1406 int WaitStatesNeeded = 0;
1407
1408 // Scan each wait-state window separately and take the max padding needed.
1409 // getWaitStatesSince supplies the minimum distance to a producer over paths.
1410 for (int Window = 1; Window <= 2; ++Window) {
1411 auto IsHazard = [&](const MachineInstr &MI) {
1412 int DataIdx = createsVALUHazard(MI);
1413 if (DataIdx < 0 ||
1414 !TRI->regsOverlap(RegA: MI.getOperand(i: DataIdx).getReg(), RegB: Reg))
1415 return false;
1416
1417 // Window 1 matches every hazard producer. Window 2 excludes BUF stores
1418 // with an SGPR SOFFSET, which only require a single wait state.
1419 if (Window == 1 || !TII->isBUF(MI))
1420 return true;
1421
1422 const MachineOperand *SOffset =
1423 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
1424 return !SOffset || !SOffset->isReg();
1425 };
1426 WaitStatesNeeded = std::max(a: WaitStatesNeeded,
1427 b: Window - getWaitStatesSince(IsHazard, Limit: Window));
1428 }
1429
1430 return WaitStatesNeeded;
1431}
1432
1433int GCNHazardRecognizer::checkVALUHazardsHelper(
1434 const MachineOperand &Def, const MachineRegisterInfo &MRI) const {
1435 // Helper to check for the hazard where VMEM instructions that store more
1436 // than 8 bytes can have their store data overwritten by the next
1437 // instruction.
1438 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1439
1440 if (!TRI->isVectorRegister(MRI, Reg: Def.getReg()))
1441 return 0;
1442
1443 if (ST.hasVDecCoExecHazard())
1444 return checkSOFFSETWindowVALUHazardsHelper(Reg: Def.getReg());
1445
1446 return checkUniformWindowVALUHazardsHelper(Reg: Def.getReg());
1447}
1448
1449/// Dest sel forwarding issue occurs if additional logic is needed to swizzle /
1450/// pack the computed value into correct bit position of the dest register. This
1451/// occurs if we have SDWA with dst_sel != DWORD or if we have op_sel with
1452/// dst_sel that is not aligned to the register. This function analayzes the \p
1453/// MI and \returns an operand with dst forwarding issue, or nullptr if
1454/// none exists.
1455static const MachineOperand *
1456getDstSelForwardingOperand(const MachineInstr &MI, const GCNSubtarget &ST) {
1457 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
1458 return nullptr;
1459
1460 const SIInstrInfo *TII = ST.getInstrInfo();
1461
1462 unsigned Opcode = MI.getOpcode();
1463
1464 // There are three different types of instructions
1465 // which produce forwarded dest: 1. SDWA with dst_sel != DWORD, 2. VOP3
1466 // which write hi bits (e.g. op_sel[3] == 1), and 3. FP8DstSelInst
1467 // (instructions with dest byte sel, e.g. CVT_SR_BF8_F32) and
1468 // op_sel[3:2]
1469 // != 0
1470 if (SIInstrInfo::isSDWA(MI)) {
1471 // Type 1: SDWA with dst_sel != DWORD
1472 if (auto *DstSel = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::dst_sel))
1473 if (DstSel->getImm() != AMDGPU::SDWA::DWORD)
1474 return TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1475 }
1476
1477 AMDGPU::FPType IsFP4OrFP8ConvOpc = AMDGPU::getFPDstSelType(Opc: Opcode);
1478 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::op_sel)) {
1479 // Type 2: VOP3 which write the hi bits
1480 if (TII->getNamedImmOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers) &
1481 SISrcMods::DST_OP_SEL)
1482 return TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1483
1484 // Type 3: FP8DstSelInst with op_sel[3:2] != 0)
1485 if (IsFP4OrFP8ConvOpc == AMDGPU::FPType::FP8 &&
1486 (TII->getNamedImmOperand(MI, OperandName: AMDGPU::OpName::src2_modifiers) &
1487 SISrcMods::OP_SEL_0))
1488 return TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1489 }
1490
1491 // Special case: nop is required for all the opsel values for fp4 sr variant
1492 // cvt scale instructions
1493 if (IsFP4OrFP8ConvOpc == AMDGPU::FPType::FP4)
1494 return TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1495
1496 return nullptr;
1497}
1498
1499/// Checks whether the provided \p MI "consumes" the operand with a Dest sel
1500/// fowarding issue \p Dst . We may "consume" the Dst via a standard explicit
1501/// RAW, or through irregular ways (e.g implicit RAW, certain types of WAW)
1502static bool consumesDstSelForwardingOperand(const MachineInstr *VALU,
1503 const MachineOperand *Dst,
1504 const SIRegisterInfo *TRI) {
1505 // We must consider implicit reads of the VALU. SDWA with dst_sel and
1506 // UNUSED_PRESERVE will implicitly read the result from forwarded dest,
1507 // and we must account for that hazard.
1508 // We also must account for WAW hazards. In particular, WAW with dest
1509 // preserve semantics (e.g. VOP3 with op_sel, VOP2 &&
1510 // !zeroesHigh16BitsOfDest) will read the forwarded dest for parity
1511 // check for ECC. Without accounting for this hazard, the ECC will be
1512 // wrong.
1513 // TODO: limit to RAW (including implicit reads) + problematic WAW (i.e.
1514 // complete zeroesHigh16BitsOfDest)
1515 for (auto &Operand : VALU->operands()) {
1516 if (Operand.isReg() && TRI->regsOverlap(RegA: Dst->getReg(), RegB: Operand.getReg())) {
1517 return true;
1518 }
1519 }
1520 return false;
1521}
1522
1523int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) const {
1524 int WaitStatesNeeded = 0;
1525
1526 if (ST.hasTransForwardingHazard() && !SIInstrInfo::isTRANS(MI: *VALU)) {
1527 const int TransDefWaitstates = 1;
1528
1529 auto IsTransDefFn = [this, VALU](const MachineInstr &MI) {
1530 if (!SIInstrInfo::isTRANS(MI))
1531 return false;
1532 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1533 const SIInstrInfo *TII = ST.getInstrInfo();
1534 Register Def = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst)->getReg();
1535
1536 for (const MachineOperand &Use : VALU->explicit_uses()) {
1537 if (Use.isReg() && TRI->regsOverlap(RegA: Def, RegB: Use.getReg()))
1538 return true;
1539 }
1540
1541 return false;
1542 };
1543
1544 int WaitStatesNeededForDef =
1545 TransDefWaitstates -
1546 getWaitStatesSince(IsHazard: IsTransDefFn, Limit: TransDefWaitstates);
1547 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1548 }
1549
1550 if (ST.hasDstSelForwardingHazard() || ST.hasCvtScaleForwardingHazard()) {
1551 const int Shift16DefWaitstates = 1;
1552
1553 auto IsShift16BitDefFn = [this, VALU](const MachineInstr &ProducerMI) {
1554 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1555 const MachineOperand *ForwardedDst =
1556 getDstSelForwardingOperand(MI: ProducerMI, ST);
1557 if (ForwardedDst) {
1558 return consumesDstSelForwardingOperand(VALU, Dst: ForwardedDst, TRI);
1559 }
1560
1561 if (ProducerMI.isInlineAsm()) {
1562 // Assume inline asm has dst forwarding hazard
1563 for (auto &Def : ProducerMI.all_defs()) {
1564 if (consumesDstSelForwardingOperand(VALU, Dst: &Def, TRI))
1565 return true;
1566 }
1567 }
1568
1569 return false;
1570 };
1571
1572 int WaitStatesNeededForDef =
1573 Shift16DefWaitstates -
1574 getWaitStatesSince(IsHazard: IsShift16BitDefFn, Limit: Shift16DefWaitstates);
1575 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1576 }
1577
1578 if (ST.hasVDecCoExecHazard()) {
1579 const int VALUWriteSGPRVALUReadWaitstates = 2;
1580 const int VALUWriteEXECRWLane = 4;
1581 const int VALUWriteVGPRReadlaneRead = 1;
1582
1583 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1584 const MachineRegisterInfo &MRI = MF.getRegInfo();
1585 Register UseReg;
1586 auto IsVALUDefSGPRFn = [&UseReg, TRI](const MachineInstr &MI) {
1587 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
1588 return false;
1589 return MI.modifiesRegister(Reg: UseReg, TRI);
1590 };
1591
1592 for (const MachineOperand &Use : VALU->explicit_uses()) {
1593 if (!Use.isReg())
1594 continue;
1595
1596 UseReg = Use.getReg();
1597 if (TRI->isSGPRReg(MRI, Reg: UseReg)) {
1598 int WaitStatesNeededForDef =
1599 VALUWriteSGPRVALUReadWaitstates -
1600 getWaitStatesSince(IsHazard: IsVALUDefSGPRFn,
1601 Limit: VALUWriteSGPRVALUReadWaitstates);
1602 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1603 }
1604 }
1605
1606 if (VALU->readsRegister(Reg: AMDGPU::VCC, TRI)) {
1607 UseReg = AMDGPU::VCC;
1608 int WaitStatesNeededForDef =
1609 VALUWriteSGPRVALUReadWaitstates -
1610 getWaitStatesSince(IsHazard: IsVALUDefSGPRFn, Limit: VALUWriteSGPRVALUReadWaitstates);
1611 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1612 }
1613
1614 switch (VALU->getOpcode()) {
1615 case AMDGPU::V_READLANE_B32:
1616 case AMDGPU::V_READFIRSTLANE_B32: {
1617 MachineOperand *Src = TII.getNamedOperand(MI&: *VALU, OperandName: AMDGPU::OpName::src0);
1618 UseReg = Src->getReg();
1619 int WaitStatesNeededForDef =
1620 VALUWriteVGPRReadlaneRead -
1621 getWaitStatesSince(IsHazard: IsVALUDefSGPRFn, Limit: VALUWriteVGPRReadlaneRead);
1622 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1623 }
1624 [[fallthrough]];
1625 case AMDGPU::V_WRITELANE_B32: {
1626 UseReg = AMDGPU::EXEC;
1627 int WaitStatesNeededForDef =
1628 VALUWriteEXECRWLane -
1629 getWaitStatesSince(IsHazard: IsVALUDefSGPRFn, Limit: VALUWriteEXECRWLane);
1630 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1631 break;
1632 }
1633 default:
1634 break;
1635 }
1636 }
1637
1638 // This checks for the hazard where VMEM instructions that store more than
1639 // 8 bytes can have there store data over written by the next instruction.
1640 if (!ST.has12DWordStoreHazard())
1641 return WaitStatesNeeded;
1642
1643 const MachineRegisterInfo &MRI = MF.getRegInfo();
1644
1645 for (const MachineOperand &Def : VALU->defs()) {
1646 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: checkVALUHazardsHelper(Def, MRI));
1647 }
1648
1649 return WaitStatesNeeded;
1650}
1651
1652int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) const {
1653 // This checks for hazards associated with inline asm statements.
1654 // Since inline asms can contain just about anything, we use this
1655 // to call/leverage other check*Hazard routines. Note that
1656 // this function doesn't attempt to address all possible inline asm
1657 // hazards (good luck), but is a collection of what has been
1658 // problematic thus far.
1659
1660 // see checkVALUHazards()
1661 if (!ST.has12DWordStoreHazard() && !ST.hasDstSelForwardingHazard() &&
1662 !ST.hasCvtScaleForwardingHazard())
1663 return 0;
1664
1665 const MachineRegisterInfo &MRI = MF.getRegInfo();
1666 int WaitStatesNeeded = 0;
1667
1668 for (const MachineOperand &Op :
1669 llvm::drop_begin(RangeOrContainer: IA->operands(), N: InlineAsm::MIOp_FirstOperand)) {
1670 if (Op.isReg() && Op.isDef()) {
1671 if (!TRI.isVectorRegister(MRI, Reg: Op.getReg()))
1672 continue;
1673
1674 if (ST.has12DWordStoreHazard()) {
1675 WaitStatesNeeded =
1676 std::max(a: WaitStatesNeeded, b: checkVALUHazardsHelper(Def: Op, MRI));
1677 }
1678 }
1679 }
1680
1681 if (ST.hasDstSelForwardingHazard()) {
1682 const int Shift16DefWaitstates = 1;
1683
1684 auto IsShift16BitDefFn = [this, &IA](const MachineInstr &ProducerMI) {
1685 const MachineOperand *Dst = getDstSelForwardingOperand(MI: ProducerMI, ST);
1686 // Assume inline asm reads the dst
1687 if (Dst)
1688 return IA->modifiesRegister(Reg: Dst->getReg(), TRI: &TRI) ||
1689 IA->readsRegister(Reg: Dst->getReg(), TRI: &TRI);
1690
1691 if (ProducerMI.isInlineAsm()) {
1692 // If MI is inline asm, assume it has dst forwarding hazard
1693 for (auto &Def : ProducerMI.all_defs()) {
1694 if (IA->modifiesRegister(Reg: Def.getReg(), TRI: &TRI) ||
1695 IA->readsRegister(Reg: Def.getReg(), TRI: &TRI)) {
1696 return true;
1697 }
1698 }
1699 }
1700
1701 return false;
1702 };
1703
1704 int WaitStatesNeededForDef =
1705 Shift16DefWaitstates -
1706 getWaitStatesSince(IsHazard: IsShift16BitDefFn, Limit: Shift16DefWaitstates);
1707 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForDef);
1708 }
1709
1710 return WaitStatesNeeded;
1711}
1712
1713int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) const {
1714 const SIInstrInfo *TII = ST.getInstrInfo();
1715 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1716 const MachineRegisterInfo &MRI = MF.getRegInfo();
1717
1718 const MachineOperand *LaneSelectOp =
1719 TII->getNamedOperand(MI&: *RWLane, OperandName: AMDGPU::OpName::src1);
1720
1721 if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, Reg: LaneSelectOp->getReg()))
1722 return 0;
1723
1724 Register LaneSelectReg = LaneSelectOp->getReg();
1725 auto IsHazardFn = [TII](const MachineInstr &MI) {
1726 return TII->isVALU(MI, /*AllowLDSDMA=*/true);
1727 };
1728
1729 const int RWLaneWaitStates = 4;
1730 int WaitStatesSince = getWaitStatesSinceDef(Reg: LaneSelectReg, IsHazardDef: IsHazardFn,
1731 Limit: RWLaneWaitStates);
1732 return RWLaneWaitStates - WaitStatesSince;
1733}
1734
1735int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) const {
1736 if (!ST.hasRFEHazards())
1737 return 0;
1738
1739 const SIInstrInfo *TII = ST.getInstrInfo();
1740
1741 const int RFEWaitStates = 1;
1742
1743 auto IsHazardFn = [TII](const MachineInstr &MI) {
1744 return getHWReg(TII, RegInstr: MI) == AMDGPU::Hwreg::ID_TRAPSTS;
1745 };
1746 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazard: IsHazardFn, Limit: RFEWaitStates);
1747 return RFEWaitStates - WaitStatesNeeded;
1748}
1749
1750int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) const {
1751 const SIInstrInfo *TII = ST.getInstrInfo();
1752 const int ReadM0WaitStates = 1;
1753 auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isSALU(MI); };
1754 return ReadM0WaitStates -
1755 getWaitStatesSinceDef(Reg: AMDGPU::M0, IsHazardDef: IsHazardFn, Limit: ReadM0WaitStates);
1756}
1757
1758void GCNHazardRecognizer::emitVNops(MachineBasicBlock &MBB,
1759 MachineBasicBlock::iterator InsertPt,
1760 int WaitStatesNeeded, bool IsHoisting) {
1761 const DebugLoc &DL = IsHoisting ? DebugLoc() : InsertPt->getDebugLoc();
1762 for (int I = 0; I < WaitStatesNeeded; ++I)
1763 BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_NOP_e32));
1764}
1765
1766void GCNHazardRecognizer::fixHazards(MachineInstr *MI) {
1767 fixVMEMtoScalarWriteHazards(MI);
1768 fixVcmpxPermlaneHazards(MI);
1769 fixSMEMtoVectorWriteHazards(MI);
1770 fixVcmpxExecWARHazard(MI);
1771 fixLdsBranchVmemWARHazard(MI);
1772 if (ST.hasLdsDirect()) {
1773 fixLdsDirectVALUHazard(MI);
1774 fixLdsDirectVMEMHazard(MI);
1775 }
1776 fixVALUPartialForwardingHazard(MI);
1777 fixVALUTransUseHazard(MI);
1778 fixVALUTransCoexecutionHazards(MI);
1779 fixWMMAHazards(MI); // fall-through if co-execution is enabled.
1780 fixWMMACoexecutionHazards(MI);
1781 fixShift64HighRegBug(MI);
1782 fixVALUMaskWriteHazard(MI);
1783 fixRequiredExportPriority(MI);
1784 if (ST.requiresWaitIdleBeforeGetReg())
1785 fixGetRegWaitIdle(MI);
1786 if (ST.hasDsAtomicAsyncBarrierArriveB64PipeBug())
1787 fixDsAtomicAsyncBarrierArriveB64(MI);
1788 if (ST.hasScratchBaseForwardingHazard())
1789 fixScratchBaseForwardingHazard(MI);
1790 if (ST.setRegModeNeedsVNOPs())
1791 fixSetRegMode(MI);
1792 if (ST.hasNeedsTDMDrain())
1793 fixTDM(MI);
1794}
1795
1796static bool isVCmpXWritesExec(const SIInstrInfo &TII, const SIRegisterInfo &TRI,
1797 const MachineInstr &MI) {
1798 return (TII.isVOPC(MI) ||
1799 (MI.isCompare() && (TII.isVOP3(MI) || TII.isSDWA(MI)))) &&
1800 MI.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &TRI);
1801}
1802
1803bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) {
1804 if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(MI: *MI))
1805 return false;
1806
1807 const SIInstrInfo *TII = ST.getInstrInfo();
1808 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1809 auto IsHazardFn = [TII, TRI](const MachineInstr &MI) {
1810 return isVCmpXWritesExec(TII: *TII, TRI: *TRI, MI);
1811 };
1812
1813 auto IsExpiredFn = [](const MachineInstr &MI, int) {
1814 unsigned Opc = MI.getOpcode();
1815 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
1816 Opc != AMDGPU::V_NOP_e32 && Opc != AMDGPU::V_NOP_e64 &&
1817 Opc != AMDGPU::V_NOP_sdwa;
1818 };
1819
1820 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
1821 std::numeric_limits<int>::max())
1822 return false;
1823
1824 // V_NOP will be discarded by SQ.
1825 // Use V_MOV_B32 v?, v?. Register must be alive so use src0 of V_PERMLANE*
1826 // which is always a VGPR and available.
1827 auto *Src0 = TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src0);
1828 Register Reg = Src0->getReg();
1829 bool IsUndef = Src0->isUndef();
1830 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
1831 MCID: TII->get(Opcode: AMDGPU::V_MOV_B32_e32))
1832 .addReg(RegNo: Reg, Flags: RegState::Define | getDeadRegState(B: IsUndef))
1833 .addReg(RegNo: Reg, Flags: IsUndef ? RegState::Undef : RegState::Kill);
1834
1835 return true;
1836}
1837
1838bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) {
1839 if (!ST.hasVMEMtoScalarWriteHazard())
1840 return false;
1841 assert(!ST.hasExtendedWaitCounts());
1842
1843 if (!SIInstrInfo::isSALU(MI: *MI) && !SIInstrInfo::isSMRD(MI: *MI))
1844 return false;
1845
1846 if (MI->getNumDefs() == 0)
1847 return false;
1848
1849 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1850
1851 auto IsHazardFn = [TRI, MI](const MachineInstr &I) {
1852 if (!SIInstrInfo::isVMEM(MI: I) && !SIInstrInfo::isDS(MI: I))
1853 return false;
1854
1855 for (const MachineOperand &Def : MI->defs()) {
1856 const MachineOperand *Op =
1857 I.findRegisterUseOperand(Reg: Def.getReg(), TRI, isKill: false);
1858 if (!Op)
1859 continue;
1860 return true;
1861 }
1862 return false;
1863 };
1864
1865 auto IsExpiredFn = [](const MachineInstr &MI, int) {
1866 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ||
1867 (MI.getOpcode() == AMDGPU::S_WAITCNT &&
1868 !MI.getOperand(i: 0).getImm()) ||
1869 (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
1870 AMDGPU::DepCtr::decodeFieldVmVsrc(Encoded: MI.getOperand(i: 0).getImm()) == 0);
1871 };
1872
1873 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
1874 std::numeric_limits<int>::max())
1875 return false;
1876
1877 const SIInstrInfo *TII = ST.getInstrInfo();
1878 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
1879 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
1880 .addImm(Val: AMDGPU::DepCtr::encodeFieldVmVsrc(VmVsrc: 0, STI: ST));
1881 return true;
1882}
1883
1884bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) {
1885 if (!ST.hasSMEMtoVectorWriteHazard())
1886 return false;
1887 assert(!ST.hasExtendedWaitCounts());
1888
1889 if (!SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true))
1890 return false;
1891
1892 AMDGPU::OpName SDSTName;
1893 switch (MI->getOpcode()) {
1894 case AMDGPU::V_READLANE_B32:
1895 case AMDGPU::V_READFIRSTLANE_B32:
1896 SDSTName = AMDGPU::OpName::vdst;
1897 break;
1898 default:
1899 SDSTName = AMDGPU::OpName::sdst;
1900 break;
1901 }
1902
1903 const SIInstrInfo *TII = ST.getInstrInfo();
1904 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1905 const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(GPU: ST.getCPU());
1906 const MachineOperand *SDST = TII->getNamedOperand(MI&: *MI, OperandName: SDSTName);
1907 if (!SDST) {
1908 for (const auto &MO : MI->implicit_operands()) {
1909 if (MO.isDef() && TRI->isSGPRClass(RC: TRI->getPhysRegBaseClass(Reg: MO.getReg()))) {
1910 SDST = &MO;
1911 break;
1912 }
1913 }
1914 }
1915
1916 if (!SDST)
1917 return false;
1918
1919 const Register SDSTReg = SDST->getReg();
1920 auto IsHazardFn = [SDSTReg, TRI](const MachineInstr &I) {
1921 return SIInstrInfo::isSMRD(MI: I) && I.readsRegister(Reg: SDSTReg, TRI);
1922 };
1923
1924 auto IsExpiredFn = [TII, IV](const MachineInstr &MI, int) {
1925 if (TII->isSALU(MI)) {
1926 switch (MI.getOpcode()) {
1927 case AMDGPU::S_SETVSKIP:
1928 case AMDGPU::S_VERSION:
1929 case AMDGPU::S_WAITCNT_VSCNT:
1930 case AMDGPU::S_WAITCNT_VMCNT:
1931 case AMDGPU::S_WAITCNT_EXPCNT:
1932 // These instructions cannot not mitigate the hazard.
1933 return false;
1934 case AMDGPU::S_WAITCNT_LGKMCNT:
1935 // Reducing lgkmcnt count to 0 always mitigates the hazard.
1936 return (MI.getOperand(i: 1).getImm() == 0) &&
1937 (MI.getOperand(i: 0).getReg() == AMDGPU::SGPR_NULL);
1938 case AMDGPU::S_WAITCNT: {
1939 const int64_t Imm = MI.getOperand(i: 0).getImm();
1940 AMDGPU::Waitcnt Decoded = AMDGPU::decodeWaitcnt(Version: IV, Encoded: Imm);
1941 // DsCnt corresponds to LGKMCnt here.
1942 return Decoded.get(T: AMDGPU::DS_CNT) == 0;
1943 }
1944 default:
1945 assert((!SIInstrInfo::isWaitcnt(MI.getOpcode()) ||
1946 MI.getOpcode() == AMDGPU::S_WAIT_IDLE) &&
1947 "unexpected wait count instruction");
1948 // SOPP instructions cannot mitigate the hazard.
1949 if (TII->isSOPP(MI))
1950 return false;
1951 // At this point the SALU can be assumed to mitigate the hazard
1952 // because either:
1953 // (a) it is independent of the at risk SMEM (breaking chain),
1954 // or
1955 // (b) it is dependent on the SMEM, in which case an appropriate
1956 // s_waitcnt lgkmcnt _must_ exist between it and the at risk
1957 // SMEM instruction.
1958 return true;
1959 }
1960 }
1961 return false;
1962 };
1963
1964 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
1965 std::numeric_limits<int>::max())
1966 return false;
1967
1968 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
1969 MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::SGPR_NULL)
1970 .addImm(Val: 0);
1971 return true;
1972}
1973
1974bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) {
1975 if (!ST.hasVcmpxExecWARHazard())
1976 return false;
1977 assert(!ST.hasExtendedWaitCounts());
1978
1979 if (!SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true))
1980 return false;
1981
1982 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1983 if (!MI->modifiesRegister(Reg: AMDGPU::EXEC, TRI))
1984 return false;
1985
1986 auto IsHazardFn = [TRI](const MachineInstr &I) {
1987 if (SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true))
1988 return false;
1989 return I.readsRegister(Reg: AMDGPU::EXEC, TRI);
1990 };
1991
1992 const SIInstrInfo *TII = ST.getInstrInfo();
1993 auto IsExpiredFn = [TII, TRI](const MachineInstr &MI, int) {
1994 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true)) {
1995 if (TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst))
1996 return true;
1997 for (auto MO : MI.implicit_operands())
1998 if (MO.isDef() && TRI->isSGPRClass(RC: TRI->getPhysRegBaseClass(Reg: MO.getReg())))
1999 return true;
2000 }
2001 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2002 AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: MI.getOperand(i: 0).getImm()) == 0)
2003 return true;
2004 return false;
2005 };
2006
2007 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2008 std::numeric_limits<int>::max())
2009 return false;
2010
2011 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2012 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
2013 .addImm(Val: AMDGPU::DepCtr::encodeFieldSaSdst(SaSdst: 0, STI: ST));
2014 return true;
2015}
2016
2017static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF,
2018 const GCNSubtarget &ST) {
2019 if (!ST.hasLdsBranchVmemWARHazard())
2020 return false;
2021
2022 // Check if the necessary condition for the hazard is met: both LDS and VMEM
2023 // instructions need to appear in the same function.
2024 bool HasLds = false;
2025 bool HasVmem = false;
2026 for (auto &MBB : MF) {
2027 for (auto &MI : MBB) {
2028 HasLds |= SIInstrInfo::isDS(MI) || SIInstrInfo::isLDSDMA(MI);
2029 HasVmem |= SIInstrInfo::isVMEM(MI);
2030 if (HasLds && HasVmem)
2031 return true;
2032 }
2033 }
2034 return false;
2035}
2036
2037static bool isStoreCountWaitZero(const MachineInstr &I) {
2038 return I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
2039 I.getOperand(i: 0).getReg() == AMDGPU::SGPR_NULL &&
2040 !I.getOperand(i: 1).getImm();
2041}
2042
2043bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) {
2044 if (!RunLdsBranchVmemWARHazardFixup)
2045 return false;
2046
2047 assert(ST.hasLdsBranchVmemWARHazard());
2048 assert(!ST.hasExtendedWaitCounts());
2049
2050 auto IsHazardInst = [](const MachineInstr &MI) {
2051 if (SIInstrInfo::isDS(MI) || SIInstrInfo::isLDSDMA(MI))
2052 return 1;
2053 if (SIInstrInfo::isVMEM(MI))
2054 return 2;
2055 return 0;
2056 };
2057
2058 auto InstType = IsHazardInst(*MI);
2059 if (!InstType)
2060 return false;
2061
2062 auto IsExpiredFn = [&IsHazardInst](const MachineInstr &I, int) {
2063 return IsHazardInst(I) || isStoreCountWaitZero(I);
2064 };
2065
2066 auto IsHazardFn = [InstType, &IsHazardInst](const MachineInstr &I) {
2067 if (!I.isBranch())
2068 return false;
2069
2070 auto IsHazardFn = [InstType, IsHazardInst](const MachineInstr &I) {
2071 auto InstType2 = IsHazardInst(I);
2072 return InstType2 && InstType != InstType2;
2073 };
2074
2075 auto IsExpiredFn = [InstType, &IsHazardInst](const MachineInstr &I, int) {
2076 auto InstType2 = IsHazardInst(I);
2077 if (InstType == InstType2)
2078 return true;
2079
2080 return isStoreCountWaitZero(I);
2081 };
2082
2083 return ::getWaitStatesSince(IsHazard: IsHazardFn, MI: &I, IsExpired: IsExpiredFn) !=
2084 std::numeric_limits<int>::max();
2085 };
2086
2087 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2088 std::numeric_limits<int>::max())
2089 return false;
2090
2091 const SIInstrInfo *TII = ST.getInstrInfo();
2092 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2093 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_VSCNT))
2094 .addReg(RegNo: AMDGPU::SGPR_NULL, Flags: RegState::Undef)
2095 .addImm(Val: 0);
2096
2097 return true;
2098}
2099
2100bool GCNHazardRecognizer::fixLdsDirectVALUHazard(MachineInstr *MI) {
2101 if (!SIInstrInfo::isLDSDIR(MI: *MI))
2102 return false;
2103
2104 const int NoHazardWaitStates = 15;
2105 const MachineOperand *VDST = TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::vdst);
2106 const Register VDSTReg = VDST->getReg();
2107
2108 bool VisitedTrans = false;
2109 auto IsHazardFn = [this, VDSTReg, &VisitedTrans](const MachineInstr &I) {
2110 if (!SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true))
2111 return false;
2112 VisitedTrans = VisitedTrans || SIInstrInfo::isTRANS(MI: I);
2113 // Cover both WAR and WAW
2114 return I.readsRegister(Reg: VDSTReg, TRI: &TRI) || I.modifiesRegister(Reg: VDSTReg, TRI: &TRI);
2115 };
2116 auto IsExpiredFn = [&](const MachineInstr &I, int WaitStates) {
2117 if (WaitStates >= NoHazardWaitStates)
2118 return true;
2119 // Instructions which cause va_vdst==0 expire hazard
2120 return SIInstrInfo::isVMEM(MI: I) || SIInstrInfo::isDS(MI: I) ||
2121 SIInstrInfo::isEXP(MI: I);
2122 };
2123 auto GetWaitStatesFn = [](const MachineInstr &MI) {
2124 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ? 1 : 0;
2125 };
2126
2127 DenseSet<const MachineBasicBlock *> Visited;
2128 auto Count = ::getWaitStatesSince(IsHazard: IsHazardFn, MBB: MI->getParent(),
2129 I: std::next(x: MI->getReverseIterator()), WaitStates: 0,
2130 IsExpired: IsExpiredFn, Visited, GetNumWaitStates: GetWaitStatesFn);
2131
2132 // Transcendentals can execute in parallel to other VALUs.
2133 // This makes va_vdst count unusable with a mixture of VALU and TRANS.
2134 if (VisitedTrans)
2135 Count = 0;
2136
2137 MachineOperand *WaitVdstOp =
2138 TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::waitvdst);
2139 WaitVdstOp->setImm(std::min(a: Count, b: NoHazardWaitStates));
2140
2141 return true;
2142}
2143
2144bool GCNHazardRecognizer::fixLdsDirectVMEMHazard(MachineInstr *MI) {
2145 if (!SIInstrInfo::isLDSDIR(MI: *MI))
2146 return false;
2147
2148 const MachineOperand *VDST = TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::vdst);
2149 const Register VDSTReg = VDST->getReg();
2150
2151 auto IsHazardFn = [this, VDSTReg](const MachineInstr &I) {
2152 if (!SIInstrInfo::isVMEM(MI: I) && !SIInstrInfo::isDS(MI: I))
2153 return false;
2154 return I.readsRegister(Reg: VDSTReg, TRI: &TRI) || I.modifiesRegister(Reg: VDSTReg, TRI: &TRI);
2155 };
2156 bool LdsdirCanWait = ST.hasLdsWaitVMSRC();
2157 // TODO: On GFX12 the hazard should expire on S_WAIT_LOADCNT/SAMPLECNT/BVHCNT
2158 // according to the type of VMEM instruction.
2159 auto IsExpiredFn = [this, LdsdirCanWait](const MachineInstr &I, int) {
2160 return SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true) ||
2161 SIInstrInfo::isEXP(MI: I) ||
2162 (I.getOpcode() == AMDGPU::S_WAITCNT && !I.getOperand(i: 0).getImm()) ||
2163 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2164 AMDGPU::DepCtr::decodeFieldVmVsrc(Encoded: I.getOperand(i: 0).getImm()) == 0) ||
2165 (LdsdirCanWait && SIInstrInfo::isLDSDIR(MI: I) &&
2166 !TII.getNamedOperand(MI: I, OperandName: AMDGPU::OpName::waitvsrc)->getImm());
2167 };
2168
2169 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2170 std::numeric_limits<int>::max())
2171 return false;
2172
2173 if (LdsdirCanWait) {
2174 TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::waitvsrc)->setImm(0);
2175 } else {
2176 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2177 MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
2178 .addImm(Val: AMDGPU::DepCtr::encodeFieldVmVsrc(VmVsrc: 0, STI: ST));
2179 }
2180
2181 return true;
2182}
2183
2184bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(MachineInstr *MI) {
2185 if (!ST.hasVALUPartialForwardingHazard())
2186 return false;
2187 assert(!ST.hasExtendedWaitCounts());
2188
2189 if (!ST.isWave64() || !SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true))
2190 return false;
2191
2192 SmallSetVector<Register, 4> SrcVGPRs;
2193
2194 for (const MachineOperand &Use : MI->explicit_uses()) {
2195 if (Use.isReg() && TRI.isVGPR(MRI: MF.getRegInfo(), Reg: Use.getReg()))
2196 SrcVGPRs.insert(X: Use.getReg());
2197 }
2198
2199 // Only applies with >= 2 unique VGPR sources
2200 if (SrcVGPRs.size() <= 1)
2201 return false;
2202
2203 // Look for the following pattern:
2204 // Va <- VALU [PreExecPos]
2205 // intv1
2206 // Exec <- SALU [ExecPos]
2207 // intv2
2208 // Vb <- VALU [PostExecPos]
2209 // intv3
2210 // MI Va, Vb (WaitState = 0)
2211 //
2212 // Where:
2213 // intv1 + intv2 <= 2 VALUs
2214 // intv3 <= 4 VALUs
2215 //
2216 // If found, insert an appropriate S_WAITCNT_DEPCTR before MI.
2217
2218 const int Intv1plus2MaxVALUs = 2;
2219 const int Intv3MaxVALUs = 4;
2220 const int IntvMaxVALUs = 6;
2221 const int NoHazardVALUWaitStates = IntvMaxVALUs + 2;
2222
2223 struct StateType {
2224 SmallDenseMap<Register, int, 4> DefPos;
2225 int ExecPos = std::numeric_limits<int>::max();
2226 int VALUs = 0;
2227
2228 static unsigned getHashValue(const StateType &State) {
2229 return hash_combine(args: State.ExecPos, args: State.VALUs,
2230 args: hash_combine_range(R: State.DefPos));
2231 }
2232 static bool isEqual(const StateType &LHS, const StateType &RHS) {
2233 return LHS.DefPos == RHS.DefPos && LHS.ExecPos == RHS.ExecPos &&
2234 LHS.VALUs == RHS.VALUs;
2235 }
2236 };
2237
2238 StateType State;
2239
2240 // This overloads expiry testing with all the hazard detection
2241 auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) {
2242 // Too many VALU states have passed
2243 if (State.VALUs > NoHazardVALUWaitStates)
2244 return HazardExpired;
2245
2246 // Instructions which cause va_vdst==0 expire hazard
2247 if (SIInstrInfo::isVMEM(MI: I) || SIInstrInfo::isDS(MI: I) ||
2248 SIInstrInfo::isEXP(MI: I) ||
2249 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2250 AMDGPU::DepCtr::decodeFieldVaVdst(Encoded: I.getOperand(i: 0).getImm()) == 0))
2251 return HazardExpired;
2252
2253 // Track registers writes
2254 bool Changed = false;
2255 if (SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true)) {
2256 for (Register Src : SrcVGPRs) {
2257 if (!State.DefPos.count(Val: Src) && I.modifiesRegister(Reg: Src, TRI: &TRI)) {
2258 State.DefPos[Src] = State.VALUs;
2259 Changed = true;
2260 }
2261 }
2262 } else if (SIInstrInfo::isSALU(MI: I)) {
2263 if (State.ExecPos == std::numeric_limits<int>::max()) {
2264 if (!State.DefPos.empty() && I.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &TRI)) {
2265 State.ExecPos = State.VALUs;
2266 Changed = true;
2267 }
2268 }
2269 }
2270
2271 // Early expiration: too many VALUs in intv3
2272 if (State.VALUs > Intv3MaxVALUs && State.DefPos.empty())
2273 return HazardExpired;
2274
2275 // Only evaluate state if something changed
2276 if (!Changed)
2277 return NoHazardFound;
2278
2279 // Determine positions of VALUs pre/post exec change
2280 if (State.ExecPos == std::numeric_limits<int>::max())
2281 return NoHazardFound;
2282
2283 int PreExecPos = std::numeric_limits<int>::max();
2284 int PostExecPos = std::numeric_limits<int>::max();
2285
2286 for (auto Entry : State.DefPos) {
2287 int DefVALUs = Entry.second;
2288 if (DefVALUs != std::numeric_limits<int>::max()) {
2289 if (DefVALUs >= State.ExecPos)
2290 PreExecPos = std::min(a: PreExecPos, b: DefVALUs);
2291 else
2292 PostExecPos = std::min(a: PostExecPos, b: DefVALUs);
2293 }
2294 }
2295
2296 // Need a VALUs post exec change
2297 if (PostExecPos == std::numeric_limits<int>::max())
2298 return NoHazardFound;
2299
2300 // Too many VALUs in intv3?
2301 int Intv3VALUs = PostExecPos;
2302 if (Intv3VALUs > Intv3MaxVALUs)
2303 return HazardExpired;
2304
2305 // Too many VALUs in intv2?
2306 int Intv2VALUs = (State.ExecPos - PostExecPos) - 1;
2307 if (Intv2VALUs > Intv1plus2MaxVALUs)
2308 return HazardExpired;
2309
2310 // Need a VALUs pre exec change
2311 if (PreExecPos == std::numeric_limits<int>::max())
2312 return NoHazardFound;
2313
2314 // Too many VALUs in intv1?
2315 int Intv1VALUs = PreExecPos - State.ExecPos;
2316 if (Intv1VALUs > Intv1plus2MaxVALUs)
2317 return HazardExpired;
2318
2319 // Too many VALUs in intv1 + intv2
2320 if (Intv1VALUs + Intv2VALUs > Intv1plus2MaxVALUs)
2321 return HazardExpired;
2322
2323 return HazardFound;
2324 };
2325 auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) {
2326 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
2327 State.VALUs += 1;
2328 };
2329
2330 if (!hasHazard<StateType>(InitialState: State, IsHazard: IsHazardFn, UpdateState: UpdateStateFn, InitialMBB: MI->getParent(),
2331 InitialI: std::next(x: MI->getReverseIterator())))
2332 return false;
2333
2334 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2335 MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
2336 .addImm(Val: AMDGPU::DepCtr::encodeFieldVaVdst(VaVdst: 0, STI: ST));
2337
2338 return true;
2339}
2340
2341bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) {
2342 if (!ST.hasVALUTransUseHazard())
2343 return false;
2344 assert(!ST.hasExtendedWaitCounts());
2345
2346 if (!SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true))
2347 return false;
2348
2349 SmallSet<Register, 4> SrcVGPRs;
2350
2351 for (const MachineOperand &Use : MI->explicit_uses()) {
2352 if (Use.isReg() && TRI.isVGPR(MRI: MF.getRegInfo(), Reg: Use.getReg()))
2353 SrcVGPRs.insert(V: Use.getReg());
2354 }
2355
2356 // Look for the following pattern:
2357 // Va <- TRANS VALU
2358 // intv
2359 // MI Va (WaitState = 0)
2360 //
2361 // Where:
2362 // intv <= 5 VALUs / 1 TRANS
2363 //
2364 // If found, insert an appropriate S_WAITCNT_DEPCTR before MI.
2365
2366 const int IntvMaxVALUs = 5;
2367 const int IntvMaxTRANS = 1;
2368
2369 struct StateType {
2370 int VALUs = 0;
2371 int TRANS = 0;
2372
2373 static unsigned getHashValue(const StateType &State) {
2374 return hash_combine(args: State.VALUs, args: State.TRANS);
2375 }
2376 static bool isEqual(const StateType &LHS, const StateType &RHS) {
2377 return LHS.VALUs == RHS.VALUs && LHS.TRANS == RHS.TRANS;
2378 }
2379 };
2380
2381 StateType State;
2382
2383 // This overloads expiry testing with all the hazard detection
2384 auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) {
2385 // Too many VALU states have passed
2386 if (State.VALUs > IntvMaxVALUs || State.TRANS > IntvMaxTRANS)
2387 return HazardExpired;
2388
2389 // Instructions which cause va_vdst==0 expire hazard
2390 if (SIInstrInfo::isVMEM(MI: I) || SIInstrInfo::isDS(MI: I) ||
2391 SIInstrInfo::isEXP(MI: I) ||
2392 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2393 AMDGPU::DepCtr::decodeFieldVaVdst(Encoded: I.getOperand(i: 0).getImm()) == 0))
2394 return HazardExpired;
2395
2396 // Track registers writes
2397 if (SIInstrInfo::isTRANS(MI: I)) {
2398 for (Register Src : SrcVGPRs) {
2399 if (I.modifiesRegister(Reg: Src, TRI: &TRI)) {
2400 return HazardFound;
2401 }
2402 }
2403 }
2404
2405 return NoHazardFound;
2406 };
2407 auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) {
2408 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
2409 State.VALUs += 1;
2410 if (SIInstrInfo::isTRANS(MI))
2411 State.TRANS += 1;
2412 };
2413
2414 if (!hasHazard<StateType>(InitialState: State, IsHazard: IsHazardFn, UpdateState: UpdateStateFn, InitialMBB: MI->getParent(),
2415 InitialI: std::next(x: MI->getReverseIterator())))
2416 return false;
2417
2418 // Hazard is observed - insert a wait on va_dst counter to ensure hazard is
2419 // avoided.
2420 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
2421 MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
2422 .addImm(Val: AMDGPU::DepCtr::encodeFieldVaVdst(VaVdst: 0, STI: ST));
2423
2424 return true;
2425}
2426
2427bool GCNHazardRecognizer::fixVALUTransCoexecutionHazards(MachineInstr *MI) {
2428 if (!ST.hasTransCoexecutionHazard() || // Coexecution disabled.
2429 !SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true) ||
2430 SIInstrInfo::isTRANS(MI: *MI))
2431 return false;
2432
2433 const SIInstrInfo *TII = ST.getInstrInfo();
2434 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2435
2436 auto IsTransHazardFn = [MI, TII, TRI](const MachineInstr &I) {
2437 if (!SIInstrInfo::isTRANS(MI: I))
2438 return false;
2439
2440 // RAW: Trans(I) writes, VALU(MI) reads.
2441 Register TransDef = TII->getNamedOperand(MI: I, OperandName: AMDGPU::OpName::vdst)->getReg();
2442 for (const MachineOperand &ValuUse : MI->explicit_uses()) {
2443 if (ValuUse.isReg() && TRI->regsOverlap(RegA: TransDef, RegB: ValuUse.getReg()))
2444 return true;
2445 }
2446
2447 auto *ValuDst = TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::vdst);
2448 if (!ValuDst || !ValuDst->isReg())
2449 return false;
2450
2451 // WAR: Trans(I) reads, VALU(MI) writes.
2452 Register ValuDef = ValuDst->getReg();
2453 for (const MachineOperand &TransUse : I.explicit_uses()) {
2454 if (TransUse.isReg() && TRI->regsOverlap(RegA: ValuDef, RegB: TransUse.getReg()))
2455 return true;
2456 }
2457
2458 return false;
2459 };
2460
2461 auto IsExpiredFn = [](const MachineInstr &I, int) {
2462 return SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true);
2463 };
2464
2465 const int HasVALU = std::numeric_limits<int>::max();
2466 if (::getWaitStatesSince(IsHazard: IsTransHazardFn, MI, IsExpired: IsExpiredFn) == HasVALU)
2467 return false;
2468
2469 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::V_NOP_e32));
2470 return true;
2471}
2472
2473bool GCNHazardRecognizer::fixWMMAHazards(MachineInstr *MI) {
2474 if (!SIInstrInfo::isWMMA(MI: *MI) && !SIInstrInfo::isSWMMAC(MI: *MI))
2475 return false;
2476
2477 const SIInstrInfo *TII = ST.getInstrInfo();
2478 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2479
2480 auto IsHazardFn = [MI, TII, TRI, this](const MachineInstr &I) {
2481 if (!SIInstrInfo::isWMMA(MI: I) && !SIInstrInfo::isSWMMAC(MI: I))
2482 return false;
2483
2484 // Src0(matrix A) or Src1(matrix B) of the current wmma instruction overlaps
2485 // with the dest(matrix D) of the previous wmma.
2486 const Register CurSrc0Reg =
2487 TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src0)->getReg();
2488 const Register CurSrc1Reg =
2489 TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src1)->getReg();
2490
2491 const Register PrevDstReg =
2492 TII->getNamedOperand(MI: I, OperandName: AMDGPU::OpName::vdst)->getReg();
2493
2494 if (TRI->regsOverlap(RegA: PrevDstReg, RegB: CurSrc0Reg) ||
2495 TRI->regsOverlap(RegA: PrevDstReg, RegB: CurSrc1Reg)) {
2496 return true;
2497 }
2498
2499 // GFX12+ allows overlap of matrix C with PrevDstReg (hardware will stall)
2500 // but Index can't overlap with PrevDstReg.
2501 if (AMDGPU::isGFX12Plus(STI: ST)) {
2502 if (SIInstrInfo::isSWMMAC(MI: *MI)) {
2503 const Register CurIndex =
2504 TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src2)->getReg();
2505 if (TRI->regsOverlap(RegA: PrevDstReg, RegB: CurIndex))
2506 return true;
2507 }
2508 return false;
2509 }
2510
2511 return false;
2512 };
2513
2514 auto IsExpiredFn = [](const MachineInstr &I, int) {
2515 return SIInstrInfo::isVALU(MI: I, /*AllowLDSDMA=*/true);
2516 };
2517
2518 if (::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn) ==
2519 std::numeric_limits<int>::max())
2520 return false;
2521
2522 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::V_NOP_e32));
2523
2524 return true;
2525}
2526
2527static bool isCoexecutableVALUInst(const MachineInstr &MI) {
2528 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
2529 !SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI) &&
2530 !SIInstrInfo::isLDSDMA(MI);
2531}
2532
2533// Classify XDL WMMA instructions into co-execution hazard categories
2534// (Refer to SPG 4.6.12.1), mainly based on instruction latency.
2535//
2536// Category 0: WMMA with Latency 8
2537// WMMA_*F16, WMMA_*BF16
2538// WMMA_*_16X16X128_{FP8,BF8}
2539// WMMA_*F8F6F4 if SRCA & SRCB are not both F4
2540//
2541// Category 1: WMMA Latency 16
2542// WMMA_IU8
2543//
2544// Category 2: SWMMAC with Latency 8
2545// SWMMAC_*F16, SWMMAC_*BF16,
2546// SWMMAC_*FP8FP8
2547// SWMMAC_*BF8FP8
2548// SWMMAC_*FP8BF8
2549// SWMMAC_*BF8BF8
2550//
2551// Category 3: SWMMAC with Latency 16
2552// SWMMAC_IU8
2553//
2554// Category 4: 16 Pass GFX1251 WMMA with latency 16
2555// V_WMMA_*_16X16X32_{F16,BF16}
2556// V_WMMA_{F32,F16}_16X16X64_{FP8,BF8}*
2557// V_WMMA_F32_16x16x128_F8F6F4 (F4 only)
2558// V_SWMMAC_*_16X16X64_{F16,BF16}
2559// V_SWMMAC_{F32,F16}_16X16X128_{FP8,BF8}*
2560//
2561// Category 5: 32 Pass GFX1251 WMMA with latency 32
2562// V_WMMA_F32_16x16x128_F8F6F4 (not all F4)
2563// V_WMMA_{F32,F16}_16X16X128_{FP8,BF8}*
2564// V_WMMA_F32_32X16X128_F4
2565// V_WMMA_I32_16X16X64_IU8
2566// V_WMMA_I32_16X16X64_IU8
2567//
2568// Category 6: gfx1250 WMMA with Latency 4 (one co-execution slot)
2569// WMMA_*_16X16X64_{FP8,BF8}
2570// WMMA_*F8F6F4 if SRCA & SRCB are both F4
2571static unsigned getWMMAHazardInstInCategory(const MachineInstr &MI,
2572 const SIInstrInfo *TII,
2573 const TargetSchedModel &SchedModel,
2574 const GCNSubtarget &ST) {
2575 assert(TII->isXDLWMMA(MI) && "must be xdl wmma");
2576 bool IsSWMMAC = SIInstrInfo::isSWMMAC(MI);
2577 bool IsLowestRateWMMA = ST.hasGFX125xLowestRateWMMA();
2578 unsigned Category = 0;
2579
2580 unsigned Latency = SchedModel.computeInstrLatency(MI: &MI);
2581 switch (Latency) {
2582 case 4:
2583 // Dense 4-cycle WMMA (gfx1250 16x16x64 FP8/BF8 and f8f6f4 with both
2584 // inputs F4). One co-execution slot; there is no 4-cycle SWMMAC.
2585 assert(!IsSWMMAC && "no 4-cycle SWMMAC expected");
2586 Category = 6;
2587 break;
2588 case 8:
2589 Category = IsSWMMAC ? 2 : 0;
2590 break;
2591 case 16:
2592 Category = IsLowestRateWMMA ? 4 : (IsSWMMAC ? 3 : 1);
2593 break;
2594 case 32:
2595 assert(IsLowestRateWMMA && "latency 32 is not expected");
2596 Category = 5;
2597 break;
2598 default:
2599 llvm_unreachable("unexpected xdl wmma latency");
2600 } // end switch.
2601
2602 return Category;
2603}
2604
2605int GCNHazardRecognizer::checkWMMACoexecutionHazards(MachineInstr *MI) const {
2606 if (!ST.hasWMMACoexecutionHazards())
2607 return 0;
2608
2609 const SIInstrInfo *TII = ST.getInstrInfo();
2610 if (!TII->isXDLWMMA(MI: *MI) && !isCoexecutableVALUInst(MI: *MI))
2611 return 0;
2612
2613 // WaitStates here is the number of V_NOPs or unrelated VALU instructions must
2614 // be in between the first WMMA and the second instruction to cover the hazard
2615 // (WMMAWaitStates if the second is also a WMMA, VALUWaitStates if the second
2616 // is a VALU). Refer to SPG 4.6.12.1. "Requirements for WMMA data hazards" for
2617 // numbers, which depends on the category of the first WMMA.
2618 const int WMMAWaitStates[] = {5, 9, 3, 5, 9, 17, 2};
2619 const int VALUWaitStates[] = {4, 8, 2, 4, 8, 16, 1};
2620 unsigned Category = 0;
2621
2622 auto IsWMMAHazardFn = [MI, TII, &Category, this](const MachineInstr &I) {
2623 if (!TII->isXDLWMMA(MI: I))
2624 return false;
2625
2626 Category = getWMMAHazardInstInCategory(MI: I, TII, SchedModel: TSchedModel, ST);
2627 return hasWMMAToWMMARegOverlap(WMMA: I, MI: *MI);
2628 };
2629
2630 auto IsVALUHazardFn = [MI, TII, &Category, this](const MachineInstr &I) {
2631 if (!TII->isXDLWMMA(MI: I))
2632 return false;
2633
2634 Category = getWMMAHazardInstInCategory(MI: I, TII, SchedModel: TSchedModel, ST);
2635 return hasWMMAToVALURegOverlap(WMMA: I, MI: *MI);
2636 };
2637
2638 int WaitStatesNeeded = -1;
2639 int ExistingVALUs = 0; // Existing number of VALU ops in between.
2640 bool IsLowestRateWMMA = ST.hasGFX125xLowestRateWMMA();
2641
2642 // getWaitStatesSinceVALU checks for a hazard between instruction 'I' and
2643 // 'MI':
2644 // - If a hazard exists: returns the number of VALUs in between and sets
2645 // 'Category' via IsWMMAHazardFn/IsVALUHazardFn for instruction 'I'.
2646 // - If no hazard exists: returns INT_MAX, making WaitStatesNeeded negative,
2647 // so no V_NOP insertion is needed.
2648 if (TII->isXDLWMMA(MI: *MI)) {
2649 // Maximum of MMAWaitStates.
2650 const int WMMAWaitsLimit = IsLowestRateWMMA ? 17 : 9;
2651 ExistingVALUs = getWaitStatesSinceVALU(IsHazard: IsWMMAHazardFn, Limit: WMMAWaitsLimit);
2652 WaitStatesNeeded = WMMAWaitStates[Category] - ExistingVALUs;
2653 } else { // Must be a co-executable VALU.
2654 // Maximum of VALUWaitStates.
2655 const int VALUWaitsLimit = IsLowestRateWMMA ? 16 : 8;
2656 ExistingVALUs = getWaitStatesSinceVALU(IsHazard: IsVALUHazardFn, Limit: VALUWaitsLimit);
2657 WaitStatesNeeded = VALUWaitStates[Category] - ExistingVALUs;
2658 }
2659
2660 return WaitStatesNeeded;
2661}
2662
2663bool GCNHazardRecognizer::hasWMMAToWMMARegOverlap(
2664 const MachineInstr &WMMA, const MachineInstr &MI) const {
2665 Register D0 = TII.getNamedOperand(MI: WMMA, OperandName: AMDGPU::OpName::vdst)->getReg();
2666 Register A1 = TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src0)->getReg();
2667 Register B1 = TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src1)->getReg();
2668
2669 // WMMA0 writes (D0), WMMA1 reads (A1/B1/Idx1).
2670 if (TRI.regsOverlap(RegA: D0, RegB: A1) || TRI.regsOverlap(RegA: D0, RegB: B1))
2671 return true;
2672
2673 if (SIInstrInfo::isSWMMAC(MI)) {
2674 Register Idx1 = TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src2)->getReg();
2675 if (TRI.regsOverlap(RegA: D0, RegB: Idx1))
2676 return true;
2677 }
2678 return false;
2679}
2680
2681bool GCNHazardRecognizer::hasWMMAToVALURegOverlap(
2682 const MachineInstr &WMMA, const MachineInstr &MI) const {
2683 // WMMA writes, VALU reads.
2684 Register D0 = TII.getNamedOperand(MI: WMMA, OperandName: AMDGPU::OpName::vdst)->getReg();
2685 for (const MachineOperand &ValuUse : MI.explicit_uses()) {
2686 if (ValuUse.isReg() && TRI.regsOverlap(RegA: D0, RegB: ValuUse.getReg()))
2687 return true;
2688 }
2689
2690 // WMMA reads or writes, VALU writes.
2691 Register A0 = TII.getNamedOperand(MI: WMMA, OperandName: AMDGPU::OpName::src0)->getReg();
2692 Register B0 = TII.getNamedOperand(MI: WMMA, OperandName: AMDGPU::OpName::src1)->getReg();
2693 SmallVector<Register, 4> WMMARegs({D0, A0, B0});
2694
2695 if (SIInstrInfo::isSWMMAC(MI: WMMA)) {
2696 Register Idx0 = TII.getNamedOperand(MI: WMMA, OperandName: AMDGPU::OpName::src2)->getReg();
2697 WMMARegs.push_back(Elt: Idx0);
2698 }
2699
2700 for (const MachineOperand &ValuDef : MI.defs()) {
2701 Register VDstReg = ValuDef.getReg();
2702 for (Register WMMAReg : WMMARegs) {
2703 if (TRI.regsOverlap(RegA: VDstReg, RegB: WMMAReg))
2704 return true;
2705 }
2706 }
2707 return false;
2708}
2709
2710bool GCNHazardRecognizer::isCoexecutionHazardFor(const MachineInstr &I,
2711 const MachineInstr &MI) const {
2712 // I is the potential WMMA hazard source, MI is the instruction being checked
2713 // for hazard.
2714 if (!TII.isXDLWMMA(MI: I))
2715 return false;
2716
2717 // Dispatch based on MI type
2718 if (TII.isXDLWMMA(MI))
2719 return hasWMMAToWMMARegOverlap(WMMA: I, MI);
2720 if (isCoexecutableVALUInst(MI))
2721 return hasWMMAToVALURegOverlap(WMMA: I, MI);
2722
2723 return false;
2724}
2725
2726bool GCNHazardRecognizer::hasWMMAHazardInLoop(MachineLoop *L, MachineInstr *MI,
2727 bool IncludeSubloops) {
2728 // Scan loop for any WMMA that hazards MI.
2729 // TODO: Avoid full loop scan when WMMA is beyond VALU distance.
2730 for (MachineBasicBlock *MBB : L->getBlocks()) {
2731 if (!IncludeSubloops && MLI->getLoopFor(BB: MBB) != L)
2732 continue;
2733 for (MachineInstr &I : *MBB) {
2734 if (&I == MI)
2735 continue;
2736 if (isCoexecutionHazardFor(I, MI: *MI))
2737 return true;
2738 }
2739 }
2740 return false;
2741}
2742
2743bool GCNHazardRecognizer::tryHoistWMMAVnopsFromLoop(MachineInstr *MI,
2744 int WaitStatesNeeded) {
2745 if (!MLI)
2746 return false;
2747
2748 MachineLoop *L = MLI->getLoopFor(BB: MI->getParent());
2749 if (!L) {
2750 ++NumWMMAHoistingBailed;
2751 return false;
2752 }
2753
2754 // If innermost loop has WMMA hazard, we can't hoist at all
2755 if (hasWMMAHazardInLoop(L, MI)) {
2756 ++NumWMMAHoistingBailed;
2757 return false;
2758 }
2759
2760 // Find outermost loop with no internal hazard
2761 MachineLoop *TargetLoop = L;
2762 while (MachineLoop *Parent = TargetLoop->getParentLoop()) {
2763 if (hasWMMAHazardInLoop(L: Parent, MI, IncludeSubloops: false))
2764 break; // Parent has hazard in its own blocks, stop here
2765 TargetLoop = Parent; // Safe to hoist further out
2766 }
2767
2768 // Need valid preheader to insert V_NOPs
2769 MachineBasicBlock *Preheader = TargetLoop->getLoopPreheader();
2770 if (!Preheader) {
2771 ++NumWMMAHoistingBailed;
2772 return false;
2773 }
2774
2775 LLVM_DEBUG(dbgs() << "WMMA V_NOP Hoisting: Moving " << WaitStatesNeeded
2776 << " V_NOPs from loop to " << printMBBReference(*Preheader)
2777 << "\n");
2778
2779 emitVNops(MBB&: *Preheader, InsertPt: Preheader->getFirstTerminator(), WaitStatesNeeded,
2780 /*IsHoisting=*/true);
2781 NumWMMANopsHoisted += WaitStatesNeeded;
2782 return true;
2783}
2784
2785bool GCNHazardRecognizer::fixWMMACoexecutionHazards(MachineInstr *MI) {
2786 int WaitStatesNeeded = checkWMMACoexecutionHazards(MI);
2787 if (WaitStatesNeeded <= 0)
2788 return false;
2789
2790 if (EnableWMMAVnopHoisting && tryHoistWMMAVnopsFromLoop(MI, WaitStatesNeeded))
2791 return true;
2792
2793 emitVNops(MBB&: *MI->getParent(), InsertPt: MI->getIterator(), WaitStatesNeeded);
2794 return true;
2795}
2796
2797bool GCNHazardRecognizer::fixShift64HighRegBug(MachineInstr *MI) {
2798 if (!ST.hasShift64HighRegBug())
2799 return false;
2800 assert(!ST.hasExtendedWaitCounts());
2801
2802 switch (MI->getOpcode()) {
2803 default:
2804 return false;
2805 case AMDGPU::V_LSHLREV_B64_e64:
2806 case AMDGPU::V_LSHRREV_B64_e64:
2807 case AMDGPU::V_ASHRREV_I64_e64:
2808 break;
2809 }
2810
2811 MachineOperand *Amt = TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src0);
2812 if (!Amt->isReg())
2813 return false;
2814
2815 Register AmtReg = Amt->getReg();
2816 const MachineRegisterInfo &MRI = MF.getRegInfo();
2817 // Check if this is a last VGPR in the allocation block.
2818 if (!TRI.isVGPR(MRI, Reg: AmtReg) || ((AmtReg - AMDGPU::VGPR0) & 7) != 7)
2819 return false;
2820
2821 if (AmtReg != AMDGPU::VGPR255 && MRI.isPhysRegUsed(PhysReg: AmtReg + 1))
2822 return false;
2823
2824 assert(ST.needsAlignedVGPRs());
2825 static_assert(AMDGPU::VGPR0 + 1 == AMDGPU::VGPR1);
2826
2827 const DebugLoc &DL = MI->getDebugLoc();
2828 MachineBasicBlock *MBB = MI->getParent();
2829 MachineOperand *Src1 = TII.getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src1);
2830
2831 // In:
2832 //
2833 // Dst = shiftrev64 Amt, Src1
2834 //
2835 // if Dst!=Src1 then avoid the bug with:
2836 //
2837 // Dst.sub0 = Amt
2838 // Dst = shift64 Dst.sub0, Src1
2839
2840 Register DstReg = MI->getOperand(i: 0).getReg();
2841 if (!Src1->isReg() || Src1->getReg() != DstReg) {
2842 Register DstLo = TRI.getSubReg(Reg: DstReg, Idx: AMDGPU::sub0);
2843 runOnInstruction(
2844 MI: BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstLo).add(MO: *Amt));
2845 Amt->setReg(DstLo);
2846 Amt->setIsKill(true);
2847 return true;
2848 }
2849
2850 bool Overlapped = MI->modifiesRegister(Reg: AmtReg, TRI: &TRI);
2851 Register NewReg;
2852 for (MCRegister Reg : Overlapped ? AMDGPU::VReg_64_Align2RegClass
2853 : AMDGPU::VGPR_32RegClass) {
2854 if (!MI->modifiesRegister(Reg, TRI: &TRI) && !MI->readsRegister(Reg, TRI: &TRI)) {
2855 NewReg = Reg;
2856 break;
2857 }
2858 }
2859
2860 Register NewAmt = Overlapped ? (Register)TRI.getSubReg(Reg: NewReg, Idx: AMDGPU::sub1)
2861 : NewReg;
2862 Register NewAmtLo;
2863
2864 if (Overlapped)
2865 NewAmtLo = TRI.getSubReg(Reg: NewReg, Idx: AMDGPU::sub0);
2866
2867 // Insert a full wait count because found register might be pending a wait.
2868 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT))
2869 .addImm(Val: 0);
2870
2871 // Insert V_SWAP_B32 instruction(s) and run hazard recognizer on them.
2872 if (Overlapped)
2873 runOnInstruction(
2874 MI: BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SWAP_B32), DestReg: NewAmtLo)
2875 .addDef(RegNo: AmtReg - 1)
2876 .addReg(RegNo: AmtReg - 1, Flags: RegState::Undef)
2877 .addReg(RegNo: NewAmtLo, Flags: RegState::Undef));
2878 runOnInstruction(MI: BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SWAP_B32), DestReg: NewAmt)
2879 .addDef(RegNo: AmtReg)
2880 .addReg(RegNo: AmtReg, Flags: RegState::Undef)
2881 .addReg(RegNo: NewAmt, Flags: RegState::Undef));
2882
2883 // Instructions emitted after the current instruction will be processed by the
2884 // parent loop of the hazard recognizer in a natural way.
2885 BuildMI(BB&: *MBB, I: std::next(x: MI->getIterator()), MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SWAP_B32),
2886 DestReg: AmtReg)
2887 .addDef(RegNo: NewAmt)
2888 .addReg(RegNo: NewAmt)
2889 .addReg(RegNo: AmtReg);
2890 if (Overlapped)
2891 BuildMI(BB&: *MBB, I: std::next(x: MI->getIterator()), MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SWAP_B32),
2892 DestReg: AmtReg - 1)
2893 .addDef(RegNo: NewAmtLo)
2894 .addReg(RegNo: NewAmtLo)
2895 .addReg(RegNo: AmtReg - 1);
2896
2897 // Re-running hazard recognizer on the modified instruction is not necessary,
2898 // inserted V_SWAP_B32 has already both read and write new registers so
2899 // hazards related to these register has already been handled.
2900 Amt->setReg(NewAmt);
2901 Amt->setIsKill(false);
2902 // We do not update liveness, so verifier may see it as undef.
2903 Amt->setIsUndef();
2904 if (Overlapped) {
2905 MI->getOperand(i: 0).setReg(NewReg);
2906 Src1->setReg(NewReg);
2907 Src1->setIsKill(false);
2908 Src1->setIsUndef();
2909 }
2910
2911 return true;
2912}
2913
2914int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) const {
2915 int NSAtoVMEMWaitStates = 1;
2916
2917 if (!ST.hasNSAtoVMEMBug())
2918 return 0;
2919
2920 if (!SIInstrInfo::isMUBUF(MI: *MI) && !SIInstrInfo::isMTBUF(MI: *MI))
2921 return 0;
2922
2923 const SIInstrInfo *TII = ST.getInstrInfo();
2924 const auto *Offset = TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::offset);
2925 if (!Offset || (Offset->getImm() & 6) == 0)
2926 return 0;
2927
2928 auto IsHazardFn = [TII](const MachineInstr &I) {
2929 if (!SIInstrInfo::isMIMG(MI: I))
2930 return false;
2931 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc: I.getOpcode());
2932 return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA &&
2933 TII->getInstSizeInBytes(MI: I) >= 16;
2934 };
2935
2936 return NSAtoVMEMWaitStates - getWaitStatesSince(IsHazard: IsHazardFn, Limit: 1);
2937}
2938
2939int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(
2940 MachineInstr *MI) const {
2941 int FPAtomicToDenormModeWaitStates = 3;
2942
2943 if (!ST.hasFPAtomicToDenormModeHazard())
2944 return 0;
2945 assert(!ST.hasExtendedWaitCounts());
2946
2947 if (MI->getOpcode() != AMDGPU::S_DENORM_MODE)
2948 return 0;
2949
2950 auto IsHazardFn = [](const MachineInstr &I) {
2951 if (!SIInstrInfo::isVMEM(MI: I))
2952 return false;
2953 return SIInstrInfo::isFPAtomic(MI: I);
2954 };
2955
2956 auto IsExpiredFn = [](const MachineInstr &MI, int WaitStates) {
2957 if (WaitStates >= 3 || SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
2958 return true;
2959
2960 return SIInstrInfo::isWaitcnt(Opcode: MI.getOpcode());
2961 };
2962
2963 return FPAtomicToDenormModeWaitStates -
2964 ::getWaitStatesSince(IsHazard: IsHazardFn, MI, IsExpired: IsExpiredFn);
2965}
2966
2967int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) const {
2968 assert(SIInstrInfo::isMAI(*MI));
2969
2970 return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI);
2971}
2972
2973int GCNHazardRecognizer::checkMFMAPadding(MachineInstr *MI) const {
2974 // Early exit if no padding is requested.
2975 if (MFMAPaddingRatio == 0)
2976 return 0;
2977
2978 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2979 if (!SIInstrInfo::isMFMA(MI: *MI) || MFI->getOccupancy() < 2)
2980 return 0;
2981
2982 int NeighborMFMALatency = 0;
2983 auto IsNeighboringMFMA = [&NeighborMFMALatency,
2984 this](const MachineInstr &MI) {
2985 if (!SIInstrInfo::isMFMA(MI))
2986 return false;
2987
2988 NeighborMFMALatency = this->getMFMAPipelineWaitStates(MI);
2989 return true;
2990 };
2991
2992 const int MaxMFMAPipelineWaitStates = 16;
2993 int WaitStatesSinceNeighborMFMA =
2994 getWaitStatesSince(IsHazard: IsNeighboringMFMA, Limit: MaxMFMAPipelineWaitStates);
2995
2996 int NeighborMFMAPaddingNeeded =
2997 (NeighborMFMALatency * MFMAPaddingRatio / 100) -
2998 WaitStatesSinceNeighborMFMA;
2999
3000 return std::max(a: 0, b: NeighborMFMAPaddingNeeded);
3001}
3002
3003int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) const {
3004 int WaitStatesNeeded = 0;
3005 unsigned Opc = MI->getOpcode();
3006
3007 auto IsVALUFn = [](const MachineInstr &MI) {
3008 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) || MI.isInlineAsm();
3009 };
3010
3011 if (Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) { // MFMA or v_accvgpr_write
3012 const int LegacyVALUWritesVGPRWaitStates = 2;
3013 const int VALUWritesExecWaitStates = 4;
3014 const int MaxWaitStates = 4;
3015
3016 int WaitStatesNeededForUse = VALUWritesExecWaitStates -
3017 getWaitStatesSinceDef(Reg: AMDGPU::EXEC, IsHazardDef: IsVALUFn, Limit: MaxWaitStates);
3018 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3019
3020 if (WaitStatesNeeded < MaxWaitStates) {
3021 for (const MachineOperand &Use : MI->explicit_uses()) {
3022 const int MaxWaitStates = 2;
3023
3024 if (!Use.isReg() || !TRI.isVGPR(MRI: MF.getRegInfo(), Reg: Use.getReg()))
3025 continue;
3026
3027 int WaitStatesNeededForUse = LegacyVALUWritesVGPRWaitStates -
3028 getWaitStatesSinceDef(Reg: Use.getReg(), IsHazardDef: IsVALUFn, Limit: MaxWaitStates);
3029 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3030
3031 if (WaitStatesNeeded == MaxWaitStates)
3032 break;
3033 }
3034 }
3035 }
3036
3037 for (const MachineOperand &Op : MI->explicit_operands()) {
3038 if (!Op.isReg() || !TRI.isAGPR(MRI: MF.getRegInfo(), Reg: Op.getReg()))
3039 continue;
3040
3041 if (Op.isDef() && Opc != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
3042 continue;
3043
3044 const int MFMAWritesAGPROverlappedSrcABWaitStates = 4;
3045 const int MFMAWritesAGPROverlappedSrcCWaitStates = 2;
3046 const int MFMA4x4WritesAGPRAccVgprReadWaitStates = 4;
3047 const int MFMA16x16WritesAGPRAccVgprReadWaitStates = 10;
3048 const int MFMA32x32WritesAGPRAccVgprReadWaitStates = 18;
3049 const int MFMA4x4WritesAGPRAccVgprWriteWaitStates = 1;
3050 const int MFMA16x16WritesAGPRAccVgprWriteWaitStates = 7;
3051 const int MFMA32x32WritesAGPRAccVgprWriteWaitStates = 15;
3052 const int MaxWaitStates = 18;
3053 Register Reg = Op.getReg();
3054 unsigned HazardDefLatency = 0;
3055
3056 auto IsOverlappedMFMAFn = [Reg, &HazardDefLatency,
3057 this](const MachineInstr &MI) {
3058 if (!SIInstrInfo::isMFMA(MI))
3059 return false;
3060 Register DstReg = MI.getOperand(i: 0).getReg();
3061 if (DstReg == Reg)
3062 return false;
3063 HazardDefLatency =
3064 std::max(a: HazardDefLatency, b: TSchedModel.computeInstrLatency(MI: &MI));
3065 return TRI.regsOverlap(RegA: DstReg, RegB: Reg);
3066 };
3067
3068 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsHazardDef: IsOverlappedMFMAFn,
3069 Limit: MaxWaitStates);
3070 int NeedWaitStates = MFMAWritesAGPROverlappedSrcABWaitStates;
3071 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2);
3072 int OpNo = Op.getOperandNo();
3073 if (OpNo == SrcCIdx) {
3074 NeedWaitStates = MFMAWritesAGPROverlappedSrcCWaitStates;
3075 } else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) {
3076 switch (HazardDefLatency) {
3077 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprReadWaitStates;
3078 break;
3079 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprReadWaitStates;
3080 break;
3081 case 16: [[fallthrough]];
3082 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprReadWaitStates;
3083 break;
3084 }
3085 } else if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
3086 switch (HazardDefLatency) {
3087 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprWriteWaitStates;
3088 break;
3089 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprWriteWaitStates;
3090 break;
3091 case 16: [[fallthrough]];
3092 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprWriteWaitStates;
3093 break;
3094 }
3095 }
3096
3097 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
3098 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3099
3100 if (WaitStatesNeeded == MaxWaitStates)
3101 return WaitStatesNeeded; // Early exit.
3102
3103 auto IsAccVgprWriteFn = [Reg, this](const MachineInstr &MI) {
3104 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
3105 return false;
3106 Register DstReg = MI.getOperand(i: 0).getReg();
3107 return TRI.regsOverlap(RegA: Reg, RegB: DstReg);
3108 };
3109
3110 const int AccVGPRWriteMFMAReadSrcCWaitStates = 1;
3111 const int AccVGPRWriteMFMAReadSrcABWaitStates = 3;
3112 const int AccVGPRWriteAccVgprReadWaitStates = 3;
3113 NeedWaitStates = AccVGPRWriteMFMAReadSrcABWaitStates;
3114 if (OpNo == SrcCIdx)
3115 NeedWaitStates = AccVGPRWriteMFMAReadSrcCWaitStates;
3116 else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64)
3117 NeedWaitStates = AccVGPRWriteAccVgprReadWaitStates;
3118
3119 WaitStatesNeededForUse = NeedWaitStates -
3120 getWaitStatesSinceDef(Reg, IsHazardDef: IsAccVgprWriteFn, Limit: MaxWaitStates);
3121 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3122
3123 if (WaitStatesNeeded == MaxWaitStates)
3124 return WaitStatesNeeded; // Early exit.
3125 }
3126
3127 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
3128 const int MFMA4x4ReadSrcCAccVgprWriteWaitStates = 0;
3129 const int MFMA16x16ReadSrcCAccVgprWriteWaitStates = 5;
3130 const int MFMA32x32ReadSrcCAccVgprWriteWaitStates = 13;
3131 const int MaxWaitStates = 13;
3132 Register DstReg = MI->getOperand(i: 0).getReg();
3133 unsigned HazardDefLatency = 0;
3134
3135 auto IsSrcCMFMAFn = [DstReg, &HazardDefLatency,
3136 this](const MachineInstr &MI) {
3137 if (!SIInstrInfo::isMFMA(MI))
3138 return false;
3139 Register Reg = TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src2)->getReg();
3140 HazardDefLatency =
3141 std::max(a: HazardDefLatency, b: TSchedModel.computeInstrLatency(MI: &MI));
3142 return TRI.regsOverlap(RegA: Reg, RegB: DstReg);
3143 };
3144
3145 int WaitStatesSince = getWaitStatesSince(IsHazard: IsSrcCMFMAFn, Limit: MaxWaitStates);
3146 int NeedWaitStates;
3147 switch (HazardDefLatency) {
3148 case 2: NeedWaitStates = MFMA4x4ReadSrcCAccVgprWriteWaitStates;
3149 break;
3150 case 8: NeedWaitStates = MFMA16x16ReadSrcCAccVgprWriteWaitStates;
3151 break;
3152 case 16: [[fallthrough]];
3153 default: NeedWaitStates = MFMA32x32ReadSrcCAccVgprWriteWaitStates;
3154 break;
3155 }
3156
3157 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSince;
3158 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3159 }
3160
3161 // Pad neighboring MFMA with noops for better inter-wave performance.
3162 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: checkMFMAPadding(MI));
3163
3164 return WaitStatesNeeded;
3165}
3166
3167static int
3168GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates(int NumPasses,
3169 bool IsGFX950) {
3170 // xdl def cycles | gfx940 | gfx950
3171 // 2 pass | 3 4
3172 // 4 pass | 5 6
3173 // 8 pass | 9 10
3174 // 16 pass | 17 18
3175 return NumPasses + 1 + IsGFX950;
3176}
3177
3178static int
3179GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates(int NumPasses,
3180 bool IsGFX950) {
3181 // xdl def cycles | gfx940 | gfx950
3182 // 2 pass | 3 3
3183 // 4 pass | 5 6
3184 // 8 pass | 9 10
3185 // 16 pass | 17 18
3186 return NumPasses + 1 + (NumPasses != 2 && IsGFX950);
3187}
3188
3189static int
3190GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates(int NumPasses) {
3191 // 2 pass -> 2
3192 // 4 pass -> 4
3193 // 8 pass -> 8
3194 // 16 pass -> 16
3195 return NumPasses;
3196}
3197
3198static int
3199GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses) {
3200 // 2 pass -> 4
3201 // 4 pass -> 6
3202 // 8 pass -> 10
3203 // 16 pass -> 18
3204 return NumPasses + 2;
3205}
3206
3207static int GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses,
3208 bool IsGFX950) {
3209 // xdl def cycles | gfx942 | gfx950
3210 // 2 pass | 5 5
3211 // 4 pass | 7 8
3212 // 8 pass | 11 12
3213 // 16 pass | 19 20
3214 return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
3215}
3216
3217int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) const {
3218 int WaitStatesNeeded = 0;
3219 unsigned Opc = MI->getOpcode();
3220
3221 auto IsLegacyVALUFn = [](const MachineInstr &MI) {
3222 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
3223 !SIInstrInfo::isMFMA(MI);
3224 };
3225
3226 auto IsLegacyVALUNotDotFn = [](const MachineInstr &MI) {
3227 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
3228 !SIInstrInfo::isMFMA(MI) && !SIInstrInfo::isDOT(MI);
3229 };
3230
3231 if (!SIInstrInfo::isMFMA(MI: *MI))
3232 return WaitStatesNeeded;
3233
3234 const int VALUWritesExecWaitStates = 4;
3235 int WaitStatesNeededForUse = VALUWritesExecWaitStates -
3236 getWaitStatesSinceDef(Reg: AMDGPU::EXEC, IsHazardDef: IsLegacyVALUFn,
3237 Limit: VALUWritesExecWaitStates);
3238 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3239
3240 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2);
3241
3242 // Loop for both DGEMM and S/HGEMM 2nd instruction.
3243 for (const MachineOperand &Use : MI->explicit_uses()) {
3244 const int LegacyVALUNotDotWritesVGPRWaitStates = 2;
3245 const int SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates = 2;
3246 const int SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates = 8;
3247 const int SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates = 16;
3248 const int SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates = 3;
3249 const int SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates = 9;
3250 const int SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates = 17;
3251 const int DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 9;
3252 const int GFX950_DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 17;
3253 const int DMFMA4x4WritesVGPROverlappedSrcCWaitStates = 4;
3254 const int SMFMA4x4WritesVGPROverlappedSrcABWaitStates = 5;
3255 const int SMFMA16x16WritesVGPROverlappedSrcABWaitStates = 11;
3256 const int SMFMA32x32WritesVGPROverlappedSrcABWaitStates = 19;
3257 const int DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates = 6;
3258 const int DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 11;
3259 const int GFX950_DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 19;
3260 const int DMFMA4x4WritesVGPRFullSrcCWaitStates = 4;
3261 const int GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates = 2;
3262 const int MaxWaitStates = 19;
3263
3264 if (!Use.isReg())
3265 continue;
3266 Register Reg = Use.getReg();
3267 bool FullReg;
3268 const MachineInstr *MI1;
3269
3270 auto IsOverlappedMFMAFn = [Reg, &FullReg, &MI1,
3271 this](const MachineInstr &MI) {
3272 if (!SIInstrInfo::isMFMA(MI))
3273 return false;
3274 Register DstReg = MI.getOperand(i: 0).getReg();
3275 FullReg = (DstReg == Reg);
3276 MI1 = &MI;
3277 return TRI.regsOverlap(RegA: DstReg, RegB: Reg);
3278 };
3279
3280 WaitStatesNeededForUse = LegacyVALUNotDotWritesVGPRWaitStates -
3281 getWaitStatesSinceDef(Reg, IsHazardDef: IsLegacyVALUNotDotFn, Limit: MaxWaitStates);
3282 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3283
3284 int NumWaitStates =
3285 getWaitStatesSinceDef(Reg, IsHazardDef: IsOverlappedMFMAFn, Limit: MaxWaitStates);
3286 if (NumWaitStates == std::numeric_limits<int>::max())
3287 continue;
3288
3289 int OpNo = Use.getOperandNo();
3290 unsigned Opc1 = MI1->getOpcode();
3291 int NeedWaitStates = 0;
3292 if (OpNo == SrcCIdx) {
3293 if (!SIInstrInfo::isDGEMM(Opcode: Opc) &&
3294 (!ST.hasGFX940Insts() && SIInstrInfo::isDGEMM(Opcode: Opc1))) {
3295 NeedWaitStates = 0;
3296 } else if (FullReg) {
3297 if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
3298 Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) &&
3299 (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
3300 Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64))
3301 NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates;
3302 else if (ST.hasGFX940Insts() &&
3303 TSchedModel.computeInstrLatency(MI: MI1) == 2)
3304 NeedWaitStates = GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates;
3305 } else {
3306 switch (Opc1) {
3307 case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
3308 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
3309 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
3310 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
3311 if (!TII.isXDL(MI: *MI))
3312 NeedWaitStates =
3313 ST.hasGFX950Insts()
3314 ? GFX950_DMFMA16x16WritesVGPROverlappedSrcCWaitStates
3315 : DMFMA16x16WritesVGPROverlappedSrcCWaitStates;
3316 break;
3317 case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
3318 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
3319 if (!TII.isXDL(MI: *MI))
3320 NeedWaitStates = DMFMA4x4WritesVGPROverlappedSrcCWaitStates;
3321 break;
3322 default:
3323 int NumPasses = TSchedModel.computeInstrLatency(MI: MI1);
3324 if (ST.hasGFX940Insts()) {
3325 if (TII.isXDL(MI: *MI) && !TII.isXDL(MI: *MI1))
3326 break;
3327
3328 NeedWaitStates =
3329 TII.isXDL(MI: *MI1)
3330 ? (TII.isXDL(MI: *MI)
3331 ? GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates(
3332 NumPasses, IsGFX950: ST.hasGFX950Insts())
3333 : GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates(
3334 NumPasses, IsGFX950: ST.hasGFX950Insts()))
3335 : GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates(
3336 NumPasses);
3337 break;
3338 }
3339
3340 switch (NumPasses) {
3341 case 2:
3342 NeedWaitStates =
3343 SIInstrInfo::isDGEMM(Opcode: Opc)
3344 ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates
3345 : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates;
3346 break;
3347 case 8:
3348 NeedWaitStates =
3349 SIInstrInfo::isDGEMM(Opcode: Opc)
3350 ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates
3351 : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates;
3352 break;
3353 case 16:
3354 NeedWaitStates =
3355 SIInstrInfo::isDGEMM(Opcode: Opc)
3356 ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates
3357 : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates;
3358 break;
3359 default:
3360 llvm_unreachable("unexpected number of passes");
3361 }
3362 }
3363 }
3364 } else {
3365 switch (Opc1) {
3366 case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
3367 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
3368 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
3369 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
3370 NeedWaitStates =
3371 ST.hasGFX950Insts()
3372 ? GFX950_DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates
3373 : DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates;
3374 break;
3375 case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
3376 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
3377 NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates;
3378 break;
3379 default:
3380 int NumPasses = TSchedModel.computeInstrLatency(MI: MI1);
3381
3382 if (ST.hasGFX940Insts()) {
3383 NeedWaitStates =
3384 TII.isXDL(MI: *MI1)
3385 ? GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(
3386 NumPasses, IsGFX950: ST.hasGFX950Insts())
3387 : GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates(
3388 NumPasses);
3389 break;
3390 }
3391
3392 switch (NumPasses) {
3393 case 2:
3394 NeedWaitStates = SMFMA4x4WritesVGPROverlappedSrcABWaitStates;
3395 break;
3396 case 4:
3397 llvm_unreachable("unexpected number of passes for mfma");
3398 case 8:
3399 NeedWaitStates = SMFMA16x16WritesVGPROverlappedSrcABWaitStates;
3400 break;
3401 case 16:
3402 default:
3403 NeedWaitStates = SMFMA32x32WritesVGPROverlappedSrcABWaitStates;
3404 }
3405 }
3406 }
3407 if (WaitStatesNeeded >= NeedWaitStates)
3408 continue;
3409
3410 WaitStatesNeededForUse = NeedWaitStates - NumWaitStates;
3411 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3412
3413 if (WaitStatesNeeded == MaxWaitStates)
3414 break;
3415 }
3416
3417 // Pad neighboring MFMA with noops for better inter-wave performance.
3418 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: checkMFMAPadding(MI));
3419
3420 return WaitStatesNeeded;
3421}
3422
3423int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) const {
3424 // On gfx90a+ relevant hazards are checked in checkMAIVALUHazards()
3425 if (!ST.hasMAIInsts() || ST.hasGFX90AInsts())
3426 return 0;
3427
3428 int WaitStatesNeeded = 0;
3429
3430 auto IsAccVgprReadFn = [](const MachineInstr &MI) {
3431 return MI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64;
3432 };
3433
3434 for (const MachineOperand &Op : MI->explicit_uses()) {
3435 if (!Op.isReg() || !TRI.isVGPR(MRI: MF.getRegInfo(), Reg: Op.getReg()))
3436 continue;
3437
3438 Register Reg = Op.getReg();
3439
3440 const int AccVgprReadLdStWaitStates = 2;
3441 const int VALUWriteAccVgprRdWrLdStDepVALUWaitStates = 1;
3442 const int MaxWaitStates = 2;
3443
3444 int WaitStatesNeededForUse = AccVgprReadLdStWaitStates -
3445 getWaitStatesSinceDef(Reg, IsHazardDef: IsAccVgprReadFn, Limit: MaxWaitStates);
3446 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3447
3448 if (WaitStatesNeeded == MaxWaitStates)
3449 return WaitStatesNeeded; // Early exit.
3450
3451 auto IsVALUAccVgprRdWrCheckFn = [Reg, this](const MachineInstr &MI) {
3452 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 &&
3453 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
3454 return false;
3455 auto IsVALUFn = [](const MachineInstr &MI) {
3456 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
3457 !SIInstrInfo::isMAI(MI);
3458 };
3459 return getWaitStatesSinceDef(Reg, IsHazardDef: IsVALUFn, Limit: 2 /*MaxWaitStates*/) <
3460 std::numeric_limits<int>::max();
3461 };
3462
3463 WaitStatesNeededForUse = VALUWriteAccVgprRdWrLdStDepVALUWaitStates -
3464 getWaitStatesSince(IsHazard: IsVALUAccVgprRdWrCheckFn, Limit: MaxWaitStates);
3465 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3466 }
3467
3468 return WaitStatesNeeded;
3469}
3470
3471int GCNHazardRecognizer::checkPermlaneHazards(MachineInstr *MI) const {
3472 assert(!ST.hasVcmpxPermlaneHazard() &&
3473 "this is a different vcmpx+permlane hazard");
3474 const SIRegisterInfo *TRI = ST.getRegisterInfo();
3475 const SIInstrInfo *TII = ST.getInstrInfo();
3476
3477 auto IsVCmpXWritesExecFn = [TII, TRI](const MachineInstr &MI) {
3478 return isVCmpXWritesExec(TII: *TII, TRI: *TRI, MI);
3479 };
3480
3481 auto IsVALUFn = [](const MachineInstr &MI) {
3482 return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true);
3483 };
3484
3485 const int VCmpXWritesExecWaitStates = 4;
3486 const int VALUWritesVDstWaitStates = 2;
3487 int WaitStatesNeeded = 0;
3488
3489 for (const MachineOperand &Op : MI->explicit_uses()) {
3490 if (!Op.isReg() || !TRI->isVGPR(MRI: MF.getRegInfo(), Reg: Op.getReg()))
3491 continue;
3492 Register Reg = Op.getReg();
3493
3494 int WaitStatesSinceDef =
3495 VALUWritesVDstWaitStates -
3496 getWaitStatesSinceDef(Reg, IsHazardDef: IsVALUFn,
3497 /*MaxWaitStates=*/Limit: VALUWritesVDstWaitStates);
3498 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesSinceDef);
3499 if (WaitStatesNeeded >= VALUWritesVDstWaitStates)
3500 break;
3501 }
3502
3503 int VCmpXHazardWaits =
3504 VCmpXWritesExecWaitStates -
3505 getWaitStatesSince(IsHazard: IsVCmpXWritesExecFn, Limit: VCmpXWritesExecWaitStates);
3506
3507 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: VCmpXHazardWaits);
3508 return WaitStatesNeeded;
3509}
3510
3511static int GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(int NumPasses) {
3512 // 2 pass -> 4
3513 // 4 pass -> 6
3514 // 8 pass -> 10
3515 // 16 pass -> 18
3516 return NumPasses + 2;
3517}
3518
3519static int GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(int NumPasses,
3520 bool IsGFX950) {
3521 // xdl def cycles | gfx942 | gfx950
3522 // 2 pass | 5 5
3523 // 4 pass | 7 8
3524 // 8 pass | 11 12
3525 // 16 pass | 19 20
3526 return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
3527}
3528
3529static int GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses,
3530 bool IsGFX950) {
3531 // xdl def cycles | gfx942 | gfx950
3532 // 2 pass | 5 5
3533 // 4 pass | 7 8
3534 // 8 pass | 11 12
3535 // 16 pass | 19 20
3536 return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
3537}
3538
3539static int GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses) {
3540 // 2 pass -> 4
3541 // 4 pass -> 6
3542 // 8 pass -> 10
3543 // 16 pass -> 18
3544 return NumPasses + 2;
3545}
3546
3547int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const {
3548 if (!ST.hasGFX90AInsts())
3549 return 0;
3550
3551 auto IsDGEMMFn = [](const MachineInstr &MI) -> bool {
3552 return SIInstrInfo::isDGEMM(Opcode: MI.getOpcode());
3553 };
3554
3555 // This is checked in checkMAIHazards90A()
3556 if (SIInstrInfo::isMFMA(MI: *MI))
3557 return 0;
3558
3559 const MachineRegisterInfo &MRI = MF.getRegInfo();
3560
3561 int WaitStatesNeeded = 0;
3562
3563 bool IsMem = SIInstrInfo::isVMEM(MI: *MI) || SIInstrInfo::isDS(MI: *MI);
3564 bool IsMemOrExport = IsMem || SIInstrInfo::isEXP(MI: *MI);
3565 bool IsVALU = SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true);
3566
3567 const MachineInstr *MFMA = nullptr;
3568 unsigned Reg;
3569 auto IsMFMAWriteFn = [&Reg, &MFMA, this](const MachineInstr &MI) {
3570 if (!SIInstrInfo::isMFMA(MI) ||
3571 !TRI.regsOverlap(RegA: MI.getOperand(i: 0).getReg(), RegB: Reg))
3572 return false;
3573 MFMA = &MI;
3574 return true;
3575 };
3576
3577 const MachineInstr *DOT = nullptr;
3578 auto IsDotWriteFn = [&Reg, &DOT, this](const MachineInstr &MI) {
3579 if (!SIInstrInfo::isDOT(MI) ||
3580 !TRI.regsOverlap(RegA: MI.getOperand(i: 0).getReg(), RegB: Reg))
3581 return false;
3582 DOT = &MI;
3583 return true;
3584 };
3585
3586 bool DGEMMAfterVALUWrite = false;
3587 auto IsDGEMMHazard = [&DGEMMAfterVALUWrite, this](const MachineInstr &MI) {
3588 // Found DGEMM on reverse traversal to def.
3589 if (SIInstrInfo::isDGEMM(Opcode: MI.getOpcode()))
3590 DGEMMAfterVALUWrite = true;
3591
3592 // Only hazard if register is defined by a VALU and a DGEMM is found after
3593 // after the def.
3594 if (!TII.isVALU(MI, /*AllowLDSDMA=*/true) || !DGEMMAfterVALUWrite)
3595 return false;
3596
3597 return true;
3598 };
3599
3600 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opcode: MI->getOpcode(),
3601 Name: AMDGPU::OpName::src2);
3602
3603 if (IsMemOrExport || IsVALU) {
3604 const int SMFMA4x4WriteVgprVALUMemExpReadWaitStates = 5;
3605 const int SMFMA16x16WriteVgprVALUMemExpReadWaitStates = 11;
3606 const int SMFMA32x32WriteVgprVALUMemExpReadWaitStates = 19;
3607 const int DMFMA4x4WriteVgprMemExpReadWaitStates = 9;
3608 const int DMFMA16x16WriteVgprMemExpReadWaitStates = 18;
3609 const int DMFMA4x4WriteVgprVALUReadWaitStates = 6;
3610 const int DMFMA16x16WriteVgprVALUReadWaitStates = 11;
3611 const int GFX950_DMFMA16x16WriteVgprVALUReadWaitStates = 19;
3612 const int DotWriteSameDotReadSrcAB = 3;
3613 const int DotWriteDifferentVALURead = 3;
3614 const int DMFMABetweenVALUWriteVMEMRead = 2;
3615 const int MaxWaitStates = 19;
3616
3617 for (const MachineOperand &Use : MI->explicit_uses()) {
3618 if (!Use.isReg())
3619 continue;
3620 Reg = Use.getReg();
3621
3622 DOT = nullptr;
3623 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsHazardDef: IsDotWriteFn,
3624 Limit: MaxWaitStates);
3625 if (DOT) {
3626 int NeedWaitStates = 0;
3627 if (DOT->getOpcode() == MI->getOpcode()) {
3628 if (&Use - &MI->getOperand(i: 0) != SrcCIdx)
3629 NeedWaitStates = DotWriteSameDotReadSrcAB;
3630 } else {
3631 NeedWaitStates = DotWriteDifferentVALURead;
3632 }
3633
3634 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
3635 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3636 }
3637
3638 // Workaround for HW data hazard bug observed only in GFX90A. When there
3639 // is a DGEMM instruction in-between a VALU and a VMEM instruction it
3640 // causes the SQ to incorrectly not insert two wait states between the two
3641 // instructions needed to avoid data hazard.
3642 if (IsMem && ST.hasGFX90AInsts() && !ST.hasGFX940Insts()) {
3643 DGEMMAfterVALUWrite = false;
3644 if (TRI.isVectorRegister(MRI, Reg)) {
3645 int WaitStatesNeededForUse =
3646 DMFMABetweenVALUWriteVMEMRead -
3647 getWaitStatesSinceDef(Reg, IsHazardDef: IsDGEMMHazard,
3648 Limit: DMFMABetweenVALUWriteVMEMRead);
3649
3650 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3651 }
3652 }
3653
3654 MFMA = nullptr;
3655 WaitStatesSinceDef =
3656 getWaitStatesSinceDef(Reg, IsHazardDef: IsMFMAWriteFn, Limit: MaxWaitStates);
3657 if (!MFMA)
3658 continue;
3659
3660 unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MI: MFMA);
3661 int NumPasses = HazardDefLatency;
3662 int NeedWaitStates = MaxWaitStates;
3663
3664 if (SIInstrInfo::isDGEMM(Opcode: MFMA->getOpcode())) {
3665 switch (HazardDefLatency) {
3666 case 4:
3667 NeedWaitStates = IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates
3668 : DMFMA4x4WriteVgprVALUReadWaitStates;
3669 break;
3670 case 8:
3671 case 16:
3672 NeedWaitStates =
3673 IsMemOrExport
3674 ? DMFMA16x16WriteVgprMemExpReadWaitStates
3675 : (ST.hasGFX950Insts()
3676 ? GFX950_DMFMA16x16WriteVgprVALUReadWaitStates
3677 : DMFMA16x16WriteVgprVALUReadWaitStates);
3678 break;
3679 default:
3680 llvm_unreachable("unexpected dgemm");
3681 }
3682 } else if (ST.hasGFX940Insts()) {
3683 NeedWaitStates =
3684 TII.isXDL(MI: *MFMA)
3685 ? GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(
3686 NumPasses, IsGFX950: ST.hasGFX950Insts())
3687 : GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates(
3688 NumPasses);
3689 } else {
3690 switch (HazardDefLatency) {
3691 case 2:
3692 NeedWaitStates = SMFMA4x4WriteVgprVALUMemExpReadWaitStates;
3693 break;
3694 case 8:
3695 NeedWaitStates = SMFMA16x16WriteVgprVALUMemExpReadWaitStates;
3696 break;
3697 case 16:
3698 NeedWaitStates = SMFMA32x32WriteVgprVALUMemExpReadWaitStates;
3699 break;
3700 default:
3701 llvm_unreachable("unexpected number of passes for mfma");
3702 }
3703 }
3704
3705 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
3706 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3707
3708 if (WaitStatesNeeded == MaxWaitStates)
3709 break;
3710 }
3711 }
3712
3713 unsigned Opc = MI->getOpcode();
3714 const int DMFMAToFMA64WaitStates = 2;
3715 if ((Opc == AMDGPU::V_FMA_F64_e64 ||
3716 Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64 ||
3717 Opc == AMDGPU::V_FMAC_F64_dpp) &&
3718 WaitStatesNeeded < DMFMAToFMA64WaitStates) {
3719 int WaitStatesNeededForUse = DMFMAToFMA64WaitStates -
3720 getWaitStatesSince(IsHazard: IsDGEMMFn, Limit: DMFMAToFMA64WaitStates);
3721 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3722 }
3723
3724 if (!IsVALU && !IsMemOrExport)
3725 return WaitStatesNeeded;
3726
3727 for (const MachineOperand &Def : MI->defs()) {
3728 const int SMFMA4x4WriteVgprVALUWawWaitStates = 5;
3729 const int SMFMA16x16WriteVgprVALUWawWaitStates = 11;
3730 const int SMFMA32x32WriteVgprVALUWawWaitStates = 19;
3731 const int SMFMA4x4ReadVgprVALUWarWaitStates = 1;
3732 const int GFX940_XDL4PassReadVgprVALUWarWaitStates = 3;
3733 const int SMFMA16x16ReadVgprVALUWarWaitStates = 7;
3734 const int SMFMA32x32ReadVgprVALUWarWaitStates = 15;
3735 const int DMFMA4x4WriteVgprVALUWriteWaitStates = 6;
3736 const int DMFMA16x16WriteVgprVALUWriteWaitStates = 11;
3737 const int DotWriteDifferentVALUWrite = 3;
3738 const int MaxWaitStates = 19;
3739 const int MaxWarWaitStates = 15;
3740
3741 Reg = Def.getReg();
3742
3743 DOT = nullptr;
3744 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsHazardDef: IsDotWriteFn,
3745 Limit: MaxWaitStates);
3746 if (DOT && DOT->getOpcode() != MI->getOpcode())
3747 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: DotWriteDifferentVALUWrite -
3748 WaitStatesSinceDef);
3749
3750 MFMA = nullptr;
3751 WaitStatesSinceDef =
3752 getWaitStatesSinceDef(Reg, IsHazardDef: IsMFMAWriteFn, Limit: MaxWaitStates);
3753 if (MFMA) {
3754 int NeedWaitStates = MaxWaitStates;
3755 int NumPasses = TSchedModel.computeInstrLatency(MI: MFMA);
3756
3757 if (SIInstrInfo::isDGEMM(Opcode: MFMA->getOpcode())) {
3758 switch (NumPasses) {
3759 case 4:
3760 NeedWaitStates = DMFMA4x4WriteVgprVALUWriteWaitStates;
3761 break;
3762 case 8:
3763 case 16:
3764 NeedWaitStates = DMFMA16x16WriteVgprVALUWriteWaitStates;
3765 break;
3766 default:
3767 llvm_unreachable("unexpected number of cycles for dgemm");
3768 }
3769 } else if (ST.hasGFX940Insts()) {
3770 NeedWaitStates =
3771 TII.isXDL(MI: *MFMA)
3772 ? GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(
3773 NumPasses, IsGFX950: ST.hasGFX950Insts())
3774 : GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(NumPasses);
3775 } else {
3776 switch (NumPasses) {
3777 case 2:
3778 NeedWaitStates = SMFMA4x4WriteVgprVALUWawWaitStates;
3779 break;
3780 case 8:
3781 NeedWaitStates = SMFMA16x16WriteVgprVALUWawWaitStates;
3782 break;
3783 case 16:
3784 NeedWaitStates = SMFMA32x32WriteVgprVALUWawWaitStates;
3785 break;
3786 default:
3787 llvm_unreachable("Unexpected number of passes for mfma");
3788 }
3789 }
3790
3791 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
3792 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3793
3794 if (WaitStatesNeeded == MaxWaitStates)
3795 break;
3796 }
3797
3798 auto IsSMFMAReadAsCFn = [&Reg, &MFMA, this](const MachineInstr &MI) {
3799 if (!SIInstrInfo::isMFMA(MI) || SIInstrInfo::isDGEMM(Opcode: MI.getOpcode()) ||
3800 !MI.readsRegister(Reg, TRI: &TRI))
3801 return false;
3802
3803 if (ST.hasGFX940Insts() && !TII.isXDL(MI))
3804 return false;
3805
3806 const MachineOperand *SrcC =
3807 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
3808 assert(SrcC);
3809 if (!SrcC->isReg() || !TRI.regsOverlap(RegA: SrcC->getReg(), RegB: Reg))
3810 return false;
3811
3812 MFMA = &MI;
3813 return true;
3814 };
3815
3816 MFMA = nullptr;
3817 int WaitStatesSinceUse = getWaitStatesSince(IsHazard: IsSMFMAReadAsCFn,
3818 Limit: MaxWarWaitStates);
3819 if (!MFMA)
3820 continue;
3821
3822 unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MI: MFMA);
3823 int NeedWaitStates = MaxWaitStates;
3824 switch (HazardDefLatency) {
3825 case 2: NeedWaitStates = SMFMA4x4ReadVgprVALUWarWaitStates;
3826 break;
3827 case 4: assert(ST.hasGFX940Insts());
3828 NeedWaitStates = GFX940_XDL4PassReadVgprVALUWarWaitStates;
3829 break;
3830 case 8: NeedWaitStates = SMFMA16x16ReadVgprVALUWarWaitStates;
3831 break;
3832 case 16: [[fallthrough]];
3833 default: NeedWaitStates = SMFMA32x32ReadVgprVALUWarWaitStates;
3834 break;
3835 }
3836
3837 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceUse;
3838 WaitStatesNeeded = std::max(a: WaitStatesNeeded, b: WaitStatesNeededForUse);
3839 }
3840
3841 return WaitStatesNeeded;
3842}
3843
3844bool GCNHazardRecognizer::ShouldPreferAnother(SUnit *SU) const {
3845 if (!SU->isInstr())
3846 return false;
3847
3848 const MachineInstr *MAI = nullptr;
3849
3850 auto IsMFMAFn = [&MAI](const MachineInstr &MI) {
3851 MAI = nullptr;
3852 if (SIInstrInfo::isMFMA(MI))
3853 MAI = &MI;
3854 return MAI != nullptr;
3855 };
3856
3857 MachineInstr *MI = SU->getInstr();
3858 if (IsMFMAFn(*MI)) {
3859 int W = getWaitStatesSince(IsHazard: IsMFMAFn, Limit: 16);
3860 if (MAI)
3861 return W < (int)TSchedModel.computeInstrLatency(MI: MAI);
3862 }
3863
3864 return false;
3865}
3866
3867// Adjust global offsets for instructions bundled with S_GETPC_B64 after
3868// insertion of a new instruction.
3869static void updateGetPCBundle(MachineInstr *NewMI) {
3870 if (!NewMI->isBundled())
3871 return;
3872
3873 // Find start of bundle.
3874 auto I = NewMI->getIterator();
3875 while (I->isBundledWithPred())
3876 I--;
3877 if (I->isBundle())
3878 I++;
3879
3880 // Bail if this is not an S_GETPC bundle.
3881 if (I->getOpcode() != AMDGPU::S_GETPC_B64)
3882 return;
3883
3884 // Update offsets of any references in the bundle.
3885 const unsigned NewBytes = 4;
3886 assert(NewMI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
3887 "Unexpected instruction insertion in bundle");
3888 auto NextMI = std::next(x: NewMI->getIterator());
3889 auto End = NewMI->getParent()->end();
3890 while (NextMI != End && NextMI->isBundledWithPred()) {
3891 for (auto &Operand : NextMI->operands()) {
3892 if (Operand.isGlobal())
3893 Operand.setOffset(Operand.getOffset() + NewBytes);
3894 }
3895 NextMI++;
3896 }
3897}
3898
3899bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) {
3900 if (!ST.hasVALUMaskWriteHazard())
3901 return false;
3902 assert(!ST.hasExtendedWaitCounts());
3903
3904 if (!ST.isWave64())
3905 return false;
3906
3907 const bool IsSALU = SIInstrInfo::isSALU(MI: *MI);
3908 const bool IsVALU = SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true);
3909 if (!IsSALU && !IsVALU)
3910 return false;
3911
3912 // The hazard sequence is three instructions:
3913 // 1. VALU reads SGPR as mask
3914 // 2. VALU/SALU writes SGPR
3915 // 3. VALU/SALU reads SGPR
3916 // The hazard can expire if the distance between 2 and 3 is sufficient,
3917 // or (2) is VALU and (3) is SALU.
3918 // In practice this happens <10% of the time, hence always assume the hazard
3919 // exists if (1) and (2) are present to avoid searching all SGPR reads.
3920
3921 const SIRegisterInfo *TRI = ST.getRegisterInfo();
3922 const MachineRegisterInfo &MRI = MF.getRegInfo();
3923
3924 auto IgnoreableSGPR = [](const Register Reg) {
3925 switch (Reg) {
3926 case AMDGPU::EXEC:
3927 case AMDGPU::EXEC_LO:
3928 case AMDGPU::EXEC_HI:
3929 case AMDGPU::M0:
3930 case AMDGPU::SGPR_NULL:
3931 case AMDGPU::SGPR_NULL64:
3932 case AMDGPU::SCC:
3933 return true;
3934 default:
3935 return false;
3936 }
3937 };
3938 auto IsVCC = [](const Register Reg) {
3939 return Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::VCC_HI;
3940 };
3941
3942 struct StateType {
3943 SmallSet<Register, 2> HazardSGPRs;
3944
3945 static unsigned getHashValue(const StateType &State) {
3946 return hash_combine_range(R: State.HazardSGPRs);
3947 }
3948 static bool isEqual(const StateType &LHS, const StateType &RHS) {
3949 return LHS.HazardSGPRs == RHS.HazardSGPRs;
3950 }
3951 };
3952
3953 SmallVector<const MachineInstr *> WaitInstrs;
3954 StateType InitialState;
3955
3956 // Look for SGPR write.
3957 MachineOperand *HazardDef = nullptr;
3958 for (MachineOperand &Op : MI->all_defs()) {
3959 Register Reg = Op.getReg();
3960 if (IgnoreableSGPR(Reg))
3961 continue;
3962 if (!IsVCC(Reg)) {
3963 if (Op.isImplicit())
3964 continue;
3965 if (!TRI->isSGPRReg(MRI, Reg))
3966 continue;
3967 }
3968
3969 HazardDef = &Op;
3970 break;
3971 }
3972
3973 if (!HazardDef)
3974 return false;
3975
3976 // Setup to track writes to individual SGPRs
3977 const Register HazardReg = HazardDef->getReg();
3978 if (AMDGPU::SReg_32RegClass.contains(Reg: HazardReg)) {
3979 InitialState.HazardSGPRs.insert(V: HazardReg);
3980 } else {
3981 assert(AMDGPU::SReg_64RegClass.contains(HazardReg));
3982 InitialState.HazardSGPRs.insert(V: TRI->getSubReg(Reg: HazardReg, Idx: AMDGPU::sub0));
3983 InitialState.HazardSGPRs.insert(V: TRI->getSubReg(Reg: HazardReg, Idx: AMDGPU::sub1));
3984 }
3985
3986 auto IsHazardFn = [&](StateType &State, const MachineInstr &I) {
3987 if (State.HazardSGPRs.empty())
3988 return HazardExpired;
3989
3990 switch (I.getOpcode()) {
3991 case AMDGPU::V_ADDC_U32_e32:
3992 case AMDGPU::V_ADDC_U32_dpp:
3993 case AMDGPU::V_CNDMASK_B16_t16_e32:
3994 case AMDGPU::V_CNDMASK_B16_fake16_e32:
3995 case AMDGPU::V_CNDMASK_B16_t16_dpp:
3996 case AMDGPU::V_CNDMASK_B16_fake16_dpp:
3997 case AMDGPU::V_CNDMASK_B32_e32:
3998 case AMDGPU::V_CNDMASK_B32_dpp:
3999 case AMDGPU::V_DIV_FMAS_F32_e64:
4000 case AMDGPU::V_DIV_FMAS_F64_e64:
4001 case AMDGPU::V_SUBB_U32_e32:
4002 case AMDGPU::V_SUBB_U32_dpp:
4003 case AMDGPU::V_SUBBREV_U32_e32:
4004 case AMDGPU::V_SUBBREV_U32_dpp: {
4005 // These implicitly read VCC as mask source.
4006 return IsVCC(HazardReg) ? HazardFound : NoHazardFound;
4007 }
4008 case AMDGPU::V_ADDC_U32_e64:
4009 case AMDGPU::V_ADDC_U32_e64_dpp:
4010 case AMDGPU::V_CNDMASK_B16_t16_e64:
4011 case AMDGPU::V_CNDMASK_B16_fake16_e64:
4012 case AMDGPU::V_CNDMASK_B16_t16_e64_dpp:
4013 case AMDGPU::V_CNDMASK_B16_fake16_e64_dpp:
4014 case AMDGPU::V_CNDMASK_B32_e64:
4015 case AMDGPU::V_CNDMASK_B32_e64_dpp:
4016 case AMDGPU::V_SUBB_U32_e64:
4017 case AMDGPU::V_SUBB_U32_e64_dpp:
4018 case AMDGPU::V_SUBBREV_U32_e64:
4019 case AMDGPU::V_SUBBREV_U32_e64_dpp: {
4020 // Only check mask register overlaps.
4021 const MachineOperand *SSRCOp = TII.getNamedOperand(MI: I, OperandName: AMDGPU::OpName::src2);
4022 assert(SSRCOp);
4023 bool Result = TRI->regsOverlap(RegA: SSRCOp->getReg(), RegB: HazardReg);
4024 return Result ? HazardFound : NoHazardFound;
4025 }
4026 default:
4027 return NoHazardFound;
4028 }
4029 };
4030
4031 auto UpdateStateFn = [&](StateType &State, const MachineInstr &I) {
4032 // Update tracking of SGPR writes.
4033 for (auto &Op : I.all_defs()) {
4034 Register Reg = Op.getReg();
4035 if (IgnoreableSGPR(Reg))
4036 continue;
4037 if (!IsVCC(Reg)) {
4038 if (Op.isImplicit())
4039 continue;
4040 if (!TRI->isSGPRReg(MRI, Reg))
4041 continue;
4042 }
4043
4044 // Stop tracking any SGPRs with writes on the basis that they will
4045 // already have an appropriate wait inserted afterwards.
4046 SmallVector<Register, 2> Found;
4047 for (Register SGPR : State.HazardSGPRs) {
4048 if (Reg == SGPR || TRI->regsOverlap(RegA: Reg, RegB: SGPR))
4049 Found.push_back(Elt: SGPR);
4050 }
4051 for (Register SGPR : Found)
4052 State.HazardSGPRs.erase(V: SGPR);
4053 }
4054 };
4055
4056 // Check for hazard
4057 if (!hasHazard<StateType>(InitialState, IsHazard: IsHazardFn, UpdateState: UpdateStateFn,
4058 InitialMBB: MI->getParent(),
4059 InitialI: std::next(x: MI->getReverseIterator())))
4060 return false;
4061
4062 // Compute counter mask
4063 unsigned DepCtr =
4064 IsVALU ? (IsVCC(HazardReg) ? AMDGPU::DepCtr::encodeFieldVaVcc(VaVcc: 0, STI: ST)
4065 : AMDGPU::DepCtr::encodeFieldVaSdst(VaSdst: 0, STI: ST))
4066 : AMDGPU::DepCtr::encodeFieldSaSdst(SaSdst: 0, STI: ST);
4067
4068 // Add s_waitcnt_depctr after SGPR write.
4069 auto NextMI = std::next(x: MI->getIterator());
4070 auto NewMI = BuildMI(BB&: *MI->getParent(), I: NextMI, MIMD: MI->getDebugLoc(),
4071 MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4072 .addImm(Val: DepCtr);
4073
4074 // SALU write may be s_getpc in a bundle.
4075 updateGetPCBundle(NewMI);
4076
4077 return true;
4078}
4079
4080static bool ensureEntrySetPrio(MachineFunction *MF, int Priority,
4081 const SIInstrInfo &TII) {
4082 MachineBasicBlock &EntryMBB = MF->front();
4083 if (EntryMBB.begin() != EntryMBB.end()) {
4084 auto &EntryMI = *EntryMBB.begin();
4085 if (EntryMI.getOpcode() == AMDGPU::S_SETPRIO &&
4086 EntryMI.getOperand(i: 0).getImm() >= Priority)
4087 return false;
4088 }
4089
4090 BuildMI(BB&: EntryMBB, I: EntryMBB.begin(), MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_SETPRIO))
4091 .addImm(Val: Priority);
4092 return true;
4093}
4094
4095bool GCNHazardRecognizer::fixRequiredExportPriority(MachineInstr *MI) {
4096 if (!ST.hasRequiredExportPriority())
4097 return false;
4098
4099 // Assume the following shader types will never have exports,
4100 // and avoid adding or adjusting S_SETPRIO.
4101 MachineBasicBlock *MBB = MI->getParent();
4102 MachineFunction *MF = MBB->getParent();
4103 auto CC = MF->getFunction().getCallingConv();
4104 switch (CC) {
4105 case CallingConv::AMDGPU_CS:
4106 case CallingConv::AMDGPU_CS_Chain:
4107 case CallingConv::AMDGPU_CS_ChainPreserve:
4108 case CallingConv::AMDGPU_KERNEL:
4109 return false;
4110 default:
4111 break;
4112 }
4113
4114 const int MaxPriority = 3;
4115 const int NormalPriority = 2;
4116 const int PostExportPriority = 0;
4117
4118 auto It = MI->getIterator();
4119 switch (MI->getOpcode()) {
4120 case AMDGPU::S_ENDPGM:
4121 case AMDGPU::S_ENDPGM_SAVED:
4122 case AMDGPU::S_ENDPGM_ORDERED_PS_DONE:
4123 case AMDGPU::SI_RETURN_TO_EPILOG:
4124 // Ensure shader with calls raises priority at entry.
4125 // This ensures correct priority if exports exist in callee.
4126 if (MF->getFrameInfo().hasCalls())
4127 return ensureEntrySetPrio(MF, Priority: NormalPriority, TII);
4128 return false;
4129 case AMDGPU::S_SETPRIO: {
4130 // Raise minimum priority unless in workaround.
4131 auto &PrioOp = MI->getOperand(i: 0);
4132 int Prio = PrioOp.getImm();
4133 bool InWA = (Prio == PostExportPriority) &&
4134 (It != MBB->begin() && TII.isEXP(MI: *std::prev(x: It)));
4135 if (InWA || Prio >= NormalPriority)
4136 return false;
4137 PrioOp.setImm(std::min(a: Prio + NormalPriority, b: MaxPriority));
4138 return true;
4139 }
4140 default:
4141 if (!TII.isEXP(MI: *MI))
4142 return false;
4143 break;
4144 }
4145
4146 // Check entry priority at each export (as there will only be a few).
4147 // Note: amdgpu_gfx can only be a callee, so defer to caller setprio.
4148 bool Changed = false;
4149 if (CC != CallingConv::AMDGPU_Gfx && CC != CallingConv::AMDGPU_Gfx_WholeWave)
4150 Changed = ensureEntrySetPrio(MF, Priority: NormalPriority, TII);
4151
4152 auto NextMI = std::next(x: It);
4153 bool EndOfShader = false;
4154 if (NextMI != MBB->end()) {
4155 // Only need WA at end of sequence of exports.
4156 if (TII.isEXP(MI: *NextMI))
4157 return Changed;
4158 // Assume appropriate S_SETPRIO after export means WA already applied.
4159 if (NextMI->getOpcode() == AMDGPU::S_SETPRIO &&
4160 NextMI->getOperand(i: 0).getImm() == PostExportPriority)
4161 return Changed;
4162 EndOfShader = NextMI->getOpcode() == AMDGPU::S_ENDPGM;
4163 }
4164
4165 const DebugLoc &DL = MI->getDebugLoc();
4166
4167 // Lower priority.
4168 BuildMI(BB&: *MBB, I: NextMI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_SETPRIO))
4169 .addImm(Val: PostExportPriority);
4170
4171 if (!EndOfShader) {
4172 // Wait for exports to complete.
4173 BuildMI(BB&: *MBB, I: NextMI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_EXPCNT))
4174 .addReg(RegNo: AMDGPU::SGPR_NULL)
4175 .addImm(Val: 0);
4176 }
4177
4178 BuildMI(BB&: *MBB, I: NextMI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_NOP)).addImm(Val: 0);
4179 BuildMI(BB&: *MBB, I: NextMI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_NOP)).addImm(Val: 0);
4180
4181 if (!EndOfShader) {
4182 // Return to normal (higher) priority.
4183 BuildMI(BB&: *MBB, I: NextMI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_SETPRIO))
4184 .addImm(Val: NormalPriority);
4185 }
4186
4187 return true;
4188}
4189
4190bool GCNHazardRecognizer::fixGetRegWaitIdle(MachineInstr *MI) {
4191 if (!isSGetReg(Opcode: MI->getOpcode()))
4192 return false;
4193
4194 const SIInstrInfo *TII = ST.getInstrInfo();
4195 switch (getHWReg(TII, RegInstr: *MI)) {
4196 default:
4197 return false;
4198 case AMDGPU::Hwreg::ID_STATUS:
4199 case AMDGPU::Hwreg::ID_STATE_PRIV:
4200 case AMDGPU::Hwreg::ID_EXCP_FLAG_PRIV:
4201 case AMDGPU::Hwreg::ID_EXCP_FLAG_USER:
4202 break;
4203 }
4204
4205 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
4206 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4207 .addImm(Val: 0);
4208 return true;
4209}
4210
4211bool GCNHazardRecognizer::fixDsAtomicAsyncBarrierArriveB64(MachineInstr *MI) {
4212 if (MI->getOpcode() != AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
4213 return false;
4214
4215 const SIInstrInfo *TII = ST.getInstrInfo();
4216 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
4217 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4218 .addImm(Val: AMDGPU::DepCtr::encodeFieldVmVsrc(VmVsrc: 0, STI: ST));
4219 BuildMI(BB&: *MI->getParent(), I: std::next(x: MI->getIterator()), MIMD: MI->getDebugLoc(),
4220 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4221 .addImm(Val: AMDGPU::DepCtr::encodeFieldVmVsrc(VmVsrc: 0, STI: ST));
4222
4223 return true;
4224}
4225
4226bool GCNHazardRecognizer::fixScratchBaseForwardingHazard(MachineInstr *MI) {
4227 // No reason to check this in pre-RA scheduling, SGPRs have to be allocated
4228 // for hazard to trigger.
4229 if (!isHazardRecognizerMode())
4230 return false;
4231
4232 const SIRegisterInfo *TRI = ST.getRegisterInfo();
4233 const SIInstrInfo *TII = ST.getInstrInfo();
4234 // Hazard expires after 10 SGPR writes by SALU or 8 SGPR writes by VALU.
4235 const int FlatScrBaseWaitStates = 10;
4236
4237 bool ReadsFlatScrLo =
4238 MI->readsRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_LO, TRI);
4239 bool ReadsFlatScrHi =
4240 MI->readsRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_HI, TRI);
4241 if (isSGetReg(Opcode: MI->getOpcode())) {
4242 switch (getHWReg(TII, RegInstr: *MI)) {
4243 default:
4244 break;
4245 case AMDGPU::Hwreg::ID_FLAT_SCR_LO:
4246 ReadsFlatScrLo = true;
4247 break;
4248 case AMDGPU::Hwreg::ID_FLAT_SCR_HI:
4249 ReadsFlatScrHi = true;
4250 break;
4251 }
4252 }
4253
4254 const MachineRegisterInfo &MRI = MF.getRegInfo();
4255
4256 auto IsRegDefHazard = [&](Register Reg) -> bool {
4257 DenseSet<const MachineBasicBlock *> Visited;
4258 auto IsHazardFn = [TRI, Reg](const MachineInstr &MI) {
4259 return MI.modifiesRegister(Reg, TRI);
4260 };
4261
4262 // This literally abuses the idea of waitstates. Instead of waitstates it
4263 // returns 1 for SGPR written and 0 otherwise.
4264 auto IsSGPRDef = [TII, TRI, &MRI](const MachineInstr &MI) -> unsigned {
4265 if (!TII->isSALU(MI) && !TII->isVALU(MI, /*AllowLDSDMA=*/true))
4266 return 0;
4267 for (const MachineOperand &MO : MI.all_defs()) {
4268 if (TRI->isSGPRReg(MRI, Reg: MO.getReg()))
4269 return 1;
4270 }
4271 return 0;
4272 };
4273
4274 auto IsExpiredFn = [=](const MachineInstr &MI, int SgprWrites) {
4275 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR) {
4276 unsigned Wait = MI.getOperand(i: 0).getImm();
4277 if (AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: Wait) == 0 &&
4278 AMDGPU::DepCtr::decodeFieldVaSdst(Encoded: Wait) == 0)
4279 return true;
4280 }
4281 return SgprWrites >= FlatScrBaseWaitStates;
4282 };
4283
4284 return ::getWaitStatesSince(
4285 IsHazard: IsHazardFn, MBB: MI->getParent(), I: std::next(x: MI->getReverseIterator()),
4286 WaitStates: 0, IsExpired: IsExpiredFn, Visited, GetNumWaitStates: IsSGPRDef) < FlatScrBaseWaitStates;
4287 };
4288
4289 if ((!ReadsFlatScrLo || MRI.isConstantPhysReg(PhysReg: AMDGPU::SGPR102) ||
4290 !IsRegDefHazard(AMDGPU::SGPR102)) &&
4291 (!ReadsFlatScrHi || MRI.isConstantPhysReg(PhysReg: AMDGPU::SGPR103) ||
4292 !IsRegDefHazard(AMDGPU::SGPR103)))
4293 return false;
4294
4295 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
4296 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
4297 .addImm(Val: AMDGPU::DepCtr::encodeFieldVaSdst(
4298 Encoded: AMDGPU::DepCtr::encodeFieldSaSdst(SaSdst: 0, STI: ST), VaSdst: 0));
4299 return true;
4300}
4301
4302bool GCNHazardRecognizer::fixSetRegMode(MachineInstr *MI) {
4303 if (!isSSetReg(Opcode: MI->getOpcode()) ||
4304 MI->getOperand(i: 1).getImm() != AMDGPU::Hwreg::ID_MODE)
4305 return false;
4306
4307 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_NOP_e32));
4308 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_NOP_e32));
4309 return true;
4310}
4311
4312bool GCNHazardRecognizer::fixTDM(MachineInstr *MI) {
4313 auto IsTDM = [&](const MachineInstr &MI) -> bool {
4314 return SIInstrInfo::usesTENSOR_CNT(MI) &&
4315 MI.getOpcode() != AMDGPU::S_WAIT_TENSORCNT;
4316 };
4317
4318 if (!IsTDM(*MI))
4319 return false;
4320
4321 auto IsExpiredFn = [](const MachineInstr &MI, int) {
4322 if (MI.getOpcode() != AMDGPU::S_WAIT_TENSORCNT)
4323 return false;
4324 return MI.getOperand(i: 0).getImm() <= 10;
4325 };
4326
4327 if (::getWaitStatesSince(IsHazard: IsTDM, MI, IsExpired: IsExpiredFn) ==
4328 std::numeric_limits<int>::max())
4329 return false;
4330
4331 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
4332 MCID: TII.get(Opcode: AMDGPU::S_WAIT_TENSORCNT))
4333 .addImm(Val: 10);
4334 return true;
4335}
4336