1//===-- GCNHazardRecognizers.h - GCN Hazard Recognizers ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines hazard recognizers for scheduling on GCN processors.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H
14#define LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H
15
16#include "AMDGPUCoExecInfo.h"
17#include "llvm/ADT/BitVector.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/CodeGen/MachineLoopInfo.h"
20#include "llvm/CodeGen/ScheduleHazardRecognizer.h"
21#include "llvm/CodeGen/TargetSchedule.h"
22#include <array>
23#include <list>
24#include <optional>
25
26namespace llvm {
27
28class MachineFunction;
29class MachineInstr;
30class MachineOperand;
31class MachineRegisterInfo;
32class SIInstrInfo;
33class SIRegisterInfo;
34class GCNSubtarget;
35
36class GCNHazardRecognizer final : public ScheduleHazardRecognizer {
37public:
38 typedef function_ref<bool(const MachineInstr &)> IsHazardFn;
39 typedef function_ref<bool(const MachineInstr &, int WaitStates)> IsExpiredFn;
40 typedef function_ref<unsigned int(const MachineInstr &)> GetNumWaitStatesFn;
41
42 /// Operating mode for the hazard recognizer. Two independent properties
43 /// follow from it.
44 ///
45 /// The scheduler modes (PreRA, PostRA) report a hazard so that the scheduler
46 /// can pick some other instruction, and to do that they track pipeline state
47 /// across the cycles they are told about. HazardRecognizerMode inserts s_nop
48 /// or v_nop to ensure correctness of the generated code.
49 ///
50 /// Operands are physical registers in every mode but PreRA, so hazards
51 /// defined by register dependences are checked only when hasPhysRegs()
52 /// holds.
53 enum class OperatingMode { PreRA, PostRA, HazardRecognizerMode };
54
55private:
56 // Operating mode determines which hazards are checked.
57 OperatingMode Mode;
58
59 // This variable stores the instruction that has been emitted this cycle. It
60 // will be added to EmittedInstrs, when AdvanceCycle() or RecedeCycle() is
61 // called.
62 MachineInstr *CurrCycleInstr;
63 std::list<MachineInstr*> EmittedInstrs;
64
65 // WMMA co-execution hazards are only resolved by VALU-pipe activity (VALU
66 // ops or V_NOPs), never by S_NOPs, so track those instructions separately
67 // from EmittedInstrs.
68 std::list<MachineInstr *> EmittedVALUInstrs;
69 // Lookahead bound for EmittedVALUInstrs. It must be at least as large as the
70 // largest WMMA co-execution wait-state requirement (the maximum value in
71 // WMMAWaitStates/VALUWaitStates in checkWMMACoexecutionHazards). Increase
72 // this if those wait-state limits grow.
73 static constexpr unsigned MaxVALULookAhead = 18;
74 // When true, an unresolved WMMA co-execution hazard is pending, so stall
75 // cycles are optimistically recorded in EmittedVALUInstrs (they will become
76 // V_NOPs unless a non-VALU instruction is scheduled into them).
77 bool HasPendingWMMACoexecHazard = false;
78
79 const MachineFunction &MF;
80 const GCNSubtarget &ST;
81 const SIInstrInfo &TII;
82 const SIRegisterInfo &TRI;
83 const TargetSchedModel &TSchedModel;
84
85 // Loop info for V_NOP hoisting, passed from the pass manager.
86 MachineLoopInfo *MLI = nullptr;
87
88 bool RunLdsBranchVmemWARHazardFixup;
89
90 //===--------------------------------------------------------------------===//
91 // WMMA Co-execution Window State
92 //===--------------------------------------------------------------------===//
93
94 /// Active WMMA co-execution info (slot masks, preferences).
95 AMDGPU::CoExecInfo ActiveCoExecInfo;
96
97 /// Current stage within the WMMA co-execution window (0-based).
98 /// nullopt when not in a WMMA window.
99 std::optional<unsigned> CurrentCoExecStage;
100
101 /// Cycle when the current WMMA window started.
102 unsigned CoExecWindowStartCycle = 0;
103
104 /// Tracks cycles until TRANS can be issued again (back-to-back TRANS hazard).
105 unsigned CyclesUntilTRANS = 0;
106
107 /// Tracks cycles until next VALU after multi-cycle VALU (CVT hazard).
108 unsigned CyclesUntilVALU = 0;
109
110 /// Debug: log of what was scheduled at each stage of the co-exec window.
111 /// '.' = not yet reached, '-' = stall, else CoExecMask short char.
112 std::array<char, AMDGPU::MaxCoExecStages> CoExecWindowLog;
113
114 /// Debug: print the co-exec window visual summary.
115 void dumpCoExecWindow() const;
116
117 /// Returns true if the co-execution window model applies to this subtarget.
118 bool hasCoExecWindowModel() const;
119
120 /// Check WMMA co-execution slot hazard.
121 /// Returns stall cycles needed before MI can be issued in the current slot.
122 unsigned checkWMMACoexecSlot(const MachineInstr &MI) const;
123
124 /// Check TRANS-after-TRANS hazard. Returns stall cycles if MI is TRANS
125 /// or multi-cycle VALU and a previous TRANS shadow is still active.
126 unsigned checkTRANSHazard(const MachineInstr &MI) const;
127
128 /// Check multi-cycle VALU hazard. Returns stall cycles if MI is a VALU
129 /// that would conflict with an active multi-cycle VALU pipeline.
130 unsigned checkMultiCycleVALUHazard(const MachineInstr &MI) const;
131
132 /// Check if we have both a TRANS and WMMA window active. If so, for VALU
133 /// instructions, return the number of stall cycles until one shadow clears.
134 unsigned checkMultiShadowHazard(const MachineInstr &MI) const;
135
136 /// Update WMMA window state when a WMMA instruction is emitted.
137 void updateWMMAWindowState(const MachineInstr &MI);
138
139 /// Update TRANS state when an instruction is emitted.
140 void updateTRANSState(const MachineInstr &MI);
141
142 /// Update multi-cycle VALU state when an instruction is emitted.
143 void updateMultiCycleVALUState(const MachineInstr &MI);
144
145 /// Scheduler-mode part of EmitInstruction().
146 void schedulerEmitInstruction(MachineInstr *MI);
147
148 /// Scheduler-mode part of AdvanceCycle().
149 void schedulerAdvanceCycle();
150
151 /// Scheduler-mode part of Reset().
152 void schedulerReset();
153
154 /// RegUnits of uses in the current soft memory clause.
155 mutable BitVector ClauseUses;
156
157 /// RegUnits of defs in the current soft memory clause.
158 mutable BitVector ClauseDefs;
159
160 void resetClause() const {
161 ClauseUses.reset();
162 ClauseDefs.reset();
163 }
164
165 void addClauseInst(const MachineInstr &MI) const;
166
167 /// \returns the number of wait states before another MFMA instruction can be
168 /// issued after \p MI.
169 unsigned getMFMAPipelineWaitStates(const MachineInstr &MI) const;
170
171 // Advance over a MachineInstr bundle. Look for hazards in the bundled
172 // instructions.
173 void processBundle();
174
175 // Run on an individual instruction in hazard recognizer mode. This can be
176 // used on a newly inserted instruction before returning from PreEmitNoops.
177 void runOnInstruction(MachineInstr *MI);
178
179 int getWaitStatesSince(IsHazardFn IsHazard, int Limit,
180 GetNumWaitStatesFn GetNumWaitStates) const;
181 int getWaitStatesSince(IsHazardFn IsHazard, int Limit) const;
182 int getWaitStatesSinceVALU(IsHazardFn IsHazard, int Limit) const;
183 int getWaitStatesSinceDef(unsigned Reg, IsHazardFn IsHazardDef,
184 int Limit) const;
185 int getWaitStatesSinceSetReg(IsHazardFn IsHazard, int Limit) const;
186
187 int checkSoftClauseHazards(MachineInstr *SMEM) const;
188 int checkSMRDHazards(MachineInstr *SMRD) const;
189 int checkVMEMHazards(MachineInstr *VMEM) const;
190 int checkDPPHazards(MachineInstr *DPP) const;
191 int checkDivFMasHazards(MachineInstr *DivFMas) const;
192 int checkGetRegHazards(MachineInstr *GetRegInstr) const;
193 int checkSetRegHazards(MachineInstr *SetRegInstr) const;
194 int createsVALUHazard(const MachineInstr &MI) const;
195 int checkVALUHazards(MachineInstr *VALU) const;
196 int checkVALUHazardsHelper(const MachineOperand &Def,
197 const MachineRegisterInfo &MRI) const;
198 int checkUniformWindowVALUHazardsHelper(Register Reg) const;
199 int checkSOFFSETWindowVALUHazardsHelper(Register Reg) const;
200 int checkRWLaneHazards(MachineInstr *RWLane) const;
201 int checkRFEHazards(MachineInstr *RFE) const;
202 int checkInlineAsmHazards(MachineInstr *IA) const;
203 int checkReadM0Hazards(MachineInstr *SMovRel) const;
204 int checkNSAtoVMEMHazard(MachineInstr *MI) const;
205 int checkFPAtomicToDenormModeHazard(MachineInstr *MI) const;
206 // Emit \p WaitStatesNeeded V_NOP instructions before \p InsertPt.
207 // If IsHoisting is true, uses empty DebugLoc for compiler-inserted NOPs.
208 void emitVNops(MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt,
209 int WaitStatesNeeded, bool IsHoisting = false);
210 void fixHazards(MachineInstr *MI);
211 bool fixVcmpxPermlaneHazards(MachineInstr *MI);
212 bool fixVMEMtoScalarWriteHazards(MachineInstr *MI);
213 bool fixSMEMtoVectorWriteHazards(MachineInstr *MI);
214 bool fixVcmpxExecWARHazard(MachineInstr *MI);
215 bool fixLdsBranchVmemWARHazard(MachineInstr *MI);
216 bool fixLdsDirectVALUHazard(MachineInstr *MI);
217 bool fixLdsDirectVMEMHazard(MachineInstr *MI);
218 bool fixVALUPartialForwardingHazard(MachineInstr *MI);
219 bool fixVALUTransUseHazard(MachineInstr *MI);
220 bool fixVALUTransCoexecutionHazards(MachineInstr *MI);
221 bool fixWMMAHazards(MachineInstr *MI);
222 int checkWMMACoexecutionHazards(MachineInstr *MI) const;
223 bool fixWMMACoexecutionHazards(MachineInstr *MI);
224 bool tryHoistWMMAVnopsFromLoop(MachineInstr *MI, int WaitStatesNeeded);
225 bool hasWMMAHazardInLoop(MachineLoop *L, MachineInstr *MI,
226 bool IncludeSubloops = true);
227 bool hasWMMAToWMMARegOverlap(const MachineInstr &WMMA,
228 const MachineInstr &MI) const;
229 bool hasWMMAToVALURegOverlap(const MachineInstr &WMMA,
230 const MachineInstr &MI) const;
231 bool isCoexecutionHazardFor(const MachineInstr &I,
232 const MachineInstr &MI) const;
233 bool fixShift64HighRegBug(MachineInstr *MI);
234 bool fixVALUMaskWriteHazard(MachineInstr *MI);
235 bool fixRequiredExportPriority(MachineInstr *MI);
236 bool fixGetRegWaitIdle(MachineInstr *MI);
237 bool fixDsAtomicAsyncBarrierArriveB64(MachineInstr *MI);
238 bool fixScratchBaseForwardingHazard(MachineInstr *MI);
239 bool fixSetRegMode(MachineInstr *MI);
240 bool fixTDM(MachineInstr *MI);
241
242 int checkMAIHazards(MachineInstr *MI) const;
243 int checkMAIHazards908(MachineInstr *MI) const;
244 int checkMAIHazards90A(MachineInstr *MI) const;
245 /// Pad the latency between neighboring MFMA instructions with s_nops. The
246 /// percentage of wait states to fill with s_nops is specified by the command
247 /// line option '-amdgpu-mfma-padding-ratio'.
248 ///
249 /// For example, with '-amdgpu-mfma-padding-ratio=100':
250 ///
251 /// 2 pass MFMA instructions have a latency of 2 wait states. Therefore, a
252 /// 'S_NOP 1' will be added between sequential MFMA instructions.
253 ///
254 /// V_MFMA_F32_4X4X1F32
255 /// V_MFMA_F32_4X4X1F32
256 ///-->
257 /// V_MFMA_F32_4X4X1F32
258 /// S_NOP 1
259 /// V_MFMA_F32_4X4X1F32
260 int checkMFMAPadding(MachineInstr *MI) const;
261 int checkMAIVALUHazards(MachineInstr *MI) const;
262 int checkMAILdStHazards(MachineInstr *MI) const;
263 int checkPermlaneHazards(MachineInstr *MI) const;
264
265public:
266 /// Construct with explicit operating mode.
267 GCNHazardRecognizer(const MachineFunction &MF, OperatingMode Mode,
268 MachineLoopInfo *MLI = nullptr);
269
270 /// Legacy constructor - defaults to PostRA mode.
271 GCNHazardRecognizer(const MachineFunction &MF,
272 MachineLoopInfo *MLI = nullptr);
273
274 ~GCNHazardRecognizer();
275
276 /// Returns the current operating mode.
277 OperatingMode getOperatingMode() const { return Mode; }
278
279 /// Returns true if running in pre-RA scheduling mode.
280 bool isPreRA() const { return Mode == OperatingMode::PreRA; }
281
282 /// Returns true if running in post-RA scheduling mode.
283 bool isPostRA() const { return Mode == OperatingMode::PostRA; }
284
285 /// Returns true if running as a scheduler (pre-RA or post-RA).
286 bool isSchedulerMode() const { return isPreRA() || isPostRA(); }
287
288 /// Returns true if instruction operands are physical registers, so that
289 /// hazards defined by register dependences can be detected.
290 bool hasPhysRegs() const { return !isPreRA(); }
291
292 /// Returns true if running as the standalone hazard recognizer pass.
293 bool isHazardRecognizerMode() const {
294 return Mode == OperatingMode::HazardRecognizerMode;
295 }
296
297 //===--------------------------------------------------------------------===//
298 // Co-execution Window Queries
299 //===--------------------------------------------------------------------===//
300
301 /// Returns true if currently inside a WMMA co-execution window.
302 bool inCoExecWindow() const { return CurrentCoExecStage.has_value(); }
303
304 /// Returns the current stage within the co-execution window, or nullopt.
305 std::optional<unsigned> getCurrentCoExecStage() const {
306 return CurrentCoExecStage;
307 }
308
309 /// Returns the active co-execution info (slot masks, preferences).
310 const AMDGPU::CoExecInfo &getActiveCoExecInfo() const {
311 return ActiveCoExecInfo;
312 }
313
314 /// Get the CoExecMask for a given instruction.
315 static AMDGPU::CoExecMaskT getCoExecMaskForMI(const MachineInstr &MI,
316 const SIInstrInfo &TII);
317 // We can only issue one instruction per cycle.
318 bool atIssueLimit() const override { return true; }
319 void EmitInstruction(SUnit *SU) override;
320 void EmitInstruction(MachineInstr *MI) override;
321 HazardType getHazardType(SUnit *SU, int Stalls) override;
322
323 /// Returns the number of wait states until all hazards for \p MI are
324 /// resolved. This is useful for scheduling heuristics that want
325 /// cycle-accurate hazard information rather than just a boolean. Unlike
326 /// PreEmitNoops, this does not modify state or fix hazards.
327 unsigned getHazardWaitStates(MachineInstr *MI) const;
328 void EmitNoop() override;
329 unsigned PreEmitNoops(MachineInstr *) override;
330 unsigned PreEmitNoopsCommon(MachineInstr *) const;
331 void AdvanceCycle() override;
332 void RecedeCycle() override;
333 bool ShouldPreferAnother(SUnit *SU) const override;
334 void Reset() override;
335};
336
337} // end namespace llvm
338
339#endif //LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H
340