1//===-- GCNHazardRecognizers.h - GCN Hazard Recognizers ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines hazard recognizers for scheduling on GCN processors.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H
14#define LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H
15
16#include "llvm/ADT/BitVector.h"
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/CodeGen/ScheduleHazardRecognizer.h"
19#include "llvm/CodeGen/TargetSchedule.h"
20#include <list>
21
22namespace llvm {
23
24class MachineFunction;
25class MachineInstr;
26class MachineOperand;
27class MachineRegisterInfo;
28class SIInstrInfo;
29class SIRegisterInfo;
30class GCNSubtarget;
31
32class GCNHazardRecognizer final : public ScheduleHazardRecognizer {
33public:
34 typedef function_ref<bool(const MachineInstr &)> IsHazardFn;
35
36private:
37 // Distinguish if we are called from scheduler or hazard recognizer
38 bool IsHazardRecognizerMode;
39
40 // This variable stores the instruction that has been emitted this cycle. It
41 // will be added to EmittedInstrs, when AdvanceCycle() or RecedeCycle() is
42 // called.
43 MachineInstr *CurrCycleInstr;
44 std::list<MachineInstr*> EmittedInstrs;
45 const MachineFunction &MF;
46 const GCNSubtarget &ST;
47 const SIInstrInfo &TII;
48 const SIRegisterInfo &TRI;
49 const TargetSchedModel &TSchedModel;
50 bool RunLdsBranchVmemWARHazardFixup;
51
52 /// RegUnits of uses in the current soft memory clause.
53 BitVector ClauseUses;
54
55 /// RegUnits of defs in the current soft memory clause.
56 BitVector ClauseDefs;
57
58 void resetClause() {
59 ClauseUses.reset();
60 ClauseDefs.reset();
61 }
62
63 void addClauseInst(const MachineInstr &MI);
64
65 /// \returns the number of wait states before another MFMA instruction can be
66 /// issued after \p MI.
67 unsigned getMFMAPipelineWaitStates(const MachineInstr &MI) const;
68
69 // Advance over a MachineInstr bundle. Look for hazards in the bundled
70 // instructions.
71 void processBundle();
72
73 // Run on an individual instruction in hazard recognizer mode. This can be
74 // used on a newly inserted instruction before returning from PreEmitNoops.
75 void runOnInstruction(MachineInstr *MI);
76
77 int getWaitStatesSince(IsHazardFn IsHazard, int Limit);
78 int getWaitStatesSinceDef(unsigned Reg, IsHazardFn IsHazardDef, int Limit);
79 int getWaitStatesSinceSetReg(IsHazardFn IsHazard, int Limit);
80
81 int checkSoftClauseHazards(MachineInstr *SMEM);
82 int checkSMRDHazards(MachineInstr *SMRD);
83 int checkVMEMHazards(MachineInstr* VMEM);
84 int checkDPPHazards(MachineInstr *DPP);
85 int checkDivFMasHazards(MachineInstr *DivFMas);
86 int checkGetRegHazards(MachineInstr *GetRegInstr);
87 int checkSetRegHazards(MachineInstr *SetRegInstr);
88 int createsVALUHazard(const MachineInstr &MI);
89 int checkVALUHazards(MachineInstr *VALU);
90 int checkVALUHazardsHelper(const MachineOperand &Def, const MachineRegisterInfo &MRI);
91 int checkRWLaneHazards(MachineInstr *RWLane);
92 int checkRFEHazards(MachineInstr *RFE);
93 int checkInlineAsmHazards(MachineInstr *IA);
94 int checkReadM0Hazards(MachineInstr *SMovRel);
95 int checkNSAtoVMEMHazard(MachineInstr *MI);
96 int checkFPAtomicToDenormModeHazard(MachineInstr *MI);
97 void fixHazards(MachineInstr *MI);
98 bool fixVcmpxPermlaneHazards(MachineInstr *MI);
99 bool fixVMEMtoScalarWriteHazards(MachineInstr *MI);
100 bool fixSMEMtoVectorWriteHazards(MachineInstr *MI);
101 bool fixVcmpxExecWARHazard(MachineInstr *MI);
102 bool fixLdsBranchVmemWARHazard(MachineInstr *MI);
103 bool fixLdsDirectVALUHazard(MachineInstr *MI);
104 bool fixLdsDirectVMEMHazard(MachineInstr *MI);
105 bool fixVALUPartialForwardingHazard(MachineInstr *MI);
106 bool fixVALUTransUseHazard(MachineInstr *MI);
107 bool fixWMMAHazards(MachineInstr *MI);
108 bool fixShift64HighRegBug(MachineInstr *MI);
109 bool fixVALUMaskWriteHazard(MachineInstr *MI);
110 bool fixRequiredExportPriority(MachineInstr *MI);
111
112 int checkMAIHazards(MachineInstr *MI);
113 int checkMAIHazards908(MachineInstr *MI);
114 int checkMAIHazards90A(MachineInstr *MI);
115 /// Pad the latency between neighboring MFMA instructions with s_nops. The
116 /// percentage of wait states to fill with s_nops is specified by the command
117 /// line option '-amdgpu-mfma-padding-ratio'.
118 ///
119 /// For example, with '-amdgpu-mfma-padding-ratio=100':
120 ///
121 /// 2 pass MFMA instructions have a latency of 2 wait states. Therefore, a
122 /// 'S_NOP 1' will be added between sequential MFMA instructions.
123 ///
124 /// V_MFMA_F32_4X4X1F32
125 /// V_MFMA_F32_4X4X1F32
126 ///-->
127 /// V_MFMA_F32_4X4X1F32
128 /// S_NOP 1
129 /// V_MFMA_F32_4X4X1F32
130 int checkMFMAPadding(MachineInstr *MI);
131 int checkMAIVALUHazards(MachineInstr *MI);
132 int checkMAILdStHazards(MachineInstr *MI);
133 int checkPermlaneHazards(MachineInstr *MI);
134
135public:
136 GCNHazardRecognizer(const MachineFunction &MF);
137 // We can only issue one instruction per cycle.
138 bool atIssueLimit() const override { return true; }
139 void EmitInstruction(SUnit *SU) override;
140 void EmitInstruction(MachineInstr *MI) override;
141 HazardType getHazardType(SUnit *SU, int Stalls) override;
142 void EmitNoop() override;
143 unsigned PreEmitNoops(MachineInstr *) override;
144 unsigned PreEmitNoopsCommon(MachineInstr *);
145 void AdvanceCycle() override;
146 void RecedeCycle() override;
147 bool ShouldPreferAnother(SUnit *SU) override;
148 void Reset() override;
149};
150
151} // end namespace llvm
152
153#endif //LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H
154