1//===-- ARMHazardRecognizer.cpp - ARM postra hazard recognizer ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ARMHazardRecognizer.h"
10#include "ARMBaseInstrInfo.h"
11#include "ARMSubtarget.h"
12#include "llvm/Analysis/ValueTracking.h"
13#include "llvm/CodeGen/MachineFrameInfo.h"
14#include "llvm/CodeGen/MachineInstr.h"
15#include "llvm/CodeGen/ScheduleDAG.h"
16#include "llvm/CodeGen/TargetRegisterInfo.h"
17#include "llvm/Support/CommandLine.h"
18
19using namespace llvm;
20
21static cl::opt<int> DataBankMask("arm-data-bank-mask", cl::init(Val: -1),
22 cl::Hidden);
23static cl::opt<bool> AssumeITCMConflict("arm-assume-itcm-bankconflict",
24 cl::init(Val: false), cl::Hidden);
25
26static bool hasRAWHazard(MachineInstr *DefMI, MachineInstr *MI,
27 const TargetRegisterInfo &TRI) {
28 // FIXME: Detect integer instructions properly.
29 const MCInstrDesc &MCID = MI->getDesc();
30 unsigned Domain = MCID.TSFlags & ARMII::DomainMask;
31 if (MI->mayStore())
32 return false;
33 unsigned Opcode = MCID.getOpcode();
34 if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
35 return false;
36 if ((Domain & ARMII::DomainVFP) || (Domain & ARMII::DomainNEON))
37 return MI->readsRegister(Reg: DefMI->getOperand(i: 0).getReg(), TRI: &TRI);
38 return false;
39}
40
41ScheduleHazardRecognizer::HazardType
42ARMHazardRecognizerFPMLx::getHazardType(SUnit *SU, int Stalls) {
43 assert(Stalls == 0 && "ARM hazards don't support scoreboard lookahead");
44
45 MachineInstr *MI = SU->getInstr();
46
47 if (!MI->isDebugInstr()) {
48 // Look for special VMLA / VMLS hazards. A VMUL / VADD / VSUB following
49 // a VMLA / VMLS will cause 4 cycle stall.
50 const MCInstrDesc &MCID = MI->getDesc();
51 if (LastMI && (MCID.TSFlags & ARMII::DomainMask) != ARMII::DomainGeneral) {
52 MachineInstr *DefMI = LastMI;
53 const MCInstrDesc &LastMCID = LastMI->getDesc();
54 const MachineFunction *MF = MI->getParent()->getParent();
55 const ARMBaseInstrInfo &TII = *static_cast<const ARMBaseInstrInfo *>(
56 MF->getSubtarget().getInstrInfo());
57
58 // Skip over one non-VFP / NEON instruction.
59 if (!LastMI->isBarrier() &&
60 !(TII.getSubtarget().hasMuxedUnits() && LastMI->mayLoadOrStore()) &&
61 (LastMCID.TSFlags & ARMII::DomainMask) == ARMII::DomainGeneral) {
62 MachineBasicBlock::iterator I = LastMI;
63 if (I != LastMI->getParent()->begin()) {
64 I = std::prev(x: I);
65 DefMI = &*I;
66 }
67 }
68
69 if (TII.isFpMLxInstruction(Opcode: DefMI->getOpcode()) &&
70 (TII.canCauseFpMLxStall(Opcode: MI->getOpcode()) ||
71 hasRAWHazard(DefMI, MI, TRI: TII.getRegisterInfo()))) {
72 // Try to schedule another instruction for the next 4 cycles.
73 if (FpMLxStalls == 0)
74 FpMLxStalls = 4;
75 return Hazard;
76 }
77 }
78 }
79 return NoHazard;
80}
81
82void ARMHazardRecognizerFPMLx::Reset() {
83 LastMI = nullptr;
84 FpMLxStalls = 0;
85}
86
87void ARMHazardRecognizerFPMLx::EmitInstruction(SUnit *SU) {
88 MachineInstr *MI = SU->getInstr();
89 if (!MI->isDebugInstr()) {
90 LastMI = MI;
91 FpMLxStalls = 0;
92 }
93}
94
95void ARMHazardRecognizerFPMLx::AdvanceCycle() {
96 if (FpMLxStalls && --FpMLxStalls == 0)
97 // Stalled for 4 cycles but still can't schedule any other instructions.
98 LastMI = nullptr;
99}
100
101void ARMHazardRecognizerFPMLx::RecedeCycle() {
102 llvm_unreachable("reverse ARM hazard checking unsupported");
103}
104
105///////// Bank conflicts handled as hazards //////////////
106
107static bool getBaseOffset(const MachineInstr &MI, const MachineOperand *&BaseOp,
108 int64_t &Offset) {
109
110 uint64_t TSFlags = MI.getDesc().TSFlags;
111 unsigned AddrMode = (TSFlags & ARMII::AddrModeMask);
112 unsigned IndexMode =
113 (TSFlags & ARMII::IndexModeMask) >> ARMII::IndexModeShift;
114
115 // Address mode tells us what we want to know about operands for T2
116 // instructions (but not size). It tells us size (but not about operands)
117 // for T1 instructions.
118 switch (AddrMode) {
119 default:
120 return false;
121 case ARMII::AddrModeT2_i8:
122 // t2LDRBT, t2LDRB_POST, t2LDRB_PRE, t2LDRBi8,
123 // t2LDRHT, t2LDRH_POST, t2LDRH_PRE, t2LDRHi8,
124 // t2LDRSBT, t2LDRSB_POST, t2LDRSB_PRE, t2LDRSBi8,
125 // t2LDRSHT, t2LDRSH_POST, t2LDRSH_PRE, t2LDRSHi8,
126 // t2LDRT, t2LDR_POST, t2LDR_PRE, t2LDRi8
127 BaseOp = &MI.getOperand(i: 1);
128 Offset = (IndexMode == ARMII::IndexModePost)
129 ? 0
130 : (IndexMode == ARMII::IndexModePre ||
131 IndexMode == ARMII::IndexModeUpd)
132 ? MI.getOperand(i: 3).getImm()
133 : MI.getOperand(i: 2).getImm();
134 return true;
135 case ARMII::AddrModeT2_i12:
136 // t2LDRBi12, t2LDRHi12
137 // t2LDRSBi12, t2LDRSHi12
138 // t2LDRi12
139 BaseOp = &MI.getOperand(i: 1);
140 Offset = MI.getOperand(i: 2).getImm();
141 return true;
142 case ARMII::AddrModeT2_i8s4:
143 // t2LDRD_POST, t2LDRD_PRE, t2LDRDi8
144 BaseOp = &MI.getOperand(i: 2);
145 Offset = (IndexMode == ARMII::IndexModePost)
146 ? 0
147 : (IndexMode == ARMII::IndexModePre ||
148 IndexMode == ARMII::IndexModeUpd)
149 ? MI.getOperand(i: 4).getImm()
150 : MI.getOperand(i: 3).getImm();
151 return true;
152 case ARMII::AddrModeT1_1:
153 // tLDRBi, tLDRBr (watch out!), TLDRSB
154 case ARMII::AddrModeT1_2:
155 // tLDRHi, tLDRHr (watch out!), TLDRSH
156 case ARMII::AddrModeT1_4:
157 // tLDRi, tLDRr (watch out!)
158 BaseOp = &MI.getOperand(i: 1);
159 Offset = MI.getOperand(i: 2).isImm() ? MI.getOperand(i: 2).getImm() : 0;
160 return MI.getOperand(i: 2).isImm();
161 }
162 return false;
163}
164
165ARMBankConflictHazardRecognizer::ARMBankConflictHazardRecognizer(
166 const ScheduleDAG *DAG, int64_t CPUBankMask, bool CPUAssumeITCMConflict)
167 : MF(DAG->MF), DL(DAG->MF.getDataLayout()),
168 DataMask(DataBankMask.getNumOccurrences() ? int64_t(DataBankMask)
169 : CPUBankMask),
170 AssumeITCMBankConflict(AssumeITCMConflict.getNumOccurrences()
171 ? AssumeITCMConflict
172 : CPUAssumeITCMConflict) {
173 MaxLookAhead = 1;
174}
175
176ScheduleHazardRecognizer::HazardType
177ARMBankConflictHazardRecognizer::CheckOffsets(unsigned O0, unsigned O1) {
178 return (((O0 ^ O1) & DataMask) != 0) ? NoHazard : Hazard;
179}
180
181ScheduleHazardRecognizer::HazardType
182ARMBankConflictHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
183 MachineInstr &L0 = *SU->getInstr();
184 if (!L0.mayLoad() || L0.mayStore() || L0.getNumMemOperands() != 1)
185 return NoHazard;
186
187 auto MO0 = *L0.memoperands().begin();
188 auto BaseVal0 = MO0->getValue();
189 auto BasePseudoVal0 = MO0->getPseudoValue();
190 int64_t Offset0 = 0;
191
192 if (!MO0->getSize().hasValue() || MO0->getSize().getValue() > 4)
193 return NoHazard;
194
195 bool SPvalid = false;
196 const MachineOperand *SP = nullptr;
197 int64_t SPOffset0 = 0;
198
199 for (auto L1 : Accesses) {
200 auto MO1 = *L1->memoperands().begin();
201 auto BaseVal1 = MO1->getValue();
202 auto BasePseudoVal1 = MO1->getPseudoValue();
203 int64_t Offset1 = 0;
204
205 // Pointers to the same object
206 if (BaseVal0 && BaseVal1) {
207 const Value *Ptr0, *Ptr1;
208 Ptr0 = GetPointerBaseWithConstantOffset(Ptr: BaseVal0, Offset&: Offset0, DL, AllowNonInbounds: true);
209 Ptr1 = GetPointerBaseWithConstantOffset(Ptr: BaseVal1, Offset&: Offset1, DL, AllowNonInbounds: true);
210 if (Ptr0 == Ptr1 && Ptr0)
211 return CheckOffsets(O0: Offset0, O1: Offset1);
212 }
213
214 if (BasePseudoVal0 && BasePseudoVal1 &&
215 BasePseudoVal0->kind() == BasePseudoVal1->kind() &&
216 BasePseudoVal0->kind() == PseudoSourceValue::FixedStack) {
217 // Spills/fills
218 auto FS0 = cast<FixedStackPseudoSourceValue>(Val: BasePseudoVal0);
219 auto FS1 = cast<FixedStackPseudoSourceValue>(Val: BasePseudoVal1);
220 Offset0 = MF.getFrameInfo().getObjectOffset(ObjectIdx: FS0->getFrameIndex());
221 Offset1 = MF.getFrameInfo().getObjectOffset(ObjectIdx: FS1->getFrameIndex());
222 return CheckOffsets(O0: Offset0, O1: Offset1);
223 }
224
225 // Constant pools (likely in ITCM)
226 if (BasePseudoVal0 && BasePseudoVal1 &&
227 BasePseudoVal0->kind() == BasePseudoVal1->kind() &&
228 BasePseudoVal0->isConstantPool() && AssumeITCMBankConflict)
229 return Hazard;
230
231 // Is this a stack pointer-relative access? We could in general try to
232 // use "is this the same register and is it unchanged?", but the
233 // memory operand tracking is highly likely to have already found that.
234 // What we're after here is bank conflicts between different objects in
235 // the stack frame.
236 if (!SPvalid) { // set up SP
237 if (!getBaseOffset(MI: L0, BaseOp&: SP, Offset&: SPOffset0) || SP->getReg().id() != ARM::SP)
238 SP = nullptr;
239 SPvalid = true;
240 }
241 if (SP) {
242 int64_t SPOffset1;
243 const MachineOperand *SP1;
244 if (getBaseOffset(MI: *L1, BaseOp&: SP1, Offset&: SPOffset1) && SP1->getReg().id() == ARM::SP)
245 return CheckOffsets(O0: SPOffset0, O1: SPOffset1);
246 }
247 }
248
249 return NoHazard;
250}
251
252void ARMBankConflictHazardRecognizer::Reset() { Accesses.clear(); }
253
254void ARMBankConflictHazardRecognizer::EmitInstruction(SUnit *SU) {
255 MachineInstr &MI = *SU->getInstr();
256 if (!MI.mayLoad() || MI.mayStore() || MI.getNumMemOperands() != 1)
257 return;
258
259 auto MO = *MI.memoperands().begin();
260 LocationSize Size1 = MO->getSize();
261 if (Size1.hasValue() && Size1.getValue() > 4)
262 return;
263 Accesses.push_back(Elt: &MI);
264}
265
266void ARMBankConflictHazardRecognizer::AdvanceCycle() { Accesses.clear(); }
267
268void ARMBankConflictHazardRecognizer::RecedeCycle() { Accesses.clear(); }
269