1//===-- AArch64A53Fix835769.cpp -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// This pass changes code to work around Cortex-A53 erratum 835769.
9// It works around it by inserting a nop instruction in code sequences that
10// in some circumstances may trigger the erratum.
11// It inserts a nop instruction between a sequence of the following 2 classes
12// of instructions:
13// instr 1: mem-instr (including loads, stores and prefetches).
14// instr 2: non-SIMD integer multiply-accumulate writing 64-bit X registers.
15//===----------------------------------------------------------------------===//
16
17#include "AArch64.h"
18#include "AArch64Subtarget.h"
19#include "llvm/ADT/Statistic.h"
20#include "llvm/CodeGen/MachineFunction.h"
21#include "llvm/CodeGen/MachineFunctionPass.h"
22#include "llvm/CodeGen/MachineInstr.h"
23#include "llvm/CodeGen/MachineInstrBuilder.h"
24#include "llvm/CodeGen/TargetInstrInfo.h"
25#include "llvm/Support/Debug.h"
26#include "llvm/Support/raw_ostream.h"
27
28using namespace llvm;
29
30#define DEBUG_TYPE "aarch64-fix-cortex-a53-835769"
31
32STATISTIC(NumNopsAdded, "Number of Nops added to work around erratum 835769");
33
34//===----------------------------------------------------------------------===//
35// Helper functions
36
37// Is the instruction a match for the instruction that comes first in the
38// sequence of instructions that can trigger the erratum?
39static bool isFirstInstructionInSequence(MachineInstr *MI) {
40 // Must return true if this instruction is a load, a store or a prefetch.
41 switch (MI->getOpcode()) {
42 case AArch64::PRFMl:
43 case AArch64::PRFMroW:
44 case AArch64::PRFMroX:
45 case AArch64::PRFMui:
46 case AArch64::PRFUMi:
47 return true;
48 default:
49 return MI->mayLoadOrStore();
50 }
51}
52
53// Is the instruction a match for the instruction that comes second in the
54// sequence that can trigger the erratum?
55static bool isSecondInstructionInSequence(MachineInstr *MI) {
56 // Must return true for non-SIMD integer multiply-accumulates, writing
57 // to a 64-bit register.
58 switch (MI->getOpcode()) {
59 // Erratum cannot be triggered when the destination register is 32 bits,
60 // therefore only include the following.
61 case AArch64::MSUBXrrr:
62 case AArch64::MADDXrrr:
63 case AArch64::SMADDLrrr:
64 case AArch64::SMSUBLrrr:
65 case AArch64::UMADDLrrr:
66 case AArch64::UMSUBLrrr:
67 // Erratum can only be triggered by multiply-adds, not by regular
68 // non-accumulating multiplies, i.e. when Ra=XZR='11111'
69 return MI->getOperand(i: 3).getReg() != AArch64::XZR;
70 default:
71 return false;
72 }
73}
74
75
76//===----------------------------------------------------------------------===//
77
78namespace {
79class AArch64A53Fix835769Impl {
80public:
81 bool run(MachineFunction &MF);
82
83private:
84 const TargetInstrInfo *TII;
85 bool runOnBasicBlock(MachineBasicBlock &MBB);
86};
87
88class AArch64A53Fix835769Legacy : public MachineFunctionPass {
89public:
90 static char ID;
91 explicit AArch64A53Fix835769Legacy() : MachineFunctionPass(ID) {}
92
93 bool runOnMachineFunction(MachineFunction &F) override;
94
95 MachineFunctionProperties getRequiredProperties() const override {
96 return MachineFunctionProperties().setNoVRegs();
97 }
98
99 StringRef getPassName() const override {
100 return "Workaround A53 erratum 835769 pass";
101 }
102
103 void getAnalysisUsage(AnalysisUsage &AU) const override {
104 AU.setPreservesCFG();
105 MachineFunctionPass::getAnalysisUsage(AU);
106 }
107};
108char AArch64A53Fix835769Legacy::ID = 0;
109
110} // end anonymous namespace
111
112INITIALIZE_PASS(AArch64A53Fix835769Legacy, "aarch64-fix-cortex-a53-835769-pass",
113 "AArch64 fix for A53 erratum 835769", false, false)
114
115//===----------------------------------------------------------------------===//
116
117PreservedAnalyses
118AArch64A53Fix835769Pass::run(MachineFunction &MF,
119 MachineFunctionAnalysisManager &MFAM) {
120 bool Changed = AArch64A53Fix835769Impl().run(MF);
121 if (!Changed)
122 return PreservedAnalyses::all();
123 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
124 PA.preserveSet<CFGAnalyses>();
125 return PA;
126}
127
128bool AArch64A53Fix835769Legacy::runOnMachineFunction(MachineFunction &F) {
129 return AArch64A53Fix835769Impl().run(MF&: F);
130}
131
132bool AArch64A53Fix835769Impl::run(MachineFunction &MF) {
133 LLVM_DEBUG(dbgs() << "***** AArch64A53Fix835769 *****\n");
134 auto &STI = MF.getSubtarget<AArch64Subtarget>();
135 // Fix not requested, skip pass.
136 if (!STI.fixCortexA53_835769())
137 return false;
138
139 bool Changed = false;
140 TII = STI.getInstrInfo();
141
142 for (auto &MBB : MF) {
143 Changed |= runOnBasicBlock(MBB);
144 }
145 return Changed;
146}
147
148// Return the block that was fallen through to get to MBB, if any,
149// otherwise nullptr.
150static MachineBasicBlock *getBBFallenThrough(MachineBasicBlock *MBB,
151 const TargetInstrInfo *TII) {
152 // Get the previous machine basic block in the function.
153 MachineFunction::iterator MBBI(MBB);
154
155 // Can't go off top of function.
156 if (MBBI == MBB->getParent()->begin())
157 return nullptr;
158
159 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
160 SmallVector<MachineOperand, 2> Cond;
161
162 MachineBasicBlock *PrevBB = &*std::prev(x: MBBI);
163 for (MachineBasicBlock *S : MBB->predecessors())
164 if (S == PrevBB && !TII->analyzeBranch(MBB&: *PrevBB, TBB, FBB, Cond) && !TBB &&
165 !FBB)
166 return S;
167
168 return nullptr;
169}
170
171// Iterate through fallen through blocks trying to find a previous non-pseudo if
172// there is one, otherwise return nullptr. Only look for instructions in
173// previous blocks, not the current block, since we only use this to look at
174// previous blocks.
175static MachineInstr *getLastNonPseudo(MachineBasicBlock &MBB,
176 const TargetInstrInfo *TII) {
177 MachineBasicBlock *FMBB = &MBB;
178
179 // If there is no non-pseudo in the current block, loop back around and try
180 // the previous block (if there is one).
181 while ((FMBB = getBBFallenThrough(MBB: FMBB, TII))) {
182 for (MachineInstr &I : llvm::reverse(C&: *FMBB))
183 if (!I.isPseudo())
184 return &I;
185 }
186
187 // There was no previous non-pseudo in the fallen through blocks
188 return nullptr;
189}
190
191static void insertNopBeforeInstruction(MachineBasicBlock &MBB, MachineInstr* MI,
192 const TargetInstrInfo *TII) {
193 // If we are the first instruction of the block, put the NOP at the end of
194 // the previous fallthrough block
195 if (MI == &MBB.front()) {
196 MachineInstr *I = getLastNonPseudo(MBB, TII);
197 assert(I && "Expected instruction");
198 DebugLoc DL = I->getDebugLoc();
199 BuildMI(BB: I->getParent(), MIMD: DL, MCID: TII->get(Opcode: AArch64::NOP));
200 } else {
201 DebugLoc DL = MI->getDebugLoc();
202 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AArch64::NOP));
203 }
204
205 ++NumNopsAdded;
206}
207
208bool AArch64A53Fix835769Impl::runOnBasicBlock(MachineBasicBlock &MBB) {
209 bool Changed = false;
210 LLVM_DEBUG(dbgs() << "Running on MBB: " << MBB
211 << " - scanning instructions...\n");
212
213 // First, scan the basic block, looking for a sequence of 2 instructions
214 // that match the conditions under which the erratum may trigger.
215
216 // List of terminating instructions in matching sequences
217 std::vector<MachineInstr*> Sequences;
218 unsigned Idx = 0;
219 MachineInstr *PrevInstr = nullptr;
220
221 // Try and find the last non-pseudo instruction in any fallen through blocks,
222 // if there isn't one, then we use nullptr to represent that.
223 PrevInstr = getLastNonPseudo(MBB, TII);
224
225 for (auto &MI : MBB) {
226 MachineInstr *CurrInstr = &MI;
227 LLVM_DEBUG(dbgs() << " Examining: " << MI);
228 if (PrevInstr) {
229 LLVM_DEBUG(dbgs() << " PrevInstr: " << *PrevInstr
230 << " CurrInstr: " << *CurrInstr
231 << " isFirstInstructionInSequence(PrevInstr): "
232 << isFirstInstructionInSequence(PrevInstr) << "\n"
233 << " isSecondInstructionInSequence(CurrInstr): "
234 << isSecondInstructionInSequence(CurrInstr) << "\n");
235 if (isFirstInstructionInSequence(MI: PrevInstr) &&
236 isSecondInstructionInSequence(MI: CurrInstr)) {
237 LLVM_DEBUG(dbgs() << " ** pattern found at Idx " << Idx << "!\n");
238 (void) Idx;
239 Sequences.push_back(x: CurrInstr);
240 }
241 }
242 if (!CurrInstr->isPseudo())
243 PrevInstr = CurrInstr;
244 ++Idx;
245 }
246
247 LLVM_DEBUG(dbgs() << "Scan complete, " << Sequences.size()
248 << " occurrences of pattern found.\n");
249
250 // Then update the basic block, inserting nops between the detected sequences.
251 for (auto &MI : Sequences) {
252 Changed = true;
253 insertNopBeforeInstruction(MBB, MI, TII);
254 }
255
256 return Changed;
257}
258
259// Factory function used by AArch64TargetMachine to add the pass to
260// the passmanager.
261FunctionPass *llvm::createAArch64A53Fix835769LegacyPass() {
262 return new AArch64A53Fix835769Legacy();
263}
264