1//===- ARMSLSHardening.cpp - Harden Straight Line Missspeculation ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass to insert code to mitigate against side channel
10// vulnerabilities that may happen under straight line miss-speculation.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ARM.h"
15#include "ARMInstrInfo.h"
16#include "ARMSubtarget.h"
17#include "llvm/CodeGen/IndirectThunks.h"
18#include "llvm/CodeGen/MachineBasicBlock.h"
19#include "llvm/CodeGen/MachineFunction.h"
20#include "llvm/CodeGen/MachineFunctionPass.h"
21#include "llvm/CodeGen/MachineInstr.h"
22#include "llvm/CodeGen/MachineInstrBuilder.h"
23#include "llvm/CodeGen/MachineOperand.h"
24#include "llvm/IR/DebugLoc.h"
25#include <cassert>
26
27using namespace llvm;
28
29#define DEBUG_TYPE "arm-sls-hardening"
30
31#define ARM_SLS_HARDENING_NAME "ARM sls hardening pass"
32
33namespace {
34
35class ARMSLSHardening : public MachineFunctionPass {
36public:
37 const TargetInstrInfo *TII;
38 const ARMSubtarget *ST;
39
40 static char ID;
41
42 ARMSLSHardening() : MachineFunctionPass(ID) {
43 initializeARMSLSHardeningPass(*PassRegistry::getPassRegistry());
44 }
45
46 bool runOnMachineFunction(MachineFunction &Fn) override;
47
48 StringRef getPassName() const override { return ARM_SLS_HARDENING_NAME; }
49
50 void getAnalysisUsage(AnalysisUsage &AU) const override {
51 AU.setPreservesCFG();
52 MachineFunctionPass::getAnalysisUsage(AU);
53 }
54
55private:
56 bool hardenReturnsAndBRs(MachineBasicBlock &MBB) const;
57 bool hardenIndirectCalls(MachineBasicBlock &MBB) const;
58 MachineBasicBlock &
59 ConvertIndirectCallToIndirectJump(MachineBasicBlock &MBB,
60 MachineBasicBlock::iterator) const;
61};
62
63} // end anonymous namespace
64
65char ARMSLSHardening::ID = 0;
66
67INITIALIZE_PASS(ARMSLSHardening, "arm-sls-hardening",
68 ARM_SLS_HARDENING_NAME, false, false)
69
70static void insertSpeculationBarrier(const ARMSubtarget *ST,
71 MachineBasicBlock &MBB,
72 MachineBasicBlock::iterator MBBI,
73 DebugLoc DL,
74 bool AlwaysUseISBDSB = false) {
75 assert(MBBI != MBB.begin() &&
76 "Must not insert SpeculationBarrierEndBB as only instruction in MBB.");
77 assert(std::prev(MBBI)->isBarrier() &&
78 "SpeculationBarrierEndBB must only follow unconditional control flow "
79 "instructions.");
80 assert(std::prev(MBBI)->isTerminator() &&
81 "SpeculationBarrierEndBB must only follow terminators.");
82 const TargetInstrInfo *TII = ST->getInstrInfo();
83 assert(ST->hasDataBarrier() || ST->hasSB());
84 bool ProduceSB = ST->hasSB() && !AlwaysUseISBDSB;
85 unsigned BarrierOpc =
86 ProduceSB ? (ST->isThumb() ? ARM::t2SpeculationBarrierSBEndBB
87 : ARM::SpeculationBarrierSBEndBB)
88 : (ST->isThumb() ? ARM::t2SpeculationBarrierISBDSBEndBB
89 : ARM::SpeculationBarrierISBDSBEndBB);
90 if (MBBI == MBB.end() || !isSpeculationBarrierEndBBOpcode(Opc: MBBI->getOpcode()))
91 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: BarrierOpc));
92}
93
94bool ARMSLSHardening::runOnMachineFunction(MachineFunction &MF) {
95 ST = &MF.getSubtarget<ARMSubtarget>();
96 TII = MF.getSubtarget().getInstrInfo();
97
98 bool Modified = false;
99 for (auto &MBB : MF) {
100 Modified |= hardenReturnsAndBRs(MBB);
101 Modified |= hardenIndirectCalls(MBB);
102 }
103
104 return Modified;
105}
106
107bool ARMSLSHardening::hardenReturnsAndBRs(MachineBasicBlock &MBB) const {
108 if (!ST->hardenSlsRetBr())
109 return false;
110 assert(!ST->isThumb1Only());
111 bool Modified = false;
112 MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator(), E = MBB.end();
113 MachineBasicBlock::iterator NextMBBI;
114 for (; MBBI != E; MBBI = NextMBBI) {
115 MachineInstr &MI = *MBBI;
116 NextMBBI = std::next(x: MBBI);
117 if (isIndirectControlFlowNotComingBack(MI)) {
118 assert(MI.isTerminator());
119 assert(!TII->isPredicated(MI));
120 insertSpeculationBarrier(ST, MBB, MBBI: std::next(x: MBBI), DL: MI.getDebugLoc());
121 Modified = true;
122 }
123 }
124 return Modified;
125}
126
127static const char SLSBLRNamePrefix[] = "__llvm_slsblr_thunk_";
128
129static const struct ThunkNameRegMode {
130 const char* Name;
131 Register Reg;
132 bool isThumb;
133} SLSBLRThunks[] = {
134 {.Name: "__llvm_slsblr_thunk_arm_r0", .Reg: ARM::R0, .isThumb: false},
135 {.Name: "__llvm_slsblr_thunk_arm_r1", .Reg: ARM::R1, .isThumb: false},
136 {.Name: "__llvm_slsblr_thunk_arm_r2", .Reg: ARM::R2, .isThumb: false},
137 {.Name: "__llvm_slsblr_thunk_arm_r3", .Reg: ARM::R3, .isThumb: false},
138 {.Name: "__llvm_slsblr_thunk_arm_r4", .Reg: ARM::R4, .isThumb: false},
139 {.Name: "__llvm_slsblr_thunk_arm_r5", .Reg: ARM::R5, .isThumb: false},
140 {.Name: "__llvm_slsblr_thunk_arm_r6", .Reg: ARM::R6, .isThumb: false},
141 {.Name: "__llvm_slsblr_thunk_arm_r7", .Reg: ARM::R7, .isThumb: false},
142 {.Name: "__llvm_slsblr_thunk_arm_r8", .Reg: ARM::R8, .isThumb: false},
143 {.Name: "__llvm_slsblr_thunk_arm_r9", .Reg: ARM::R9, .isThumb: false},
144 {.Name: "__llvm_slsblr_thunk_arm_r10", .Reg: ARM::R10, .isThumb: false},
145 {.Name: "__llvm_slsblr_thunk_arm_r11", .Reg: ARM::R11, .isThumb: false},
146 {.Name: "__llvm_slsblr_thunk_arm_sp", .Reg: ARM::SP, .isThumb: false},
147 {.Name: "__llvm_slsblr_thunk_arm_pc", .Reg: ARM::PC, .isThumb: false},
148 {.Name: "__llvm_slsblr_thunk_thumb_r0", .Reg: ARM::R0, .isThumb: true},
149 {.Name: "__llvm_slsblr_thunk_thumb_r1", .Reg: ARM::R1, .isThumb: true},
150 {.Name: "__llvm_slsblr_thunk_thumb_r2", .Reg: ARM::R2, .isThumb: true},
151 {.Name: "__llvm_slsblr_thunk_thumb_r3", .Reg: ARM::R3, .isThumb: true},
152 {.Name: "__llvm_slsblr_thunk_thumb_r4", .Reg: ARM::R4, .isThumb: true},
153 {.Name: "__llvm_slsblr_thunk_thumb_r5", .Reg: ARM::R5, .isThumb: true},
154 {.Name: "__llvm_slsblr_thunk_thumb_r6", .Reg: ARM::R6, .isThumb: true},
155 {.Name: "__llvm_slsblr_thunk_thumb_r7", .Reg: ARM::R7, .isThumb: true},
156 {.Name: "__llvm_slsblr_thunk_thumb_r8", .Reg: ARM::R8, .isThumb: true},
157 {.Name: "__llvm_slsblr_thunk_thumb_r9", .Reg: ARM::R9, .isThumb: true},
158 {.Name: "__llvm_slsblr_thunk_thumb_r10", .Reg: ARM::R10, .isThumb: true},
159 {.Name: "__llvm_slsblr_thunk_thumb_r11", .Reg: ARM::R11, .isThumb: true},
160 {.Name: "__llvm_slsblr_thunk_thumb_sp", .Reg: ARM::SP, .isThumb: true},
161 {.Name: "__llvm_slsblr_thunk_thumb_pc", .Reg: ARM::PC, .isThumb: true},
162};
163
164// An enum for tracking whether Arm and Thumb thunks have been inserted into the
165// current module so far.
166enum ArmInsertedThunks { NoThunk = 0, ArmThunk = 1, ThumbThunk = 2 };
167
168inline ArmInsertedThunks &operator|=(ArmInsertedThunks &X,
169 ArmInsertedThunks Y) {
170 return X = static_cast<ArmInsertedThunks>(X | Y);
171}
172
173namespace {
174struct SLSBLRThunkInserter
175 : ThunkInserter<SLSBLRThunkInserter, ArmInsertedThunks> {
176 const char *getThunkPrefix() { return SLSBLRNamePrefix; }
177 bool mayUseThunk(const MachineFunction &MF) {
178 ComdatThunks &= !MF.getSubtarget<ARMSubtarget>().hardenSlsNoComdat();
179 return MF.getSubtarget<ARMSubtarget>().hardenSlsBlr();
180 }
181 ArmInsertedThunks insertThunks(MachineModuleInfo &MMI, MachineFunction &MF,
182 ArmInsertedThunks InsertedThunks);
183 void populateThunk(MachineFunction &MF);
184
185private:
186 bool ComdatThunks = true;
187};
188} // namespace
189
190ArmInsertedThunks
191SLSBLRThunkInserter::insertThunks(MachineModuleInfo &MMI, MachineFunction &MF,
192 ArmInsertedThunks InsertedThunks) {
193 if ((InsertedThunks & ArmThunk &&
194 !MF.getSubtarget<ARMSubtarget>().isThumb()) ||
195 (InsertedThunks & ThumbThunk &&
196 MF.getSubtarget<ARMSubtarget>().isThumb()))
197 return NoThunk;
198 // FIXME: It probably would be possible to filter which thunks to produce
199 // based on which registers are actually used in indirect calls in this
200 // function. But would that be a worthwhile optimization?
201 const ARMSubtarget *ST = &MF.getSubtarget<ARMSubtarget>();
202 for (auto T : SLSBLRThunks)
203 if (ST->isThumb() == T.isThumb)
204 createThunkFunction(MMI, Name: T.Name, Comdat: ComdatThunks,
205 TargetAttrs: T.isThumb ? "+thumb-mode" : "");
206 return ST->isThumb() ? ThumbThunk : ArmThunk;
207}
208
209void SLSBLRThunkInserter::populateThunk(MachineFunction &MF) {
210 assert(MF.getFunction().hasComdat() == ComdatThunks &&
211 "ComdatThunks value changed since MF creation");
212 // FIXME: How to better communicate Register number, rather than through
213 // name and lookup table?
214 assert(MF.getName().starts_with(getThunkPrefix()));
215 auto ThunkIt = llvm::find_if(
216 Range: SLSBLRThunks, P: [&MF](auto T) { return T.Name == MF.getName(); });
217 assert(ThunkIt != std::end(SLSBLRThunks));
218 Register ThunkReg = ThunkIt->Reg;
219 bool isThumb = ThunkIt->isThumb;
220
221 const TargetInstrInfo *TII = MF.getSubtarget<ARMSubtarget>().getInstrInfo();
222 MachineBasicBlock *Entry = &MF.front();
223 Entry->clear();
224
225 // These thunks need to consist of the following instructions:
226 // __llvm_slsblr_thunk_(arm/thumb)_rN:
227 // bx rN
228 // barrierInsts
229 Entry->addLiveIn(PhysReg: ThunkReg);
230 if (isThumb)
231 BuildMI(BB: Entry, MIMD: DebugLoc(), MCID: TII->get(Opcode: ARM::tBX))
232 .addReg(RegNo: ThunkReg)
233 .add(MOs: predOps(Pred: ARMCC::AL));
234 else
235 BuildMI(BB: Entry, MIMD: DebugLoc(), MCID: TII->get(Opcode: ARM::BX))
236 .addReg(RegNo: ThunkReg);
237
238 // Make sure the thunks do not make use of the SB extension in case there is
239 // a function somewhere that will call to it that for some reason disabled
240 // the SB extension locally on that function, even though it's enabled for
241 // the module otherwise. Therefore set AlwaysUseISBSDB to true.
242 insertSpeculationBarrier(ST: &MF.getSubtarget<ARMSubtarget>(), MBB&: *Entry,
243 MBBI: Entry->end(), DL: DebugLoc(), AlwaysUseISBDSB: true /*AlwaysUseISBDSB*/);
244}
245
246MachineBasicBlock &ARMSLSHardening::ConvertIndirectCallToIndirectJump(
247 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
248 // Transform an indirect call to an indirect jump as follows:
249 // Before:
250 // |-----------------------------|
251 // | ... |
252 // | instI |
253 // | BLX rN |
254 // | instJ |
255 // | ... |
256 // |-----------------------------|
257 //
258 // After:
259 // |---------- -------------------------|
260 // | ... |
261 // | instI |
262 // | *call* __llvm_slsblr_thunk_mode_xN |
263 // | instJ |
264 // | ... |
265 // |--------------------------------------|
266 //
267 // __llvm_slsblr_thunk_mode_xN:
268 // |-----------------------------|
269 // | BX rN |
270 // | barrierInsts |
271 // |-----------------------------|
272 //
273 // The __llvm_slsblr_thunk_mode_xN thunks are created by the
274 // SLSBLRThunkInserter.
275 // This function merely needs to transform an indirect call to a direct call
276 // to __llvm_slsblr_thunk_xN.
277 MachineInstr &IndirectCall = *MBBI;
278 assert(isIndirectCall(IndirectCall) && !IndirectCall.isReturn());
279 int RegOpIdxOnIndirectCall = -1;
280 bool isThumb;
281 switch (IndirectCall.getOpcode()) {
282 case ARM::BLX: // !isThumb2
283 case ARM::BLX_noip: // !isThumb2
284 isThumb = false;
285 RegOpIdxOnIndirectCall = 0;
286 break;
287 case ARM::tBLXr: // isThumb2
288 case ARM::tBLXr_noip: // isThumb2
289 isThumb = true;
290 RegOpIdxOnIndirectCall = 2;
291 break;
292 default:
293 llvm_unreachable("unhandled Indirect Call");
294 }
295
296 Register Reg = IndirectCall.getOperand(i: RegOpIdxOnIndirectCall).getReg();
297 // Since linkers are allowed to clobber R12 on function calls, the above
298 // mitigation only works if the original indirect call instruction was not
299 // using R12. Code generation before must make sure that no indirect call
300 // using R12 was produced if the mitigation is enabled.
301 // Also, the transformation is incorrect if the indirect call uses LR, so
302 // also have to avoid that.
303 assert(Reg != ARM::R12 && Reg != ARM::LR);
304 bool RegIsKilled = IndirectCall.getOperand(i: RegOpIdxOnIndirectCall).isKill();
305
306 DebugLoc DL = IndirectCall.getDebugLoc();
307
308 MachineFunction &MF = *MBBI->getMF();
309 auto ThunkIt = llvm::find_if(Range: SLSBLRThunks, P: [Reg, isThumb](auto T) {
310 return T.Reg == Reg && T.isThumb == isThumb;
311 });
312 assert(ThunkIt != std::end(SLSBLRThunks));
313 Module *M = MF.getFunction().getParent();
314 const GlobalValue *GV = cast<GlobalValue>(Val: M->getNamedValue(Name: ThunkIt->Name));
315
316 MachineInstr *BL =
317 isThumb ? BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::tBL))
318 .addImm(Val: IndirectCall.getOperand(i: 0).getImm())
319 .addReg(RegNo: IndirectCall.getOperand(i: 1).getReg())
320 .addGlobalAddress(GV)
321 : BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::BL)).addGlobalAddress(GV);
322
323 // Now copy the implicit operands from IndirectCall to BL and copy other
324 // necessary info.
325 // However, both IndirectCall and BL instructions implictly use SP and
326 // implicitly define LR. Blindly copying implicit operands would result in SP
327 // and LR operands to be present multiple times. While this may not be too
328 // much of an issue, let's avoid that for cleanliness, by removing those
329 // implicit operands from the BL created above before we copy over all
330 // implicit operands from the IndirectCall.
331 int ImpLROpIdx = -1;
332 int ImpSPOpIdx = -1;
333 for (unsigned OpIdx = BL->getNumExplicitOperands();
334 OpIdx < BL->getNumOperands(); OpIdx++) {
335 MachineOperand Op = BL->getOperand(i: OpIdx);
336 if (!Op.isReg())
337 continue;
338 if (Op.getReg() == ARM::LR && Op.isDef())
339 ImpLROpIdx = OpIdx;
340 if (Op.getReg() == ARM::SP && !Op.isDef())
341 ImpSPOpIdx = OpIdx;
342 }
343 assert(ImpLROpIdx != -1);
344 assert(ImpSPOpIdx != -1);
345 int FirstOpIdxToRemove = std::max(a: ImpLROpIdx, b: ImpSPOpIdx);
346 int SecondOpIdxToRemove = std::min(a: ImpLROpIdx, b: ImpSPOpIdx);
347 BL->removeOperand(OpNo: FirstOpIdxToRemove);
348 BL->removeOperand(OpNo: SecondOpIdxToRemove);
349 // Now copy over the implicit operands from the original IndirectCall
350 BL->copyImplicitOps(MF, MI: IndirectCall);
351 MF.moveAdditionalCallInfo(Old: &IndirectCall, New: BL);
352 // Also add the register called in the IndirectCall as being used in the
353 // called thunk.
354 BL->addOperand(Op: MachineOperand::CreateReg(Reg, isDef: false /*isDef*/, isImp: true /*isImp*/,
355 isKill: RegIsKilled /*isKill*/));
356 // Remove IndirectCallinstruction
357 MBB.erase(I: MBBI);
358 return MBB;
359}
360
361bool ARMSLSHardening::hardenIndirectCalls(MachineBasicBlock &MBB) const {
362 if (!ST->hardenSlsBlr())
363 return false;
364 bool Modified = false;
365 MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
366 MachineBasicBlock::iterator NextMBBI;
367 for (; MBBI != E; MBBI = NextMBBI) {
368 MachineInstr &MI = *MBBI;
369 NextMBBI = std::next(x: MBBI);
370 // Tail calls are both indirect calls and "returns".
371 // They are also indirect jumps, so should be handled by sls-harden-retbr,
372 // rather than sls-harden-blr.
373 if (isIndirectCall(MI) && !MI.isReturn()) {
374 ConvertIndirectCallToIndirectJump(MBB, MBBI);
375 Modified = true;
376 }
377 }
378 return Modified;
379}
380
381
382
383FunctionPass *llvm::createARMSLSHardeningPass() {
384 return new ARMSLSHardening();
385}
386
387namespace {
388class ARMIndirectThunks : public ThunkInserterPass<SLSBLRThunkInserter> {
389public:
390 static char ID;
391
392 ARMIndirectThunks() : ThunkInserterPass(ID) {}
393
394 StringRef getPassName() const override { return "ARM Indirect Thunks"; }
395};
396} // end anonymous namespace
397
398char ARMIndirectThunks::ID = 0;
399
400FunctionPass *llvm::createARMIndirectThunks() {
401 return new ARMIndirectThunks();
402}
403