1//==- X86IndirectThunks.cpp - Construct indirect call/jump thunks for x86 --=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9///
10/// Pass that injects an MI thunk that is used to lower indirect calls in a way
11/// that prevents speculation on some x86 processors and can be used to mitigate
12/// security vulnerabilities due to targeted speculative execution and side
13/// channels such as CVE-2017-5715.
14///
15/// Currently supported thunks include:
16/// - Retpoline -- A RET-implemented trampoline that lowers indirect calls
17/// - LVI Thunk -- A CALL/JMP-implemented thunk that forces load serialization
18/// before making an indirect call/jump
19///
20/// Note that the reason that this is implemented as a MachineFunctionPass and
21/// not a ModulePass is that ModulePasses at this point in the LLVM X86 pipeline
22/// serialize all transformations, which can consume lots of memory.
23///
24/// TODO(chandlerc): All of this code could use better comments and
25/// documentation.
26///
27//===----------------------------------------------------------------------===//
28
29#include "X86.h"
30#include "X86InstrBuilder.h"
31#include "X86Subtarget.h"
32#include "llvm/CodeGen/IndirectThunks.h"
33#include "llvm/CodeGen/MachineFunction.h"
34#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
35#include "llvm/CodeGen/MachineInstrBuilder.h"
36#include "llvm/CodeGen/MachineModuleInfo.h"
37#include "llvm/CodeGen/MachinePassManager.h"
38#include "llvm/CodeGen/Passes.h"
39#include "llvm/CodeGen/TargetPassConfig.h"
40#include "llvm/IR/Analysis.h"
41#include "llvm/IR/Instructions.h"
42#include "llvm/Target/TargetMachine.h"
43
44using namespace llvm;
45
46#define DEBUG_TYPE "x86-retpoline-thunks"
47
48static const char RetpolineNamePrefix[] = "__llvm_retpoline_";
49static const char R11RetpolineName[] = "__llvm_retpoline_r11";
50static const char EAXRetpolineName[] = "__llvm_retpoline_eax";
51static const char ECXRetpolineName[] = "__llvm_retpoline_ecx";
52static const char EDXRetpolineName[] = "__llvm_retpoline_edx";
53static const char EDIRetpolineName[] = "__llvm_retpoline_edi";
54
55static const char LVIThunkNamePrefix[] = "__llvm_lvi_thunk_";
56static const char R11LVIThunkName[] = "__llvm_lvi_thunk_r11";
57
58struct RetpolineThunkInserter : ThunkInserter<RetpolineThunkInserter> {
59 const char *getThunkPrefix() { return RetpolineNamePrefix; }
60 bool mayUseThunk(const MachineFunction &MF) {
61 const auto &STI = MF.getSubtarget<X86Subtarget>();
62 return (STI.useRetpolineIndirectCalls() ||
63 STI.useRetpolineIndirectBranches()) &&
64 !STI.useRetpolineExternalThunk();
65 }
66 bool insertThunks(MachineModuleInfo &MMI, MachineFunction &MF,
67 bool ExistingThunks);
68 void populateThunk(MachineFunction &MF);
69};
70
71struct LVIThunkInserter : ThunkInserter<LVIThunkInserter> {
72 const char *getThunkPrefix() { return LVIThunkNamePrefix; }
73 bool mayUseThunk(const MachineFunction &MF) {
74 return MF.getSubtarget<X86Subtarget>().useLVIControlFlowIntegrity();
75 }
76 bool insertThunks(MachineModuleInfo &MMI, MachineFunction &MF,
77 bool ExistingThunks) {
78 if (ExistingThunks)
79 return false;
80 createThunkFunction(MMI, Name: R11LVIThunkName);
81 return true;
82 }
83 void populateThunk(MachineFunction &MF) {
84 assert (MF.size() == 1);
85 MachineBasicBlock *Entry = &MF.front();
86 Entry->clear();
87
88 // This code mitigates LVI by replacing each indirect call/jump with a
89 // direct call/jump to a thunk that looks like:
90 // ```
91 // lfence
92 // jmpq *%r11
93 // ```
94 // This ensures that if the value in register %r11 was loaded from memory,
95 // then the value in %r11 is (architecturally) correct prior to the jump.
96 const TargetInstrInfo *TII = MF.getSubtarget<X86Subtarget>().getInstrInfo();
97 BuildMI(BB: &MF.front(), MIMD: DebugLoc(), MCID: TII->get(Opcode: X86::LFENCE));
98 BuildMI(BB: &MF.front(), MIMD: DebugLoc(), MCID: TII->get(Opcode: X86::JMP64r)).addReg(RegNo: X86::R11);
99 MF.front().addLiveIn(PhysReg: X86::R11);
100 }
101};
102
103namespace {
104
105class X86IndirectThunksLegacy
106 : public ThunkInserterPass<RetpolineThunkInserter, LVIThunkInserter> {
107public:
108 static char ID;
109
110 X86IndirectThunksLegacy() : ThunkInserterPass(ID) {}
111
112 StringRef getPassName() const override { return "X86 Indirect Thunks"; }
113};
114
115} // end anonymous namespace
116
117bool RetpolineThunkInserter::insertThunks(MachineModuleInfo &MMI,
118 MachineFunction &MF,
119 bool ExistingThunks) {
120 if (ExistingThunks)
121 return false;
122 if (MMI.getTarget().getTargetTriple().isX86_64())
123 createThunkFunction(MMI, Name: R11RetpolineName);
124 else
125 for (StringRef Name : {EAXRetpolineName, ECXRetpolineName, EDXRetpolineName,
126 EDIRetpolineName})
127 createThunkFunction(MMI, Name);
128 return true;
129}
130
131void RetpolineThunkInserter::populateThunk(MachineFunction &MF) {
132 bool Is64Bit = MF.getFunction().getParent()->getTargetTriple().isX86_64();
133 Register ThunkReg;
134 if (Is64Bit) {
135 assert(MF.getName() == "__llvm_retpoline_r11" &&
136 "Should only have an r11 thunk on 64-bit targets");
137
138 // __llvm_retpoline_r11:
139 // callq .Lr11_call_target
140 // .Lr11_capture_spec:
141 // pause
142 // lfence
143 // jmp .Lr11_capture_spec
144 // .align 16
145 // .Lr11_call_target:
146 // movq %r11, (%rsp)
147 // retq
148 ThunkReg = X86::R11;
149 } else {
150 // For 32-bit targets we need to emit a collection of thunks for various
151 // possible scratch registers as well as a fallback that uses EDI, which is
152 // normally callee saved.
153 // __llvm_retpoline_eax:
154 // calll .Leax_call_target
155 // .Leax_capture_spec:
156 // pause
157 // jmp .Leax_capture_spec
158 // .align 16
159 // .Leax_call_target:
160 // movl %eax, (%esp) # Clobber return addr
161 // retl
162 //
163 // __llvm_retpoline_ecx:
164 // ... # Same setup
165 // movl %ecx, (%esp)
166 // retl
167 //
168 // __llvm_retpoline_edx:
169 // ... # Same setup
170 // movl %edx, (%esp)
171 // retl
172 //
173 // __llvm_retpoline_edi:
174 // ... # Same setup
175 // movl %edi, (%esp)
176 // retl
177 if (MF.getName() == EAXRetpolineName)
178 ThunkReg = X86::EAX;
179 else if (MF.getName() == ECXRetpolineName)
180 ThunkReg = X86::ECX;
181 else if (MF.getName() == EDXRetpolineName)
182 ThunkReg = X86::EDX;
183 else if (MF.getName() == EDIRetpolineName)
184 ThunkReg = X86::EDI;
185 else
186 llvm_unreachable("Invalid thunk name on x86-32!");
187 }
188
189 const TargetInstrInfo *TII = MF.getSubtarget<X86Subtarget>().getInstrInfo();
190 assert (MF.size() == 1);
191 MachineBasicBlock *Entry = &MF.front();
192 Entry->clear();
193
194 MachineBasicBlock *CaptureSpec =
195 MF.CreateMachineBasicBlock(BB: Entry->getBasicBlock());
196 MachineBasicBlock *CallTarget =
197 MF.CreateMachineBasicBlock(BB: Entry->getBasicBlock());
198 MCSymbol *TargetSym = MF.getContext().createTempSymbol();
199 MF.push_back(MBB: CaptureSpec);
200 MF.push_back(MBB: CallTarget);
201
202 const unsigned CallOpc = Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32;
203 const unsigned RetOpc = Is64Bit ? X86::RET64 : X86::RET32;
204
205 Entry->addLiveIn(PhysReg: ThunkReg);
206 BuildMI(BB: Entry, MIMD: DebugLoc(), MCID: TII->get(Opcode: CallOpc)).addSym(Sym: TargetSym);
207
208 // The MIR verifier thinks that the CALL in the entry block will fall through
209 // to CaptureSpec, so mark it as the successor. Technically, CaptureTarget is
210 // the successor, but the MIR verifier doesn't know how to cope with that.
211 Entry->addSuccessor(Succ: CaptureSpec);
212
213 // In the capture loop for speculation, we want to stop the processor from
214 // speculating as fast as possible. On Intel processors, the PAUSE instruction
215 // will block speculation without consuming any execution resources. On AMD
216 // processors, the PAUSE instruction is (essentially) a nop, so we also use an
217 // LFENCE instruction which they have advised will stop speculation as well
218 // with minimal resource utilization. We still end the capture with a jump to
219 // form an infinite loop to fully guarantee that no matter what implementation
220 // of the x86 ISA, speculating this code path never escapes.
221 BuildMI(BB: CaptureSpec, MIMD: DebugLoc(), MCID: TII->get(Opcode: X86::PAUSE));
222 BuildMI(BB: CaptureSpec, MIMD: DebugLoc(), MCID: TII->get(Opcode: X86::LFENCE));
223 BuildMI(BB: CaptureSpec, MIMD: DebugLoc(), MCID: TII->get(Opcode: X86::JMP_1)).addMBB(MBB: CaptureSpec);
224 CaptureSpec->setMachineBlockAddressTaken();
225 CaptureSpec->addSuccessor(Succ: CaptureSpec);
226
227 CallTarget->addLiveIn(PhysReg: ThunkReg);
228 CallTarget->setMachineBlockAddressTaken();
229 CallTarget->setAlignment(Align(16));
230
231 // Insert return address clobber
232 const unsigned MovOpc = Is64Bit ? X86::MOV64mr : X86::MOV32mr;
233 const Register SPReg = Is64Bit ? X86::RSP : X86::ESP;
234 addRegOffset(MIB: BuildMI(BB: CallTarget, MIMD: DebugLoc(), MCID: TII->get(Opcode: MovOpc)), Reg: SPReg, Offset: 0)
235 .addReg(RegNo: ThunkReg);
236
237 CallTarget->back().setPreInstrSymbol(MF, Symbol: TargetSym);
238 BuildMI(BB: CallTarget, MIMD: DebugLoc(), MCID: TII->get(Opcode: RetOpc));
239}
240
241FunctionPass *llvm::createX86IndirectThunksLegacyPass() {
242 return new X86IndirectThunksLegacy();
243}
244
245char X86IndirectThunksLegacy::ID = 0;
246
247X86IndirectThunksPass::X86IndirectThunksPass()
248 : RetpolineInserter(std::make_unique<RetpolineThunkInserter>()),
249 LVIInserter(std::make_unique<LVIThunkInserter>()) {}
250X86IndirectThunksPass::X86IndirectThunksPass(X86IndirectThunksPass &&) =
251 default;
252X86IndirectThunksPass::~X86IndirectThunksPass() = default;
253
254PreservedAnalyses
255X86IndirectThunksPass::run(MachineFunction &MF,
256 MachineFunctionAnalysisManager &MFAM) {
257 MachineModuleAnalysis::Result *MMI =
258 MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(IR&: MF)
259 .getCachedResult<MachineModuleAnalysis>(
260 IR&: *MF.getFunction().getParent());
261 assert(MMI && "MachineModuleAnalysis must be available");
262
263 bool Changed = RetpolineInserter->run(MMI&: MMI->getMMI(), MF);
264 Changed |= LVIInserter->run(MMI&: MMI->getMMI(), MF);
265 if (!Changed)
266 return PreservedAnalyses::all();
267 return getMachineFunctionPassPreservedAnalyses().preserveSet<CFGAnalyses>();
268}
269