| 1 | //==- X86IndirectThunks.cpp - Construct indirect call/jump thunks for x86 --=// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | /// \file |
| 9 | /// |
| 10 | /// Pass that injects an MI thunk that is used to lower indirect calls in a way |
| 11 | /// that prevents speculation on some x86 processors and can be used to mitigate |
| 12 | /// security vulnerabilities due to targeted speculative execution and side |
| 13 | /// channels such as CVE-2017-5715. |
| 14 | /// |
| 15 | /// Currently supported thunks include: |
| 16 | /// - Retpoline -- A RET-implemented trampoline that lowers indirect calls |
| 17 | /// - LVI Thunk -- A CALL/JMP-implemented thunk that forces load serialization |
| 18 | /// before making an indirect call/jump |
| 19 | /// |
| 20 | /// Note that the reason that this is implemented as a MachineFunctionPass and |
| 21 | /// not a ModulePass is that ModulePasses at this point in the LLVM X86 pipeline |
| 22 | /// serialize all transformations, which can consume lots of memory. |
| 23 | /// |
| 24 | /// TODO(chandlerc): All of this code could use better comments and |
| 25 | /// documentation. |
| 26 | /// |
| 27 | //===----------------------------------------------------------------------===// |
| 28 | |
| 29 | #include "X86.h" |
| 30 | #include "X86InstrBuilder.h" |
| 31 | #include "X86Subtarget.h" |
| 32 | #include "llvm/CodeGen/IndirectThunks.h" |
| 33 | #include "llvm/CodeGen/MachineFunction.h" |
| 34 | #include "llvm/CodeGen/MachineFunctionAnalysisManager.h" |
| 35 | #include "llvm/CodeGen/MachineInstrBuilder.h" |
| 36 | #include "llvm/CodeGen/MachineModuleInfo.h" |
| 37 | #include "llvm/CodeGen/MachinePassManager.h" |
| 38 | #include "llvm/CodeGen/Passes.h" |
| 39 | #include "llvm/CodeGen/TargetPassConfig.h" |
| 40 | #include "llvm/IR/Analysis.h" |
| 41 | #include "llvm/IR/Instructions.h" |
| 42 | #include "llvm/Target/TargetMachine.h" |
| 43 | |
| 44 | using namespace llvm; |
| 45 | |
| 46 | #define DEBUG_TYPE "x86-retpoline-thunks" |
| 47 | |
| 48 | static const char RetpolineNamePrefix[] = "__llvm_retpoline_" ; |
| 49 | static const char R11RetpolineName[] = "__llvm_retpoline_r11" ; |
| 50 | static const char EAXRetpolineName[] = "__llvm_retpoline_eax" ; |
| 51 | static const char ECXRetpolineName[] = "__llvm_retpoline_ecx" ; |
| 52 | static const char EDXRetpolineName[] = "__llvm_retpoline_edx" ; |
| 53 | static const char EDIRetpolineName[] = "__llvm_retpoline_edi" ; |
| 54 | |
| 55 | static const char LVIThunkNamePrefix[] = "__llvm_lvi_thunk_" ; |
| 56 | static const char R11LVIThunkName[] = "__llvm_lvi_thunk_r11" ; |
| 57 | |
| 58 | struct RetpolineThunkInserter : ThunkInserter<RetpolineThunkInserter> { |
| 59 | const char *getThunkPrefix() { return RetpolineNamePrefix; } |
| 60 | bool mayUseThunk(const MachineFunction &MF) { |
| 61 | const auto &STI = MF.getSubtarget<X86Subtarget>(); |
| 62 | return (STI.useRetpolineIndirectCalls() || |
| 63 | STI.useRetpolineIndirectBranches()) && |
| 64 | !STI.useRetpolineExternalThunk(); |
| 65 | } |
| 66 | bool insertThunks(MachineModuleInfo &MMI, MachineFunction &MF, |
| 67 | bool ExistingThunks); |
| 68 | void populateThunk(MachineFunction &MF); |
| 69 | }; |
| 70 | |
| 71 | struct LVIThunkInserter : ThunkInserter<LVIThunkInserter> { |
| 72 | const char *getThunkPrefix() { return LVIThunkNamePrefix; } |
| 73 | bool mayUseThunk(const MachineFunction &MF) { |
| 74 | return MF.getSubtarget<X86Subtarget>().useLVIControlFlowIntegrity(); |
| 75 | } |
| 76 | bool insertThunks(MachineModuleInfo &MMI, MachineFunction &MF, |
| 77 | bool ExistingThunks) { |
| 78 | if (ExistingThunks) |
| 79 | return false; |
| 80 | createThunkFunction(MMI, Name: R11LVIThunkName); |
| 81 | return true; |
| 82 | } |
| 83 | void populateThunk(MachineFunction &MF) { |
| 84 | assert (MF.size() == 1); |
| 85 | MachineBasicBlock *Entry = &MF.front(); |
| 86 | Entry->clear(); |
| 87 | |
| 88 | // This code mitigates LVI by replacing each indirect call/jump with a |
| 89 | // direct call/jump to a thunk that looks like: |
| 90 | // ``` |
| 91 | // lfence |
| 92 | // jmpq *%r11 |
| 93 | // ``` |
| 94 | // This ensures that if the value in register %r11 was loaded from memory, |
| 95 | // then the value in %r11 is (architecturally) correct prior to the jump. |
| 96 | const TargetInstrInfo *TII = MF.getSubtarget<X86Subtarget>().getInstrInfo(); |
| 97 | BuildMI(BB: &MF.front(), MIMD: DebugLoc(), MCID: TII->get(Opcode: X86::LFENCE)); |
| 98 | BuildMI(BB: &MF.front(), MIMD: DebugLoc(), MCID: TII->get(Opcode: X86::JMP64r)).addReg(RegNo: X86::R11); |
| 99 | MF.front().addLiveIn(PhysReg: X86::R11); |
| 100 | } |
| 101 | }; |
| 102 | |
| 103 | namespace { |
| 104 | |
| 105 | class X86IndirectThunksLegacy |
| 106 | : public ThunkInserterPass<RetpolineThunkInserter, LVIThunkInserter> { |
| 107 | public: |
| 108 | static char ID; |
| 109 | |
| 110 | X86IndirectThunksLegacy() : ThunkInserterPass(ID) {} |
| 111 | |
| 112 | StringRef getPassName() const override { return "X86 Indirect Thunks" ; } |
| 113 | }; |
| 114 | |
| 115 | } // end anonymous namespace |
| 116 | |
| 117 | bool RetpolineThunkInserter::insertThunks(MachineModuleInfo &MMI, |
| 118 | MachineFunction &MF, |
| 119 | bool ExistingThunks) { |
| 120 | if (ExistingThunks) |
| 121 | return false; |
| 122 | if (MMI.getTarget().getTargetTriple().isX86_64()) |
| 123 | createThunkFunction(MMI, Name: R11RetpolineName); |
| 124 | else |
| 125 | for (StringRef Name : {EAXRetpolineName, ECXRetpolineName, EDXRetpolineName, |
| 126 | EDIRetpolineName}) |
| 127 | createThunkFunction(MMI, Name); |
| 128 | return true; |
| 129 | } |
| 130 | |
| 131 | void RetpolineThunkInserter::populateThunk(MachineFunction &MF) { |
| 132 | bool Is64Bit = MF.getFunction().getParent()->getTargetTriple().isX86_64(); |
| 133 | Register ThunkReg; |
| 134 | if (Is64Bit) { |
| 135 | assert(MF.getName() == "__llvm_retpoline_r11" && |
| 136 | "Should only have an r11 thunk on 64-bit targets" ); |
| 137 | |
| 138 | // __llvm_retpoline_r11: |
| 139 | // callq .Lr11_call_target |
| 140 | // .Lr11_capture_spec: |
| 141 | // pause |
| 142 | // lfence |
| 143 | // jmp .Lr11_capture_spec |
| 144 | // .align 16 |
| 145 | // .Lr11_call_target: |
| 146 | // movq %r11, (%rsp) |
| 147 | // retq |
| 148 | ThunkReg = X86::R11; |
| 149 | } else { |
| 150 | // For 32-bit targets we need to emit a collection of thunks for various |
| 151 | // possible scratch registers as well as a fallback that uses EDI, which is |
| 152 | // normally callee saved. |
| 153 | // __llvm_retpoline_eax: |
| 154 | // calll .Leax_call_target |
| 155 | // .Leax_capture_spec: |
| 156 | // pause |
| 157 | // jmp .Leax_capture_spec |
| 158 | // .align 16 |
| 159 | // .Leax_call_target: |
| 160 | // movl %eax, (%esp) # Clobber return addr |
| 161 | // retl |
| 162 | // |
| 163 | // __llvm_retpoline_ecx: |
| 164 | // ... # Same setup |
| 165 | // movl %ecx, (%esp) |
| 166 | // retl |
| 167 | // |
| 168 | // __llvm_retpoline_edx: |
| 169 | // ... # Same setup |
| 170 | // movl %edx, (%esp) |
| 171 | // retl |
| 172 | // |
| 173 | // __llvm_retpoline_edi: |
| 174 | // ... # Same setup |
| 175 | // movl %edi, (%esp) |
| 176 | // retl |
| 177 | if (MF.getName() == EAXRetpolineName) |
| 178 | ThunkReg = X86::EAX; |
| 179 | else if (MF.getName() == ECXRetpolineName) |
| 180 | ThunkReg = X86::ECX; |
| 181 | else if (MF.getName() == EDXRetpolineName) |
| 182 | ThunkReg = X86::EDX; |
| 183 | else if (MF.getName() == EDIRetpolineName) |
| 184 | ThunkReg = X86::EDI; |
| 185 | else |
| 186 | llvm_unreachable("Invalid thunk name on x86-32!" ); |
| 187 | } |
| 188 | |
| 189 | const TargetInstrInfo *TII = MF.getSubtarget<X86Subtarget>().getInstrInfo(); |
| 190 | assert (MF.size() == 1); |
| 191 | MachineBasicBlock *Entry = &MF.front(); |
| 192 | Entry->clear(); |
| 193 | |
| 194 | MachineBasicBlock *CaptureSpec = |
| 195 | MF.CreateMachineBasicBlock(BB: Entry->getBasicBlock()); |
| 196 | MachineBasicBlock *CallTarget = |
| 197 | MF.CreateMachineBasicBlock(BB: Entry->getBasicBlock()); |
| 198 | MCSymbol *TargetSym = MF.getContext().createTempSymbol(); |
| 199 | MF.push_back(MBB: CaptureSpec); |
| 200 | MF.push_back(MBB: CallTarget); |
| 201 | |
| 202 | const unsigned CallOpc = Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32; |
| 203 | const unsigned RetOpc = Is64Bit ? X86::RET64 : X86::RET32; |
| 204 | |
| 205 | Entry->addLiveIn(PhysReg: ThunkReg); |
| 206 | BuildMI(BB: Entry, MIMD: DebugLoc(), MCID: TII->get(Opcode: CallOpc)).addSym(Sym: TargetSym); |
| 207 | |
| 208 | // The MIR verifier thinks that the CALL in the entry block will fall through |
| 209 | // to CaptureSpec, so mark it as the successor. Technically, CaptureTarget is |
| 210 | // the successor, but the MIR verifier doesn't know how to cope with that. |
| 211 | Entry->addSuccessor(Succ: CaptureSpec); |
| 212 | |
| 213 | // In the capture loop for speculation, we want to stop the processor from |
| 214 | // speculating as fast as possible. On Intel processors, the PAUSE instruction |
| 215 | // will block speculation without consuming any execution resources. On AMD |
| 216 | // processors, the PAUSE instruction is (essentially) a nop, so we also use an |
| 217 | // LFENCE instruction which they have advised will stop speculation as well |
| 218 | // with minimal resource utilization. We still end the capture with a jump to |
| 219 | // form an infinite loop to fully guarantee that no matter what implementation |
| 220 | // of the x86 ISA, speculating this code path never escapes. |
| 221 | BuildMI(BB: CaptureSpec, MIMD: DebugLoc(), MCID: TII->get(Opcode: X86::PAUSE)); |
| 222 | BuildMI(BB: CaptureSpec, MIMD: DebugLoc(), MCID: TII->get(Opcode: X86::LFENCE)); |
| 223 | BuildMI(BB: CaptureSpec, MIMD: DebugLoc(), MCID: TII->get(Opcode: X86::JMP_1)).addMBB(MBB: CaptureSpec); |
| 224 | CaptureSpec->setMachineBlockAddressTaken(); |
| 225 | CaptureSpec->addSuccessor(Succ: CaptureSpec); |
| 226 | |
| 227 | CallTarget->addLiveIn(PhysReg: ThunkReg); |
| 228 | CallTarget->setMachineBlockAddressTaken(); |
| 229 | CallTarget->setAlignment(Align(16)); |
| 230 | |
| 231 | // Insert return address clobber |
| 232 | const unsigned MovOpc = Is64Bit ? X86::MOV64mr : X86::MOV32mr; |
| 233 | const Register SPReg = Is64Bit ? X86::RSP : X86::ESP; |
| 234 | addRegOffset(MIB: BuildMI(BB: CallTarget, MIMD: DebugLoc(), MCID: TII->get(Opcode: MovOpc)), Reg: SPReg, Offset: 0) |
| 235 | .addReg(RegNo: ThunkReg); |
| 236 | |
| 237 | CallTarget->back().setPreInstrSymbol(MF, Symbol: TargetSym); |
| 238 | BuildMI(BB: CallTarget, MIMD: DebugLoc(), MCID: TII->get(Opcode: RetOpc)); |
| 239 | } |
| 240 | |
| 241 | FunctionPass *llvm::createX86IndirectThunksLegacyPass() { |
| 242 | return new X86IndirectThunksLegacy(); |
| 243 | } |
| 244 | |
| 245 | char X86IndirectThunksLegacy::ID = 0; |
| 246 | |
| 247 | X86IndirectThunksPass::X86IndirectThunksPass() |
| 248 | : RetpolineInserter(std::make_unique<RetpolineThunkInserter>()), |
| 249 | LVIInserter(std::make_unique<LVIThunkInserter>()) {} |
| 250 | X86IndirectThunksPass::X86IndirectThunksPass(X86IndirectThunksPass &&) = |
| 251 | default; |
| 252 | X86IndirectThunksPass::~X86IndirectThunksPass() = default; |
| 253 | |
| 254 | PreservedAnalyses |
| 255 | X86IndirectThunksPass::run(MachineFunction &MF, |
| 256 | MachineFunctionAnalysisManager &MFAM) { |
| 257 | MachineModuleAnalysis::Result *MMI = |
| 258 | MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(IR&: MF) |
| 259 | .getCachedResult<MachineModuleAnalysis>( |
| 260 | IR&: *MF.getFunction().getParent()); |
| 261 | assert(MMI && "MachineModuleAnalysis must be available" ); |
| 262 | |
| 263 | bool Changed = RetpolineInserter->run(MMI&: MMI->getMMI(), MF); |
| 264 | Changed |= LVIInserter->run(MMI&: MMI->getMMI(), MF); |
| 265 | if (!Changed) |
| 266 | return PreservedAnalyses::all(); |
| 267 | return getMachineFunctionPassPreservedAnalyses().preserveSet<CFGAnalyses>(); |
| 268 | } |
| 269 | |