1//===- X86MCLFIRewriter.cpp -------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the X86MCLFIRewriter class, which rewrites X86-64
10// instructions for LFI (Lightweight Fault Isolation) sandboxing.
11//
12//===----------------------------------------------------------------------===//
13
14#include "X86MCLFIRewriter.h"
15#include "X86BaseInfo.h"
16#include "X86MCTargetDesc.h"
17#include "llvm/MC/MCContext.h"
18#include "llvm/MC/MCExpr.h"
19#include "llvm/MC/MCInst.h"
20#include "llvm/MC/MCInstBuilder.h"
21#include "llvm/MC/MCStreamer.h"
22#include "llvm/MC/MCSubtargetInfo.h"
23
24using namespace llvm;
25
26// LFI reserved registers.
27static constexpr MCRegister LFIBaseReg = X86::R14;
28static constexpr MCRegister LFIScratchReg = X86::R11;
29static constexpr MCRegister LFITPReg = X86::R15;
30
31// Byte offset into the context register file (pointed to by R15) where the
32// thread pointer is stored.
33static constexpr int TPOffset = 16;
34
35static bool isSyscall(const MCInst &Inst) {
36 return Inst.getOpcode() == X86::SYSCALL;
37}
38
39static bool isDirectCall(const MCInst &Inst) {
40 return Inst.getOpcode() == X86::CALL64pcrel32;
41}
42
43static bool isSupportedIndirectBranch(const MCInst &Inst) {
44 switch (Inst.getOpcode()) {
45 case X86::JMP64r:
46 case X86::JMP64r_NT:
47 case X86::JMP64m:
48 case X86::JMP64m_NT:
49 case X86::CALL64r:
50 case X86::CALL64r_NT:
51 case X86::CALL64m:
52 case X86::CALL64m_NT:
53 return true;
54 default:
55 return false;
56 }
57}
58
59static bool hasNoTrackPrefix(const MCInst &Inst, const MCInstrInfo &InstInfo) {
60 return (InstInfo.get(Opcode: Inst.getOpcode()).TSFlags & X86II::NOTRACK) ||
61 (Inst.getFlags() & X86::IP_HAS_NOTRACK);
62}
63
64// Find the index of the memory operand if it has an %fs segment override.
65// Returns -1 if there is no memory operand or no %fs override.
66static int findFSMemOperand(const MCInst &Inst, const MCInstrInfo &InstInfo) {
67 int MemIdx = X86II::getMemoryOperandIdx(Desc: InstInfo.get(Opcode: Inst.getOpcode()));
68 if (MemIdx < 0)
69 return -1;
70 const MCOperand &Seg = Inst.getOperand(i: MemIdx + X86::AddrSegmentReg);
71 if (Seg.isReg() && Seg.getReg() == X86::FS)
72 return MemIdx;
73 return -1;
74}
75
76// Return true if the instruction reads from Reg.
77static bool readsRegister(const MCInst &Inst, const MCInstrDesc &Desc,
78 MCRegister Reg, const MCRegisterInfo &RI) {
79 for (unsigned I = Desc.getNumDefs(), E = Inst.getNumOperands(); I < E; ++I) {
80 const MCOperand &Op = Inst.getOperand(i: I);
81 if (Op.isReg() && Op.getReg() && RI.regsOverlap(RegA: Op.getReg(), RegB: Reg))
82 return true;
83 }
84 for (MCPhysReg Use : Desc.implicit_uses())
85 if (RI.regsOverlap(RegA: Use, RegB: Reg))
86 return true;
87 return false;
88}
89
90// Return true if Reg is absent or a 64-bit general-purpose register.
91static bool isGR64OrNone(MCRegister Reg) {
92 return Reg == X86::NoRegister ||
93 getX86MCRegisterClass(RC: X86::GR64RegClassID).contains(Reg);
94}
95
96// syscall
97// ->
98// .bundle_lock
99// leaq .Ltmp(%rip), %r11
100// jmpq *(%r14)
101// .Ltmp:
102// .bundle_unlock
103void X86::X86MCLFIRewriter::rewriteSyscall(const MCInst &Inst, MCStreamer &Out,
104 const MCSubtargetInfo &STI) {
105 Out.emitBundleLock(/*AlignToEnd=*/false, STI);
106
107 MCSymbol *Symbol = Out.getContext().createTempSymbol();
108
109 // leaq .Ltmp(%rip), %r11
110 Out.emitInstruction(
111 Inst: MCInstBuilder(X86::LEA64r)
112 .addReg(Reg: LFIScratchReg)
113 .addReg(Reg: X86::RIP)
114 .addImm(Val: 1)
115 .addReg(Reg: X86::NoRegister)
116 .addExpr(Val: MCSymbolRefExpr::create(Symbol, Ctx&: Out.getContext()))
117 .addReg(Reg: X86::NoRegister),
118 STI);
119
120 // jmpq *-8(%r14)
121 Out.emitInstruction(Inst: MCInstBuilder(X86::JMP64m)
122 .addReg(Reg: LFIBaseReg)
123 .addImm(Val: 1)
124 .addReg(Reg: X86::NoRegister)
125 .addImm(Val: -8)
126 .addReg(Reg: X86::NoRegister),
127 STI);
128
129 Out.emitLabel(Symbol);
130 Out.emitBundleUnlock(STI);
131}
132
133// andl $-LFIBundleSize, %eX
134// addq %r14, %rX
135void X86::X86MCLFIRewriter::emitSandboxBranchReg(MCRegister Reg,
136 MCStreamer &Out,
137 const MCSubtargetInfo &STI) {
138 MCRegister Reg32 = RegInfo->getSubReg(Reg, Idx: X86::sub_32bit);
139
140 Out.emitInstruction(Inst: MCInstBuilder(X86::AND32ri8)
141 .addReg(Reg: Reg32)
142 .addReg(Reg: Reg32)
143 .addImm(Val: -static_cast<int64_t>(LFIBundleSize)),
144 STI);
145
146 Out.emitInstruction(
147 Inst: MCInstBuilder(X86::ADD64rr).addReg(Reg).addReg(Reg).addReg(Reg: LFIBaseReg),
148 STI);
149}
150
151// Rewrite an indirect jump or call so that it can only target a bundle
152// boundary inside the sandbox.
153//
154// jmpq *%rX
155// ->
156// .bundle_lock
157// andl $-32, %eX
158// addq %r14, %rX
159// jmpq *%rX
160// .bundle_unlock
161//
162// A branch through memory loads its target into the scratch register first,
163// and then dispatches through it.
164//
165// jmpq *(%rdi)
166// ->
167// movq (%rdi), %r11
168// .bundle_lock
169// andl $-32, %r11d
170// addq %r14, %r11
171// jmpq *%r11
172// .bundle_unlock
173void X86::X86MCLFIRewriter::rewriteIndirectBranch(const MCInst &Inst,
174 MCStreamer &Out,
175 const MCSubtargetInfo &STI) {
176 MCRegister Target;
177 int MemIdx = X86II::getMemoryOperandIdx(Desc: InstInfo->get(Opcode: Inst.getOpcode()));
178 if (MemIdx >= 0) {
179 Target = LFIScratchReg;
180
181 // Construct the load and then apply the rewriter to it.
182 MCInstBuilder Mov(X86::MOV64rm);
183 Mov.addReg(Reg: Target);
184 for (unsigned I = 0; I < X86::AddrNumOperands; ++I)
185 Mov.addOperand(Op: Inst.getOperand(i: MemIdx + I));
186 Mov.setLoc(Inst.getLoc());
187 doRewriteInst(Inst: Mov, Out, STI);
188 } else {
189 Target = Inst.getOperand(i: 0).getReg();
190
191 if (Target == LFIBaseReg || Target == LFITPReg || Target == X86::RSP)
192 return error(Inst, Msg: "indirect branch through reserved register");
193 }
194
195 Out.emitBundleLock(/*AlignToEnd=*/isCall(Inst), STI);
196
197 emitSandboxBranchReg(Reg: Target, Out, STI);
198
199 MCInst Branch =
200 MCInstBuilder(isCall(Inst) ? X86::CALL64r : X86::JMP64r).addReg(Reg: Target);
201 if (hasNoTrackPrefix(Inst, InstInfo: *InstInfo))
202 Branch.setFlags(Branch.getFlags() | X86::IP_HAS_NOTRACK);
203 Out.emitInstruction(Inst: Branch, STI);
204
205 Out.emitBundleUnlock(STI);
206}
207
208// Direct calls are not rewritten, but must be placed at the end of a bundle
209// so that the return address they push is bundle-aligned.
210void X86::X86MCLFIRewriter::rewriteDirectCall(const MCInst &Inst,
211 MCStreamer &Out,
212 const MCSubtargetInfo &STI) {
213 Out.emitBundleLock(/*AlignToEnd=*/true, STI);
214 Out.emitInstruction(Inst, STI);
215 Out.emitBundleUnlock(STI);
216}
217
218// ret
219// ->
220// popq %r11
221// .bundle_lock
222// andl $-32, %r11d
223// addq %r14, %r11
224// jmpq *%r11
225// .bundle_unlock
226//
227// A return with an immediate additionally pops the immediate off the stack
228// after loading the return address.
229//
230// retq $16
231// ->
232// popq %r11
233// addq $16, %rsp
234// .bundle_lock
235// andl $-32, %r11d
236// addq %r14, %r11
237// jmpq *%r11
238// .bundle_unlock
239void X86::X86MCLFIRewriter::rewriteReturn(const MCInst &Inst, MCStreamer &Out,
240 const MCSubtargetInfo &STI) {
241 if (Inst.getOpcode() != X86::RET64 && Inst.getOpcode() != X86::RETI64)
242 return error(Inst, Msg: "unsupported return instruction");
243
244 Out.emitInstruction(Inst: MCInstBuilder(X86::POP64r).addReg(Reg: LFIScratchReg), STI);
245
246 if (Inst.getOpcode() == X86::RETI64) {
247 // Return with an immediate is rewritten recursively so that the stack
248 // pointer modification goes through the rewriter.
249 doRewriteInst(Inst: MCInstBuilder(X86::ADD64ri32)
250 .addReg(Reg: X86::RSP)
251 .addReg(Reg: X86::RSP)
252 .addOperand(Op: Inst.getOperand(i: 0))
253 .setLoc(Inst.getLoc()),
254 Out, STI);
255 }
256
257 Out.emitBundleLock(/*AlignToEnd=*/false, STI);
258
259 emitSandboxBranchReg(Reg: LFIScratchReg, Out, STI);
260
261 Out.emitInstruction(Inst: MCInstBuilder(X86::JMP64r).addReg(Reg: LFIScratchReg), STI);
262
263 Out.emitBundleUnlock(STI);
264}
265
266// Emit: movq TPOffset(%r15), %Reg
267static void emitTPLoad(MCRegister Reg, MCStreamer &Out,
268 const MCSubtargetInfo &STI) {
269 Out.emitInstruction(Inst: MCInstBuilder(X86::MOV64rm)
270 .addReg(Reg)
271 .addReg(Reg: LFITPReg)
272 .addImm(Val: 1)
273 .addReg(Reg: X86::NoRegister)
274 .addImm(Val: TPOffset)
275 .addReg(Reg: X86::NoRegister),
276 STI);
277}
278
279bool X86::X86MCLFIRewriter::isFSAccess(const MCInst &Inst) {
280 return (mayLoad(Inst) || mayStore(Inst)) &&
281 findFSMemOperand(Inst, InstInfo: *InstInfo) >= 0;
282}
283
284// Rewrite %fs-segment memory accesses to use the virtual thread pointer stored
285// at TPOffset(%r15). The actual memory access is currently unsandboxed because
286// load/store sandboxing is not yet supported. Example rewrites:
287//
288// movq %fs:0, %rax
289// ->
290// movq 16(%r15), %rax
291//
292// movq %fs:(%rdi), %rax
293// ->
294// movq 16(%r15), %rax
295// movq (%rax, %rdi), %rax
296//
297// movq %fs:8(%rdi, %rsi, 2), %rax
298// ->
299// movq 16(%r15), %rax
300// leaq (%rax, %rdi), %rax
301// movq 8(%rax, %rsi, 2), %rax
302void X86::X86MCLFIRewriter::rewriteFSAccess(const MCInst &Inst, MCStreamer &Out,
303 const MCSubtargetInfo &STI) {
304 int MemIdx = findFSMemOperand(Inst, InstInfo: *InstInfo);
305 assert(MemIdx >= 0);
306
307 MCRegister BaseReg = Inst.getOperand(i: MemIdx + X86::AddrBaseReg).getReg();
308 MCRegister IndexReg = Inst.getOperand(i: MemIdx + X86::AddrIndexReg).getReg();
309 bool HasBase = BaseReg != X86::NoRegister;
310 bool HasIndex = IndexReg != X86::NoRegister;
311 bool HasDisp = !Inst.getOperand(i: MemIdx + X86::AddrDisp).isImm() ||
312 Inst.getOperand(i: MemIdx + X86::AddrDisp).getImm() != 0;
313
314 // %fs:0 -> TPOffset(%r15)
315 if (!HasBase && !HasIndex && !HasDisp) {
316 MCInst Modified(Inst);
317 Modified.getOperand(i: MemIdx + X86::AddrBaseReg).setReg(LFITPReg);
318 Modified.getOperand(i: MemIdx + X86::AddrDisp).setImm(TPOffset);
319 Modified.getOperand(i: MemIdx + X86::AddrSegmentReg).setReg(X86::NoRegister);
320 return Out.emitInstruction(Inst: Modified, STI);
321 }
322
323 if (!isGR64OrNone(Reg: BaseReg) || !isGR64OrNone(Reg: IndexReg) ||
324 BaseReg == X86::RSP || BaseReg == X86::RIP)
325 return error(Inst, Msg: "unsupported addressing mode for %fs access");
326
327 const MCInstrDesc &Desc = InstInfo->get(Opcode: Inst.getOpcode());
328
329 // Reuse operand 0 as the TP temporary when the instruction writes it without
330 // also reading it, otherwise use %r11.
331 MCRegister TPDest = LFIScratchReg;
332 if (MemIdx > 0 && Inst.getOperand(i: 0).isReg()) {
333 MCRegister DestReg = Inst.getOperand(i: 0).getReg();
334 if (Desc.getNumDefs() > 0 &&
335 getX86MCRegisterClass(RC: X86::GR64RegClassID).contains(Reg: DestReg) &&
336 !readsRegister(Inst, Desc, Reg: DestReg, RI: *RegInfo))
337 TPDest = DestReg;
338 }
339
340 if (TPDest == LFIScratchReg &&
341 readsRegister(Inst, Desc, Reg: LFIScratchReg, RI: *RegInfo))
342 return error(Inst, Msg: "%fs access reads reserved register %r11");
343
344 emitTPLoad(Reg: TPDest, Out, STI);
345
346 // Both slots occupied: the compute base via lea. For example:
347 //
348 // movq %fs:8(%rdi,%rsi,2), %rax
349 // ->
350 // movq 16(%r15), %rax
351 // leaq (%rax,%rdi), %rax
352 // movq 8(%rax,%rsi,2), %rax
353 if (HasBase && HasIndex) {
354 Out.emitInstruction(Inst: MCInstBuilder(X86::LEA64r)
355 .addReg(Reg: TPDest)
356 .addReg(Reg: TPDest)
357 .addImm(Val: 1)
358 .addReg(Reg: BaseReg)
359 .addImm(Val: 0)
360 .addReg(Reg: X86::NoRegister),
361 STI);
362 }
363
364 // Emit the access with TPDest as the new base, and the original base
365 // (offset from %fs) as the new index. For example:
366 //
367 // movq %fs:(%rdi), %rax
368 // ->
369 // movq 16(%r15), %rax
370 // movq (%rax,%rdi), %rax
371 MCInst Modified(Inst);
372 Modified.getOperand(i: MemIdx + X86::AddrBaseReg).setReg(TPDest);
373 if (HasBase && !HasIndex)
374 Modified.getOperand(i: MemIdx + X86::AddrIndexReg).setReg(BaseReg);
375 Modified.getOperand(i: MemIdx + X86::AddrSegmentReg).setReg(X86::NoRegister);
376 Out.emitInstruction(Inst: Modified, STI);
377}
378
379void X86::X86MCLFIRewriter::doRewriteInst(const MCInst &Inst, MCStreamer &Out,
380 const MCSubtargetInfo &STI) {
381 if (!STI.hasFeature(Feature: X86::Is64Bit))
382 return error(Inst, Msg: "LFI only supports 64-bit mode");
383
384 if (mayModifyRegister(Inst, Reg: LFIBaseReg) || mayModifyRegister(Inst, Reg: LFITPReg))
385 return error(Inst, Msg: "illegal modification of reserved LFI register");
386
387 if (isSyscall(Inst))
388 return rewriteSyscall(Inst, Out, STI);
389
390 if (isReturn(Inst))
391 return rewriteReturn(Inst, Out, STI);
392
393 if (isDirectCall(Inst))
394 return rewriteDirectCall(Inst, Out, STI);
395
396 // jmpabs is disallowed since it jumps to an absolute address.
397 if (Inst.getOpcode() == X86::JMPABS64i)
398 return error(Inst, Msg: "unsupported branch instruction");
399
400 if (isIndirectBranch(Inst) || isCall(Inst)) {
401 if (!isSupportedIndirectBranch(Inst))
402 return error(Inst, Msg: "unsupported indirect branch");
403 return rewriteIndirectBranch(Inst, Out, STI);
404 }
405
406 if (isFSAccess(Inst))
407 return rewriteFSAccess(Inst, Out, STI);
408
409 Out.emitInstruction(Inst, STI);
410}
411
412bool X86::X86MCLFIRewriter::rewriteInst(const MCInst &Inst, MCStreamer &Out,
413 const MCSubtargetInfo &STI) {
414 // The guard prevents rewrite-recursion when we emit instructions from inside
415 // the rewriter (such instructions should not be rewritten).
416 if (!Enabled || Guard)
417 return false;
418 Guard = true;
419
420 doRewriteInst(Inst, Out, STI);
421
422 Guard = false;
423 return true;
424}
425