1//===- X86MCLFIRewriter.cpp -------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the X86MCLFIRewriter class, which rewrites X86-64
10// instructions for LFI (Lightweight Fault Isolation) sandboxing.
11//
12//===----------------------------------------------------------------------===//
13
14#include "X86MCLFIRewriter.h"
15#include "X86BaseInfo.h"
16#include "X86MCTargetDesc.h"
17#include "llvm/MC/MCContext.h"
18#include "llvm/MC/MCExpr.h"
19#include "llvm/MC/MCInst.h"
20#include "llvm/MC/MCInstBuilder.h"
21#include "llvm/MC/MCStreamer.h"
22#include "llvm/MC/MCSubtargetInfo.h"
23#include <algorithm>
24#include <optional>
25
26using namespace llvm;
27
28// LFI reserved registers.
29static constexpr MCRegister LFIBaseReg = X86::R14;
30static constexpr MCRegister LFIScratchReg = X86::R11;
31static constexpr MCRegister LFICtxReg = X86::R15;
32
33// Segment register whose base always holds the sandbox base address.
34static constexpr MCRegister LFIBaseSeg = X86::GS;
35
36// Byte offset into the context register file (pointed to by R15) where the
37// thread pointer is stored.
38static constexpr int TPOffset = 16;
39
40// Byte offset from the context register where the syscall handler address is
41// stored.
42static constexpr int SyscallOffset = -8;
43
44static bool isPrefix(const MCInst &Inst, const MCInstrInfo &InstInfo) {
45 return X86II::isPrefix(TSFlags: InstInfo.get(Opcode: Inst.getOpcode()).TSFlags);
46}
47
48// Return true if Inst is a supported stand-alone prefix.
49static bool isSupportedPrefix(const MCInst &Inst) {
50 switch (Inst.getOpcode()) {
51 case X86::FS_PREFIX:
52 case X86::GS_PREFIX:
53 case X86::ADDR16_PREFIX:
54 case X86::ADDR32_PREFIX:
55 return false;
56 default:
57 return true;
58 }
59}
60
61static bool isSyscall(const MCInst &Inst) {
62 return Inst.getOpcode() == X86::SYSCALL;
63}
64
65static bool isDirectCall(const MCInst &Inst) {
66 return Inst.getOpcode() == X86::CALL64pcrel32;
67}
68
69static bool isSupportedIndirectBranch(const MCInst &Inst) {
70 switch (Inst.getOpcode()) {
71 case X86::JMP64r:
72 case X86::JMP64r_NT:
73 case X86::JMP64m:
74 case X86::JMP64m_NT:
75 case X86::CALL64r:
76 case X86::CALL64r_NT:
77 case X86::CALL64m:
78 case X86::CALL64m_NT:
79 return true;
80 default:
81 return false;
82 }
83}
84
85static bool isWriteSegmentBase(unsigned Opcode) {
86 switch (Opcode) {
87 case X86::WRFSBASE:
88 case X86::WRFSBASE64:
89 case X86::WRGSBASE:
90 case X86::WRGSBASE64:
91 return true;
92 default:
93 return false;
94 }
95}
96
97static bool hasNoTrackPrefix(const MCInst &Inst, const MCInstrInfo &InstInfo) {
98 return (InstInfo.get(Opcode: Inst.getOpcode()).TSFlags & X86II::NOTRACK) ||
99 (Inst.getFlags() & X86::IP_HAS_NOTRACK);
100}
101
102// Find the index of the memory operand if it has an %fs segment override.
103// Returns -1 if there is no memory operand or no %fs override.
104static int findFSMemOperand(const MCInst &Inst, const MCInstrInfo &InstInfo) {
105 int MemIdx = X86II::getMemoryOperandIdx(Desc: InstInfo.get(Opcode: Inst.getOpcode()));
106 if (MemIdx < 0)
107 return -1;
108 const MCOperand &Seg = Inst.getOperand(i: MemIdx + X86::AddrSegmentReg);
109 if (Seg.isReg() && Seg.getReg() == X86::FS)
110 return MemIdx;
111 return -1;
112}
113
114// Return true if the instruction reads from Reg.
115static bool readsRegister(const MCInst &Inst, const MCInstrDesc &Desc,
116 MCRegister Reg, const MCRegisterInfo &RI) {
117 for (unsigned I = Desc.getNumDefs(), E = Inst.getNumOperands(); I < E; ++I) {
118 const MCOperand &Op = Inst.getOperand(i: I);
119 if (Op.isReg() && Op.getReg() && RI.regsOverlap(RegA: Op.getReg(), RegB: Reg))
120 return true;
121 }
122 for (MCPhysReg Use : Desc.implicit_uses())
123 if (RI.regsOverlap(RegA: Use, RegB: Reg))
124 return true;
125 return false;
126}
127
128// Return true if Reg is absent or a 64-bit general-purpose register.
129static bool isGR64OrNone(MCRegister Reg) {
130 return Reg == X86::NoRegister ||
131 getX86MCRegisterClass(RC: X86::GR64RegClassID).contains(Reg);
132}
133
134MCRegister X86::X86MCLFIRewriter::getReg32(MCRegister Reg) const {
135 if (getX86MCRegisterClass(RC: X86::GR32RegClassID).contains(Reg))
136 return Reg;
137 return RegInfo->getSubReg(Reg, Idx: X86::sub_32bit);
138}
139
140void X86::X86MCLFIRewriter::emitWithPrefixes(const MCInst &Inst,
141 MCStreamer &Out,
142 const MCSubtargetInfo &STI) {
143 for (const MCInst &Prefix : Prefixes)
144 Out.emitInstruction(Inst: Prefix, STI);
145 Prefixes.clear();
146 Out.emitInstruction(Inst, STI);
147}
148
149void X86::X86MCLFIRewriter::discardPrefixes() {
150 for (const MCInst &Prefix : Prefixes)
151 error(Inst: Prefix, Msg: "unsupported instruction prefix");
152 Prefixes.clear();
153}
154
155// syscall
156// ->
157// .bundle_lock
158// leaq .Ltmp(%rip), %r11
159// jmpq *-8(%r15)
160// .Ltmp:
161// .bundle_unlock
162void X86::X86MCLFIRewriter::rewriteSyscall(const MCInst &Inst, MCStreamer &Out,
163 const MCSubtargetInfo &STI) {
164 Out.emitBundleLock(/*AlignToEnd=*/false, STI);
165
166 MCSymbol *Symbol = Out.getContext().createTempSymbol();
167
168 // leaq .Ltmp(%rip), %r11
169 Out.emitInstruction(
170 Inst: MCInstBuilder(X86::LEA64r)
171 .addReg(Reg: LFIScratchReg)
172 .addReg(Reg: X86::RIP)
173 .addImm(Val: 1)
174 .addReg(Reg: X86::NoRegister)
175 .addExpr(Val: MCSymbolRefExpr::create(Symbol, Ctx&: Out.getContext()))
176 .addReg(Reg: X86::NoRegister),
177 STI);
178
179 // jmpq *-8(%r15)
180 Out.emitInstruction(Inst: MCInstBuilder(X86::JMP64m)
181 .addReg(Reg: LFICtxReg)
182 .addImm(Val: 1)
183 .addReg(Reg: X86::NoRegister)
184 .addImm(Val: SyscallOffset)
185 .addReg(Reg: X86::NoRegister),
186 STI);
187
188 Out.emitLabel(Symbol);
189 Out.emitBundleUnlock(STI);
190}
191
192// andl $-LFIBundleSize, %eX
193// addq %r14, %rX
194void X86::X86MCLFIRewriter::emitSandboxBranchReg(MCRegister Reg,
195 MCStreamer &Out,
196 const MCSubtargetInfo &STI) {
197 MCRegister Reg32 = RegInfo->getSubReg(Reg, Idx: X86::sub_32bit);
198
199 Out.emitInstruction(Inst: MCInstBuilder(X86::AND32ri8)
200 .addReg(Reg: Reg32)
201 .addReg(Reg: Reg32)
202 .addImm(Val: -static_cast<int64_t>(LFIBundleSize)),
203 STI);
204
205 Out.emitInstruction(
206 Inst: MCInstBuilder(X86::ADD64rr).addReg(Reg).addReg(Reg).addReg(Reg: LFIBaseReg),
207 STI);
208}
209
210// Rewrite an indirect jump or call so that it can only target a bundle
211// boundary inside the sandbox.
212//
213// jmpq *%rX
214// ->
215// .bundle_lock
216// andl $-32, %eX
217// addq %r14, %rX
218// jmpq *%rX
219// .bundle_unlock
220//
221// A branch through memory loads its target into the scratch register first,
222// and then dispatches through it.
223//
224// jmpq *(%rdi)
225// ->
226// movq %gs:(%edi), %r11
227// .bundle_lock
228// andl $-32, %r11d
229// addq %r14, %r11
230// jmpq *%r11
231// .bundle_unlock
232void X86::X86MCLFIRewriter::rewriteIndirectBranch(const MCInst &Inst,
233 MCStreamer &Out,
234 const MCSubtargetInfo &STI) {
235 discardPrefixes();
236
237 MCRegister Target;
238 int MemIdx = X86II::getMemoryOperandIdx(Desc: InstInfo->get(Opcode: Inst.getOpcode()));
239 if (MemIdx >= 0) {
240 Target = LFIScratchReg;
241
242 // Construct the load and then apply the rewriter to it.
243 MCInstBuilder Mov(X86::MOV64rm);
244 Mov.addReg(Reg: Target);
245 for (unsigned I = 0; I < X86::AddrNumOperands; ++I)
246 Mov.addOperand(Op: Inst.getOperand(i: MemIdx + I));
247 Mov.setLoc(Inst.getLoc());
248 doRewriteInst(Inst: Mov, Out, STI);
249 } else {
250 Target = Inst.getOperand(i: 0).getReg();
251
252 if (Target == LFIBaseReg || Target == LFICtxReg || Target == X86::RSP)
253 return error(Inst, Msg: "indirect branch through reserved register");
254 }
255
256 Out.emitBundleLock(/*AlignToEnd=*/isCall(Inst), STI);
257
258 emitSandboxBranchReg(Reg: Target, Out, STI);
259
260 MCInst Branch =
261 MCInstBuilder(isCall(Inst) ? X86::CALL64r : X86::JMP64r).addReg(Reg: Target);
262 if (hasNoTrackPrefix(Inst, InstInfo: *InstInfo))
263 Branch.setFlags(Branch.getFlags() | X86::IP_HAS_NOTRACK);
264 Out.emitInstruction(Inst: Branch, STI);
265
266 Out.emitBundleUnlock(STI);
267}
268
269// Direct calls are not rewritten, but must be placed at the end of a bundle
270// so that the return address they push is bundle-aligned.
271void X86::X86MCLFIRewriter::rewriteDirectCall(const MCInst &Inst,
272 MCStreamer &Out,
273 const MCSubtargetInfo &STI) {
274 Out.emitBundleLock(/*AlignToEnd=*/true, STI);
275 emitWithPrefixes(Inst, Out, STI);
276 Out.emitBundleUnlock(STI);
277}
278
279// ret
280// ->
281// popq %r11
282// .bundle_lock
283// andl $-32, %r11d
284// addq %r14, %r11
285// jmpq *%r11
286// .bundle_unlock
287//
288// A return with an immediate additionally pops the immediate off the stack
289// after loading the return address.
290//
291// retq $16
292// ->
293// popq %r11
294// addq $16, %rsp
295// .bundle_lock
296// andl $-32, %r11d
297// addq %r14, %r11
298// jmpq *%r11
299// .bundle_unlock
300void X86::X86MCLFIRewriter::rewriteReturn(const MCInst &Inst, MCStreamer &Out,
301 const MCSubtargetInfo &STI) {
302 if (Inst.getOpcode() != X86::RET64 && Inst.getOpcode() != X86::RETI64)
303 return error(Inst, Msg: "unsupported return instruction");
304
305 discardPrefixes();
306
307 Out.emitInstruction(Inst: MCInstBuilder(X86::POP64r).addReg(Reg: LFIScratchReg), STI);
308
309 if (Inst.getOpcode() == X86::RETI64) {
310 // Return with an immediate is rewritten recursively so that the stack
311 // pointer modification goes through the rewriter.
312 doRewriteInst(Inst: MCInstBuilder(X86::ADD64ri32)
313 .addReg(Reg: X86::RSP)
314 .addReg(Reg: X86::RSP)
315 .addOperand(Op: Inst.getOperand(i: 0))
316 .setLoc(Inst.getLoc()),
317 Out, STI);
318 }
319
320 Out.emitBundleLock(/*AlignToEnd=*/false, STI);
321
322 emitSandboxBranchReg(Reg: LFIScratchReg, Out, STI);
323
324 Out.emitInstruction(Inst: MCInstBuilder(X86::JMP64r).addReg(Reg: LFIScratchReg), STI);
325
326 Out.emitBundleUnlock(STI);
327}
328
329// Emit: movq TPOffset(%r15), %Reg
330static void emitTPLoad(MCRegister Reg, MCStreamer &Out,
331 const MCSubtargetInfo &STI) {
332 Out.emitInstruction(Inst: MCInstBuilder(X86::MOV64rm)
333 .addReg(Reg)
334 .addReg(Reg: LFICtxReg)
335 .addImm(Val: 1)
336 .addReg(Reg: X86::NoRegister)
337 .addImm(Val: TPOffset)
338 .addReg(Reg: X86::NoRegister),
339 STI);
340}
341
342bool X86::X86MCLFIRewriter::isFSAccess(const MCInst &Inst) {
343 return (mayLoad(Inst) || mayStore(Inst)) &&
344 findFSMemOperand(Inst, InstInfo: *InstInfo) >= 0;
345}
346
347// Rewrite %fs-segment memory accesses to use the virtual thread pointer stored
348// at TPOffset(%r15). The rewritten access is fed back through the rewriter so
349// that it is sandboxed like other memory accesses. Example rewrites:
350//
351// movq %fs:0, %rax
352// ->
353// movq 16(%r15), %rax
354//
355// movq %fs:(%rdi), %rax
356// ->
357// movq 16(%r15), %rax
358// movq %gs:(%eax,%edi), %rax
359//
360// movq %fs:8(%rdi, %rsi, 2), %rax
361// ->
362// movq 16(%r15), %rax
363// leaq (%rax, %rdi), %rax
364// movq %gs:8(%eax, %esi, 2), %rax
365void X86::X86MCLFIRewriter::rewriteFSAccess(const MCInst &Inst, MCStreamer &Out,
366 const MCSubtargetInfo &STI) {
367 int MemIdx = findFSMemOperand(Inst, InstInfo: *InstInfo);
368 assert(MemIdx >= 0 && "expected an %fs memory operand");
369
370 MCRegister BaseReg = Inst.getOperand(i: MemIdx + X86::AddrBaseReg).getReg();
371 MCRegister IndexReg = Inst.getOperand(i: MemIdx + X86::AddrIndexReg).getReg();
372 bool HasBase = BaseReg != X86::NoRegister;
373 bool HasIndex = IndexReg != X86::NoRegister;
374 bool HasDisp = !Inst.getOperand(i: MemIdx + X86::AddrDisp).isImm() ||
375 Inst.getOperand(i: MemIdx + X86::AddrDisp).getImm() != 0;
376
377 // %fs:0 -> TPOffset(%r15). The context register always points into the
378 // runtime's thread-local block, so this access is safe.
379 if (!HasBase && !HasIndex && !HasDisp) {
380 MCInst Modified(Inst);
381 Modified.getOperand(i: MemIdx + X86::AddrBaseReg).setReg(LFICtxReg);
382 Modified.getOperand(i: MemIdx + X86::AddrDisp).setImm(TPOffset);
383 Modified.getOperand(i: MemIdx + X86::AddrSegmentReg).setReg(X86::NoRegister);
384 return emitWithPrefixes(Inst: Modified, Out, STI);
385 }
386
387 if (!isGR64OrNone(Reg: BaseReg) || !isGR64OrNone(Reg: IndexReg) ||
388 BaseReg == X86::RSP || BaseReg == X86::RIP)
389 return error(Inst, Msg: "unsupported addressing mode for %fs access");
390
391 const MCInstrDesc &Desc = InstInfo->get(Opcode: Inst.getOpcode());
392
393 // Reuse operand 0 as the TP temporary when the instruction writes it without
394 // also reading it, otherwise use %r11.
395 MCRegister TPDest = LFIScratchReg;
396 if (MemIdx > 0 && Inst.getOperand(i: 0).isReg()) {
397 MCRegister DestReg = Inst.getOperand(i: 0).getReg();
398 if (Desc.getNumDefs() > 0 &&
399 getX86MCRegisterClass(RC: X86::GR64RegClassID).contains(Reg: DestReg) &&
400 !readsRegister(Inst, Desc, Reg: DestReg, RI: *RegInfo))
401 TPDest = DestReg;
402 }
403
404 if (TPDest == LFIScratchReg &&
405 readsRegister(Inst, Desc, Reg: LFIScratchReg, RI: *RegInfo))
406 return error(Inst, Msg: "%fs access reads reserved register %r11");
407
408 emitTPLoad(Reg: TPDest, Out, STI);
409
410 // Both slots occupied: the compute base via lea. For example:
411 //
412 // movq %fs:8(%rdi,%rsi,2), %rax
413 // ->
414 // movq 16(%r15), %rax
415 // leaq (%rax,%rdi), %rax
416 // movq %gs:8(%eax,%esi,2), %rax
417 if (HasBase && HasIndex) {
418 Out.emitInstruction(Inst: MCInstBuilder(X86::LEA64r)
419 .addReg(Reg: TPDest)
420 .addReg(Reg: TPDest)
421 .addImm(Val: 1)
422 .addReg(Reg: BaseReg)
423 .addImm(Val: 0)
424 .addReg(Reg: X86::NoRegister),
425 STI);
426 }
427
428 // Emit the access with TPDest as the new base, and the original base
429 // (offset from %fs) as the new index. For example:
430 //
431 // movq %fs:(%rdi), %rax
432 // ->
433 // movq 16(%r15), %rax
434 // movq %gs:(%eax,%edi), %rax
435 MCInst Modified(Inst);
436 Modified.getOperand(i: MemIdx + X86::AddrBaseReg).setReg(TPDest);
437 if (HasBase && !HasIndex)
438 Modified.getOperand(i: MemIdx + X86::AddrIndexReg).setReg(BaseReg);
439 Modified.getOperand(i: MemIdx + X86::AddrSegmentReg).setReg(X86::NoRegister);
440 doRewriteInst(Inst: Modified, Out, STI);
441}
442
443// Return true if Reg always contains an address within the sandbox.
444static bool isSandboxBaseReg(MCRegister Reg) {
445 return Reg == LFIBaseReg || Reg == X86::RSP || Reg == X86::RIP;
446}
447
448// Return true if Inst reads or writes memory through a pointer register
449// instead of an addressing mode, and so cannot be sandboxed by rewriting its
450// operands.
451static bool hasImplicitMemPointer(unsigned Opcode) {
452 switch (Opcode) {
453 // xlat loads from (%rbx,%al).
454 case X86::XLAT:
455 // maskmov stores to (%rdi).
456 case X86::MASKMOVDQU:
457 case X86::MASKMOVDQU64:
458 case X86::VMASKMOVDQU:
459 case X86::VMASKMOVDQU64:
460 case X86::MMX_MASKMOVQ:
461 case X86::MMX_MASKMOVQ64:
462 // clzero zeroes the cache line at (%rax).
463 case X86::CLZERO32r:
464 case X86::CLZERO64r:
465 // movdir64b and enqcmd store to the address in their register operand.
466 case X86::MOVDIR64B32:
467 case X86::MOVDIR64B64:
468 case X86::MOVDIR64B32_EVEX:
469 case X86::MOVDIR64B64_EVEX:
470 case X86::ENQCMD32:
471 case X86::ENQCMD64:
472 case X86::ENQCMD32_EVEX:
473 case X86::ENQCMD64_EVEX:
474 case X86::ENQCMDS32:
475 case X86::ENQCMDS64:
476 case X86::ENQCMDS32_EVEX:
477 case X86::ENQCMDS64_EVEX:
478 return true;
479 default:
480 return false;
481 }
482}
483
484// Return true if Inst is a bit test of memory with a register bit offset. The
485// offset is added to the address of the memory operand, so the bit it accesses
486// may be beyond the address computed by the addressing mode.
487static bool isBitTestWithRegOffset(unsigned Opcode) {
488 switch (Opcode) {
489 case X86::BT16mr:
490 case X86::BT32mr:
491 case X86::BT64mr:
492 case X86::BTC16mr:
493 case X86::BTC32mr:
494 case X86::BTC64mr:
495 case X86::BTR16mr:
496 case X86::BTR32mr:
497 case X86::BTR64mr:
498 case X86::BTS16mr:
499 case X86::BTS32mr:
500 case X86::BTS64mr:
501 case X86::LOCK_BTC_RM16rm:
502 case X86::LOCK_BTC_RM32rm:
503 case X86::LOCK_BTC_RM64rm:
504 case X86::LOCK_BTR_RM16rm:
505 case X86::LOCK_BTR_RM32rm:
506 case X86::LOCK_BTR_RM64rm:
507 case X86::LOCK_BTS_RM16rm:
508 case X86::LOCK_BTS_RM32rm:
509 case X86::LOCK_BTS_RM64rm:
510 return true;
511 default:
512 return false;
513 }
514}
515
516// Return true if Inst has a memory operand that is not dereferenced.
517static bool ignoresMemOperand(unsigned Opcode) {
518 switch (Opcode) {
519 case X86::LEA16r:
520 case X86::LEA32r:
521 case X86::LEA64r:
522 case X86::LEA64_32r:
523 case X86::NOOPW:
524 case X86::NOOPL:
525 case X86::NOOPQ:
526 return true;
527 default:
528 return false;
529 }
530}
531
532// Return true if the index register of the memory operand at MemIdx is a
533// general-purpose register, and so can be truncated along with the base.
534static bool hasGPRIndex(const MCInst &Inst, int MemIdx) {
535 MCRegister Index = Inst.getOperand(i: MemIdx + X86::AddrIndexReg).getReg();
536 return Index && getX86MCRegisterClass(RC: X86::GR64RegClassID).contains(Reg: Index);
537}
538
539// Return true if the memory operand at MemIdx uses a 32-bit address size,
540// which is selected by a 32-bit base or index register.
541static bool has32BitAddress(const MCInst &Inst, int MemIdx) {
542 MCRegister Base = Inst.getOperand(i: MemIdx + X86::AddrBaseReg).getReg();
543 MCRegister Index = Inst.getOperand(i: MemIdx + X86::AddrIndexReg).getReg();
544 const MCRegisterClass &GR32 = getX86MCRegisterClass(RC: X86::GR32RegClassID);
545 return Base == X86::EIP || GR32.contains(Reg: Base) || Index == X86::EIZ ||
546 GR32.contains(Reg: Index);
547}
548
549// Rewrite the addressing mode of the memory operand at MemIdx so that the
550// access it performs stays inside the sandbox.
551//
552// movq 8(%rax,%rcx,4), %rdi
553// ->
554// movq %gs:8(%eax,%ecx,4), %rdi
555void X86::X86MCLFIRewriter::sandboxMemOperand(MCInst &Inst, int MemIdx) const {
556 MCOperand &Base = Inst.getOperand(i: MemIdx + X86::AddrBaseReg);
557 MCOperand &Scale = Inst.getOperand(i: MemIdx + X86::AddrScaleAmt);
558 MCOperand &Index = Inst.getOperand(i: MemIdx + X86::AddrIndexReg);
559 MCOperand &Segment = Inst.getOperand(i: MemIdx + X86::AddrSegmentReg);
560
561 // %riz only encodes the absence of an index register, so drop it and handle
562 // the access as one without an index.
563 if (Index.getReg() == X86::RIZ) {
564 Index.setReg(X86::NoRegister);
565 Scale.setImm(1);
566 }
567
568 // A base register that always points into the sandbox needs no rewriting, as
569 // long as there is no index register.
570 if (isSandboxBaseReg(Reg: Base.getReg()) && !Index.getReg())
571 return;
572
573 // An absolute address has no register to truncate, so make it relative to
574 // the sandbox base instead.
575 if (!Base.getReg() && !Index.getReg()) {
576 Base.setReg(LFIBaseReg);
577 return;
578 }
579
580 // If there is no Base and the Index is not a GPR, then we use r14d (all
581 // zeroes) as the base for the rewritten addressing mode.
582 bool GPRIndex = hasGPRIndex(Inst, MemIdx);
583
584 Segment.setReg(LFIBaseSeg);
585 if (Base.getReg())
586 Base.setReg(getReg32(Reg: Base.getReg()));
587 else if (!GPRIndex)
588 Base.setReg(getReg32(Reg: LFIBaseReg));
589 if (GPRIndex)
590 Index.setReg(getReg32(Reg: Index.getReg()));
591}
592
593void X86::X86MCLFIRewriter::sandboxMemAccess(MCInst &Inst) const {
594 if (ignoresMemOperand(Opcode: Inst.getOpcode()))
595 return;
596
597 int MemIdx = X86II::getMemoryOperandIdx(Desc: InstInfo->get(Opcode: Inst.getOpcode()));
598 if (MemIdx >= 0)
599 sandboxMemOperand(Inst, MemIdx);
600}
601
602void X86::X86MCLFIRewriter::rewriteMemAccess(const MCInst &Inst,
603 MCStreamer &Out,
604 const MCSubtargetInfo &STI) {
605 MCInst Sandboxed(Inst);
606 sandboxMemAccess(Inst&: Sandboxed);
607 emitWithPrefixes(Inst: Sandboxed, Out, STI);
608}
609
610namespace {
611struct StringOpPointers {
612 bool UsesDest;
613 bool UsesSrc;
614};
615} // namespace
616
617static std::optional<StringOpPointers> getStringOpPointers(unsigned Opcode) {
618 switch (Opcode) {
619 case X86::STOSB:
620 case X86::STOSW:
621 case X86::STOSL:
622 case X86::STOSQ:
623 case X86::REP_STOSB_64:
624 case X86::REP_STOSW_64:
625 case X86::REP_STOSD_64:
626 case X86::REP_STOSQ_64:
627 case X86::SCASB:
628 case X86::SCASW:
629 case X86::SCASL:
630 case X86::SCASQ:
631 return StringOpPointers{.UsesDest: true, .UsesSrc: false};
632 case X86::MOVSB:
633 case X86::MOVSW:
634 case X86::MOVSL:
635 case X86::MOVSQ:
636 case X86::REP_MOVSB_64:
637 case X86::REP_MOVSW_64:
638 case X86::REP_MOVSD_64:
639 case X86::REP_MOVSQ_64:
640 case X86::CMPSB:
641 case X86::CMPSW:
642 case X86::CMPSL:
643 case X86::CMPSQ:
644 return StringOpPointers{.UsesDest: true, .UsesSrc: true};
645 case X86::LODSB:
646 case X86::LODSW:
647 case X86::LODSL:
648 case X86::LODSQ:
649 return StringOpPointers{.UsesDest: false, .UsesSrc: true};
650 default:
651 return std::nullopt;
652 }
653}
654
655static void emitStringPtrFixup(MCRegister Reg, MCRegister Reg32,
656 MCStreamer &Out, const MCSubtargetInfo &STI) {
657 Out.emitInstruction(Inst: MCInstBuilder(X86::MOV32rr).addReg(Reg: Reg32).addReg(Reg: Reg32),
658 STI);
659
660 Out.emitInstruction(Inst: MCInstBuilder(X86::LEA64r)
661 .addReg(Reg)
662 .addReg(Reg: LFIBaseReg)
663 .addImm(Val: 1)
664 .addReg(Reg)
665 .addImm(Val: 0)
666 .addReg(Reg: X86::NoRegister),
667 STI);
668}
669
670// rep movsq
671// ->
672// .bundle_lock
673// movl %edi, %edi
674// leaq (%r14,%rdi), %rdi
675// movl %esi, %esi
676// leaq (%r14,%rsi), %rsi
677// rep movsq
678// .bundle_unlock
679void X86::X86MCLFIRewriter::rewriteStringOperation(const MCInst &Inst,
680 MCStreamer &Out,
681 const MCSubtargetInfo &STI) {
682 std::optional<StringOpPointers> Ptrs = getStringOpPointers(Opcode: Inst.getOpcode());
683 assert(Ptrs && "expected a string instruction");
684
685 // The guards below sandbox the full 64-bit %rdi and %rsi, so the access must
686 // use them with a 64-bit address size and no %fs segment override.
687 for (const MCOperand &Op : Inst) {
688 if (!Op.isReg())
689 continue;
690 if (Op.getReg() == X86::EDI || Op.getReg() == X86::ESI)
691 return error(Inst, Msg: "unsupported address size for string instruction");
692 if (Op.getReg() == X86::FS)
693 return error(Inst, Msg: "unsupported segment override for string instruction");
694 }
695
696 Out.emitBundleLock(/*AlignToEnd=*/false, STI);
697
698 if (Ptrs->UsesDest)
699 emitStringPtrFixup(Reg: X86::RDI, Reg32: X86::EDI, Out, STI);
700 if (Ptrs->UsesSrc)
701 emitStringPtrFixup(Reg: X86::RSI, Reg32: X86::ESI, Out, STI);
702
703 emitWithPrefixes(Inst, Out, STI);
704
705 Out.emitBundleUnlock(STI);
706}
707
708// The register form of an instruction that modifies the stack and reads its
709// source from memory, if there is one.
710static std::optional<unsigned> memToRegOpcode(unsigned Opcode) {
711 switch (Opcode) {
712 case X86::MOV64rm:
713 return X86::MOV64rr;
714 case X86::ADD64rm:
715 return X86::ADD64rr;
716 case X86::SUB64rm:
717 return X86::SUB64rr;
718 case X86::AND64rm:
719 return X86::AND64rr;
720 case X86::OR64rm:
721 return X86::OR64rr;
722 case X86::XOR64rm:
723 return X86::XOR64rr;
724 default:
725 return std::nullopt;
726 }
727}
728
729// The 32-bit form of a 64-bit instruction, if it has one that can be used to
730// modify the stack pointer.
731static std::optional<unsigned> demote64BitOpcode(unsigned Opcode) {
732 switch (Opcode) {
733 case X86::MOV64rr:
734 return X86::MOV32rr;
735 case X86::MOV64ri32:
736 return X86::MOV32ri;
737 case X86::LEA64r:
738 return X86::LEA64_32r;
739 case X86::ADD64rr:
740 return X86::ADD32rr;
741 case X86::ADD64ri8:
742 return X86::ADD32ri8;
743 case X86::ADD64ri32:
744 return X86::ADD32ri;
745 case X86::SUB64rr:
746 return X86::SUB32rr;
747 case X86::SUB64ri8:
748 return X86::SUB32ri8;
749 case X86::SUB64ri32:
750 return X86::SUB32ri;
751 case X86::AND64rr:
752 return X86::AND32rr;
753 case X86::AND64ri8:
754 return X86::AND32ri8;
755 case X86::AND64ri32:
756 return X86::AND32ri;
757 case X86::OR64rr:
758 return X86::OR32rr;
759 case X86::OR64ri8:
760 return X86::OR32ri8;
761 case X86::OR64ri32:
762 return X86::OR32ri;
763 case X86::XOR64rr:
764 return X86::XOR32rr;
765 case X86::XOR64ri8:
766 return X86::XOR32ri8;
767 case X86::XOR64ri32:
768 return X86::XOR32ri;
769 default:
770 return std::nullopt;
771 }
772}
773
774MCRegister X86::X86MCLFIRewriter::getWrittenStackReg(const MCInst &Inst) const {
775 const MCInstrDesc &Desc = InstInfo->get(Opcode: Inst.getOpcode());
776 for (unsigned I = 0, E = Desc.getNumDefs(); I < E; ++I) {
777 const MCOperand &Op = Inst.getOperand(i: I);
778 if (Op.isReg() && RegInfo->regsOverlap(RegA: Op.getReg(), RegB: X86::RSP))
779 return Op.getReg();
780 }
781 return X86::NoRegister;
782}
783
784// Emit leaq (%rsp,%r14), %rsp
785static void emitStackFixup(MCStreamer &Out, const MCSubtargetInfo &STI) {
786 Out.emitInstruction(Inst: MCInstBuilder(X86::LEA64r)
787 .addReg(Reg: X86::RSP)
788 .addReg(Reg: X86::RSP)
789 .addImm(Val: 1)
790 .addReg(Reg: LFIBaseReg)
791 .addImm(Val: 0)
792 .addReg(Reg: X86::NoRegister),
793 STI);
794}
795
796// addq $8, %rsp
797// ->
798// .bundle_lock
799// addl $8, %esp
800// leaq (%rsp,%r14), %rsp
801// .bundle_unlock
802void X86::X86MCLFIRewriter::rewriteStackModification(
803 MCRegister StackReg, const MCInst &Inst, MCStreamer &Out,
804 const MCSubtargetInfo &STI) {
805 if (StackReg != X86::RSP)
806 return error(Inst, Msg: "unsupported modification of the stack pointer");
807
808 discardPrefixes();
809
810 // popq %rsp loads through the stack pointer that it overwrites, so it cannot
811 // simply be demoted. Pop into the scratch register instead, and move the
812 // result into %esp.
813 if (Inst.getOpcode() == X86::POP64r) {
814 Out.emitInstruction(Inst: MCInstBuilder(X86::POP64r).addReg(Reg: LFIScratchReg), STI);
815
816 Out.emitBundleLock(/*AlignToEnd=*/false, STI);
817 Out.emitInstruction(
818 Inst: MCInstBuilder(X86::MOV32rr).addReg(Reg: X86::ESP).addReg(Reg: X86::R11D), STI);
819 emitStackFixup(Out, STI);
820 Out.emitBundleUnlock(STI);
821 return;
822 }
823
824 // movq 8(%rdi), %rsp
825 // ->
826 // movq %gs:8(%edi), %r11
827 // .bundle_lock
828 // movl %r11d, %esp
829 // leaq (%rsp,%r14), %rsp
830 // .bundle_unlock
831 int MemIdx = X86II::getMemoryOperandIdx(Desc: InstInfo->get(Opcode: Inst.getOpcode()));
832 if (MemIdx >= 0 && !ignoresMemOperand(Opcode: Inst.getOpcode())) {
833 std::optional<unsigned> RegOpcode = memToRegOpcode(Opcode: Inst.getOpcode());
834 if (!RegOpcode)
835 return error(Inst, Msg: "unsupported modification of the stack pointer");
836
837 MCInstBuilder Load(X86::MOV64rm);
838 Load.addReg(Reg: LFIScratchReg);
839 for (unsigned I = 0; I < X86::AddrNumOperands; ++I)
840 Load.addOperand(Op: Inst.getOperand(i: MemIdx + I));
841 Load.setLoc(Inst.getLoc());
842 doRewriteInst(Inst: Load, Out, STI);
843
844 MCInstBuilder RegForm(*RegOpcode);
845 for (int I = 0; I < MemIdx; ++I)
846 RegForm.addOperand(Op: Inst.getOperand(i: I));
847 RegForm.addReg(Reg: LFIScratchReg);
848 RegForm.setLoc(Inst.getLoc());
849 return doRewriteInst(Inst: RegForm, Out, STI);
850 }
851
852 std::optional<unsigned> Opcode = demote64BitOpcode(Opcode: Inst.getOpcode());
853 if (!Opcode)
854 return error(Inst, Msg: "unsupported modification of the stack pointer");
855
856 MCInst Modified(Inst);
857 Modified.setOpcode(*Opcode);
858
859 ArrayRef<MCOperandInfo> OpInfo = InstInfo->get(Opcode: *Opcode).operands();
860 for (unsigned I = 0, E = std::min<unsigned>(a: OpInfo.size(),
861 b: Modified.getNumOperands());
862 I < E; ++I) {
863 MCOperand &Op = Modified.getOperand(i: I);
864 if (OpInfo[I].OperandType != MCOI::OPERAND_REGISTER || !Op.isReg())
865 continue;
866 if (getX86MCRegisterClass(RC: X86::GR64RegClassID).contains(Reg: Op.getReg()))
867 Op.setReg(RegInfo->getSubReg(Reg: Op.getReg(), Idx: X86::sub_32bit));
868 }
869
870 Out.emitBundleLock(/*AlignToEnd=*/false, STI);
871 Out.emitInstruction(Inst: Modified, STI);
872 emitStackFixup(Out, STI);
873 Out.emitBundleUnlock(STI);
874}
875
876// leave
877// ->
878// .bundle_lock
879// movl %ebp, %esp
880// leaq (%rsp,%r14), %rsp
881// .bundle_unlock
882// popq %rbp
883void X86::X86MCLFIRewriter::rewriteLeave(const MCInst &Inst, MCStreamer &Out,
884 const MCSubtargetInfo &STI) {
885 doRewriteInst(Inst: MCInstBuilder(X86::MOV64rr)
886 .addReg(Reg: X86::RSP)
887 .addReg(Reg: X86::RBP)
888 .setLoc(Inst.getLoc()),
889 Out, STI);
890 Out.emitInstruction(Inst: MCInstBuilder(X86::POP64r).addReg(Reg: X86::RBP), STI);
891}
892
893void X86::X86MCLFIRewriter::doRewriteInst(const MCInst &Inst, MCStreamer &Out,
894 const MCSubtargetInfo &STI) {
895 if (!STI.hasFeature(Feature: X86::Is64Bit))
896 return error(Inst, Msg: "LFI only supports 64-bit mode");
897
898 // Hold a stand-alone prefix back, so it can be applied to the rewritten
899 // instruction.
900 if (isPrefix(Inst, InstInfo: *InstInfo)) {
901 if (!isSupportedPrefix(Inst))
902 return error(Inst, Msg: "unsupported instruction prefix");
903 Prefixes.push_back(Elt: Inst);
904 return;
905 }
906
907 // A lock prefix is only valid on an instruction that writes memory, and
908 // would otherwise be dropped by the rewrites below.
909 if ((Inst.getFlags() & X86::IP_HAS_LOCK) && !mayStore(Inst))
910 return error(Inst, Msg: "unsupported lock prefix");
911
912 if (mayModifyRegister(Inst, Reg: LFIBaseReg) ||
913 mayModifyRegister(Inst, Reg: LFICtxReg) || mayModifyRegister(Inst, Reg: LFIBaseSeg))
914 return error(Inst, Msg: "illegal modification of reserved LFI register");
915
916 if (isWriteSegmentBase(Opcode: Inst.getOpcode()))
917 return error(Inst, Msg: "illegal modification of reserved segment base");
918
919 if (isSyscall(Inst))
920 return rewriteSyscall(Inst, Out, STI);
921
922 if (isReturn(Inst))
923 return rewriteReturn(Inst, Out, STI);
924
925 if (isDirectCall(Inst))
926 return rewriteDirectCall(Inst, Out, STI);
927
928 // jmpabs is disallowed since it jumps to an absolute address.
929 if (Inst.getOpcode() == X86::JMPABS64i)
930 return error(Inst, Msg: "unsupported branch instruction");
931
932 if (isIndirectBranch(Inst) || isCall(Inst)) {
933 if (!isSupportedIndirectBranch(Inst))
934 return error(Inst, Msg: "unsupported indirect branch");
935 return rewriteIndirectBranch(Inst, Out, STI);
936 }
937
938 for (const MCOperand &Op : Inst)
939 if (Op.isReg() && Op.getReg() == LFIBaseSeg)
940 return error(Inst, Msg: "invalid use of reserved segment register %gs");
941
942 if (MCRegister StackReg = getWrittenStackReg(Inst))
943 return rewriteStackModification(StackReg, Inst, Out, STI);
944
945 // These accesses cannot be sandboxed by rewriting the addressing mode. The
946 // moffs form of mov encodes an absolute address in the instruction instead
947 // of an addressing mode.
948 if ((InstInfo->get(Opcode: Inst.getOpcode()).TSFlags & X86II::FormMask) ==
949 X86II::RawFrmMemOffs ||
950 hasImplicitMemPointer(Opcode: Inst.getOpcode()))
951 return error(Inst, Msg: "unsupported memory access");
952
953 if (isBitTestWithRegOffset(Opcode: Inst.getOpcode()))
954 return error(Inst, Msg: "unsupported bit test with a register bit offset");
955
956 if (isFSAccess(Inst))
957 return rewriteFSAccess(Inst, Out, STI);
958
959 // The compiler never generates 32-bit addressing in LP64 mode.
960 int MemIdx = X86II::getMemoryOperandIdx(Desc: InstInfo->get(Opcode: Inst.getOpcode()));
961 if (MemIdx >= 0 && !ignoresMemOperand(Opcode: Inst.getOpcode()) &&
962 has32BitAddress(Inst, MemIdx))
963 return error(Inst, Msg: "unsupported 32-bit address size");
964
965 if (getStringOpPointers(Opcode: Inst.getOpcode()))
966 return rewriteStringOperation(Inst, Out, STI);
967
968 if (Inst.getOpcode() == X86::LEAVE64)
969 return rewriteLeave(Inst, Out, STI);
970
971 if (Inst.getOpcode() == X86::ENTER &&
972 (!Inst.getOperand(i: 1).isImm() ||
973 static_cast<uint64_t>(Inst.getOperand(i: 1).getImm()) > 1))
974 return error(Inst, Msg: "unsupported nesting level for enter");
975
976 rewriteMemAccess(Inst, Out, STI);
977}
978
979bool X86::X86MCLFIRewriter::rewriteInst(const MCInst &Inst, MCStreamer &Out,
980 const MCSubtargetInfo &STI) {
981 // The guard prevents rewrite-recursion when we emit instructions from inside
982 // the rewriter (such instructions should not be rewritten).
983 if (!Enabled || Guard)
984 return false;
985 Guard = true;
986
987 size_t PrefixesBefore = Prefixes.size();
988
989 doRewriteInst(Inst, Out, STI);
990
991 if (Prefixes.size() == PrefixesBefore)
992 discardPrefixes();
993
994 Guard = false;
995 return true;
996}
997
998void X86::X86MCLFIRewriter::onLabel(const MCSymbol *Symbol, MCStreamer &Out) {
999 discardPrefixes();
1000}
1001
1002void X86::X86MCLFIRewriter::finish(MCStreamer &Out) { discardPrefixes(); }
1003