1//===- AArch64MCLFIRewriter.cpp ---------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the AArch64MCLFIRewriter class, the AArch64 specific
10// subclass of MCLFIRewriter.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AArch64MCLFIRewriter.h"
15#include "AArch64AddressingModes.h"
16#include "MCTargetDesc/AArch64MCTargetDesc.h"
17#include "Utils/AArch64BaseInfo.h"
18
19#include "llvm/ADT/Twine.h"
20#include "llvm/MC/MCInst.h"
21#include "llvm/MC/MCInstBuilder.h"
22#include "llvm/MC/MCInstrDesc.h"
23#include "llvm/MC/MCInstrInfo.h"
24#include "llvm/MC/MCStreamer.h"
25#include "llvm/MC/MCSubtargetInfo.h"
26#include "llvm/MC/MCSymbol.h"
27#include "llvm/Support/CommandLine.h"
28
29using namespace llvm;
30
31static cl::opt<bool>
32 LFIGuardElim("aarch64-lfi-guard-elim", cl::Hidden,
33 cl::desc("Enable the LFI guard elimination optimization"),
34 cl::init(Val: true));
35
36namespace llvm::AArch64 {
37struct LFIVariantEntry {
38 unsigned Inst;
39 uint8_t AddrMode;
40 uint8_t Log2Size;
41 unsigned RoWInst;
42};
43struct PairVariantEntry {
44 unsigned Inst;
45 bool IsPre;
46 uint8_t Scale;
47 unsigned BaseInst;
48};
49struct SIMDPostEntry {
50 unsigned Inst;
51 uint8_t NaturalOffset;
52 unsigned BaseInst;
53};
54struct MemInfoEntry {
55 unsigned Inst;
56 uint8_t BaseIdx;
57 uint8_t OffsetIdx;
58 bool HasOffset;
59 bool IsPrePost;
60 bool IsLiteral;
61};
62
63// LFI addressing-mode codes (must match AArch64LFI.td's LFI_AM_* defs).
64enum LFIAddrMode : uint8_t {
65 LFI_AM_Ui = 0,
66 LFI_AM_RoW = 1,
67 LFI_AM_RoX = 2,
68 LFI_AM_Pre = 3,
69 LFI_AM_Post = 4,
70};
71
72#define GET_LFIVariantTable_DECL
73#define GET_PairVariantTable_DECL
74#define GET_SIMDPostTable_DECL
75#define GET_MemInfoTable_DECL
76#define GET_LFIVariantTable_IMPL
77#define GET_PairVariantTable_IMPL
78#define GET_SIMDPostTable_IMPL
79#define GET_MemInfoTable_IMPL
80// The LFI tables defined in AArch64LFI.td are emitted into this file alongside
81// the system operand tables (single -gen-searchable-tables output).
82#include "AArch64GenSystemOperands.inc"
83} // namespace llvm::AArch64
84
85// LFI reserved registers.
86static constexpr MCRegister LFIBaseReg = AArch64::X27;
87static constexpr MCRegister LFIAddrReg = AArch64::X28;
88static constexpr MCRegister LFIScratchReg = AArch64::X26;
89static constexpr MCRegister LFICtxReg = AArch64::X25;
90
91// Offset into the context register block (pointed to by LFICtxReg) where the
92// thread pointer is stored. This is a scaled offset (multiplied by 8 for
93// 64-bit loads), so a value of 2 means an actual byte offset of 16.
94static constexpr unsigned LFITPOffset = 2;
95
96// Byte offset from the sandbox base register where the syscall handler address
97// is stored (negative because it is below the sandbox base).
98static constexpr int LFISyscallOffset = -8;
99
100static bool isSyscall(const MCInst &Inst) {
101 return Inst.getOpcode() == AArch64::SVC;
102}
103
104static bool isPrivilegedTP(int64_t Reg) {
105 return Reg == AArch64SysReg::TPIDR_EL1 || Reg == AArch64SysReg::TPIDR_EL2 ||
106 Reg == AArch64SysReg::TPIDR_EL3;
107}
108
109static bool isTPRead(const MCInst &Inst) {
110 return Inst.getOpcode() == AArch64::MRS &&
111 Inst.getOperand(i: 1).getImm() == AArch64SysReg::TPIDR_EL0;
112}
113
114static bool isTPWrite(const MCInst &Inst) {
115 return Inst.getOpcode() == AArch64::MSR &&
116 Inst.getOperand(i: 0).getImm() == AArch64SysReg::TPIDR_EL0;
117}
118
119static bool isPrivilegedTPAccess(const MCInst &Inst) {
120 if (Inst.getOpcode() == AArch64::MRS)
121 return isPrivilegedTP(Reg: Inst.getOperand(i: 1).getImm());
122 if (Inst.getOpcode() == AArch64::MSR)
123 return isPrivilegedTP(Reg: Inst.getOperand(i: 0).getImm());
124 return false;
125}
126
127// Classification functions are limited to Armv8.1-A. Instructions outside of
128// this subset are not guaranteed to be rewritten and as a result may fail LFI
129// verification after compilation.
130
131// Instructions that have mayLoad/mayStore set in TableGen but don't actually
132// perform memory accesses.
133static bool isFakeMemAccess(const MCInst &Inst) {
134 switch (Inst.getOpcode()) {
135 case AArch64::CLREX:
136 case AArch64::DMB:
137 case AArch64::DSB:
138 case AArch64::ISB:
139 case AArch64::HINT:
140 // The range of sub-architectures supported by LFI do not include any load
141 // or store instructions in the HINT space.
142 return true;
143 default:
144 return false;
145 }
146}
147
148static bool mayPrefetch(const MCInst &Inst) {
149 switch (Inst.getOpcode()) {
150 case AArch64::PRFMl:
151 case AArch64::PRFMroW:
152 case AArch64::PRFMroX:
153 case AArch64::PRFMui:
154 case AArch64::PRFUMi:
155 return true;
156 default:
157 return false;
158 }
159}
160
161static bool isAuthenticatedBranch(unsigned Opcode) {
162 switch (Opcode) {
163 case AArch64::BRAA:
164 case AArch64::BRAAZ:
165 case AArch64::BRAB:
166 case AArch64::BRABZ:
167 return true;
168 default:
169 return false;
170 }
171}
172
173static bool isAuthenticatedCall(unsigned Opcode) {
174 switch (Opcode) {
175 case AArch64::BLRAA:
176 case AArch64::BLRAAZ:
177 case AArch64::BLRAB:
178 case AArch64::BLRABZ:
179 return true;
180 default:
181 return false;
182 }
183}
184
185static bool isAuthenticatedReturn(unsigned Opcode) {
186 return Opcode == AArch64::RETAA || Opcode == AArch64::RETAB;
187}
188
189static bool isExceptionReturn(unsigned Opcode) {
190 return Opcode == AArch64::ERET || Opcode == AArch64::ERETAA ||
191 Opcode == AArch64::ERETAB;
192}
193
194static bool pacWritesLR(const MCInst &Inst) {
195 switch (Inst.getOpcode()) {
196 case AArch64::AUTIASP:
197 case AArch64::AUTIBSP:
198 case AArch64::AUTIAZ:
199 case AArch64::AUTIBZ:
200 case AArch64::XPACLRI:
201 return true;
202 default:
203 return false;
204 }
205}
206
207// User-mode DC/IC instructions that take a virtual address operand. Encoded as
208// SYSxt with op1=3, Cn=7, op2=1 where the Cm field selects the operation.
209static bool isVASysOp(const MCInst &Inst) {
210 if (Inst.getOpcode() != AArch64::SYSxt)
211 return false;
212 if (Inst.getOperand(i: 0).getImm() != 3 || Inst.getOperand(i: 1).getImm() != 7 ||
213 Inst.getOperand(i: 3).getImm() != 1)
214 return false;
215 switch (Inst.getOperand(i: 2).getImm()) {
216 case 4: // DC ZVA
217 case 5: // IC IVAU
218 case 10: // DC CVAC
219 case 11: // DC CVAU
220 case 12: // DC CVAP
221 case 13: // DC CVADP
222 case 14: // DC CIVAC
223 return true;
224 default:
225 return false;
226 }
227}
228
229static MCInst replaceRegAt(const MCInst &Inst, unsigned Idx,
230 MCRegister NewReg) {
231 MCInst New = Inst;
232 assert(New.getOperand(Idx).isReg());
233 New.getOperand(i: Idx).setReg(NewReg);
234 return New;
235}
236
237// AArch64 load/store opcode suffixes used throughout this file:
238// Ui: Unsigned immediate offset, scaled by access size: [Xn, #imm].
239// RoW: Register offset with 32-bit W register: [Xn, Wm, uxtw #shift].
240// RoX: Register offset with 64-bit X register: [Xn, Xm, lsl #shift].
241
242// Scalar load/store variant lookup. If Op is a scalar mem instruction with
243// addressing mode ExpectedMode, returns the RoW variant of the same family.
244// Returns INSTRUCTION_LIST_END otherwise.
245static unsigned convertVariantToRoW(unsigned Op, unsigned ExpectedMode) {
246 const AArch64::LFIVariantEntry *E = AArch64::lookupLFIVariantByOpcode(Inst: Op);
247 if (!E || E->AddrMode != ExpectedMode)
248 return AArch64::INSTRUCTION_LIST_END;
249 return E->RoWInst;
250}
251
252static unsigned convertRoXToRoW(unsigned Op, unsigned &Shift) {
253 Shift = 0;
254 const AArch64::LFIVariantEntry *E = AArch64::lookupLFIVariantByOpcode(Inst: Op);
255 if (!E || E->AddrMode != AArch64::LFI_AM_RoX)
256 return AArch64::INSTRUCTION_LIST_END;
257 Shift = E->Log2Size;
258 return E->RoWInst;
259}
260
261static bool getRoWShift(unsigned Op, unsigned &Shift) {
262 Shift = 0;
263 const AArch64::LFIVariantEntry *E = AArch64::lookupLFIVariantByOpcode(Inst: Op);
264 if (!E || E->AddrMode != AArch64::LFI_AM_RoW)
265 return false;
266 Shift = E->Log2Size;
267 return true;
268}
269
270// Pre/post-index conversion to base form. Both LDP/STP pair pre/post forms and
271// SIMD post-index forms come from generated lookup tables. The pair table sets
272// IsPre to distinguish pre-index from post-index. The SIMD table is
273// post-index-only so IsNoOffset is set to indicate the demoted base form takes
274// no immediate offset.
275static unsigned convertPrePostToBase(unsigned Op, bool &IsPre,
276 bool &IsNoOffset) {
277 IsPre = false;
278 IsNoOffset = false;
279 if (const auto *E = AArch64::lookupPairVariantByOpcode(Inst: Op)) {
280 IsPre = E->IsPre;
281 return E->BaseInst;
282 }
283 if (const auto *E = AArch64::lookupSIMDPostByOpcode(Inst: Op)) {
284 IsNoOffset = true;
285 return E->BaseInst;
286 }
287 return AArch64::INSTRUCTION_LIST_END;
288}
289
290bool AArch64MCLFIRewriter::mayModifySP(const MCInst &Inst) const {
291 return mayModifyRegister(Inst, Reg: AArch64::SP);
292}
293
294MCRegister AArch64MCLFIRewriter::mayModifyReserved(const MCInst &Inst) const {
295 for (MCRegister Reg : {LFIAddrReg, LFIBaseReg, LFICtxReg}) {
296 if (mayModifyRegister(Inst, Reg))
297 return Reg;
298 }
299 return {};
300}
301
302void AArch64MCLFIRewriter::onLabel(const MCSymbol *Symbol, MCStreamer &Out) {
303 if (Guard)
304 return;
305
306 // Flush a deferred LR guard before the label, since the label is a potential
307 // branch target and code reached through it may use LR for control flow.
308 if (DeferredLRGuard && LastSTI && !Symbol->isTemporary()) {
309 emitAddMask(Dest: AArch64::LR, Src: AArch64::LR, Out, STI: *LastSTI);
310 DeferredLRGuard = false;
311 }
312
313 // Invalidate guard state since the label is a potential branch target.
314 ActiveGuardReg = std::nullopt;
315}
316
317void AArch64MCLFIRewriter::finish(MCStreamer &Out) {
318 // Flush a deferred LR guard at the end of the stream.
319 if (DeferredLRGuard && LastSTI) {
320 emitAddMask(Dest: AArch64::LR, Src: AArch64::LR, Out, STI: *LastSTI);
321 DeferredLRGuard = false;
322 }
323}
324
325void AArch64MCLFIRewriter::emitInst(const MCInst &Inst, MCStreamer &Out,
326 const MCSubtargetInfo &STI) {
327 // Invalidate the active guard if this instruction modifies the guarded
328 // register, modifies x28 itself, or may affect control flow.
329 if (ActiveGuardReg) {
330 const MCInstrDesc &Desc = InstInfo->get(Opcode: Inst.getOpcode());
331 if (Desc.mayAffectControlFlow(MI: Inst, RI: *RegInfo) ||
332 mayModifyRegister(Inst, Reg: *ActiveGuardReg) ||
333 mayModifyRegister(Inst, Reg: getWRegFromXReg(Reg: *ActiveGuardReg)) ||
334 mayModifyRegister(Inst, Reg: LFIAddrReg))
335 ActiveGuardReg = std::nullopt;
336 }
337
338 Out.emitInstruction(Inst, STI);
339}
340
341void AArch64MCLFIRewriter::emitAddMask(MCRegister Dest, MCRegister Src,
342 MCStreamer &Out,
343 const MCSubtargetInfo &STI) {
344 // If x28 already holds the guarded value of Src, this guard is redundant and
345 // can be skipped.
346 if (LFIGuardElim && Dest == LFIAddrReg && ActiveGuardReg == Src)
347 return;
348
349 // add Dest, LFIBaseReg, W(Src), uxtw
350 emitInst(Inst: MCInstBuilder(AArch64::ADDXrx)
351 .addReg(Reg: Dest)
352 .addReg(Reg: LFIBaseReg)
353 .addReg(Reg: getWRegFromXReg(Reg: Src))
354 .addImm(Val: AArch64_AM::getArithExtendImm(ET: AArch64_AM::UXTW, Imm: 0)),
355 Out, STI);
356
357 // Record Src as the new active guard.
358 if (Dest == LFIAddrReg)
359 ActiveGuardReg = Src;
360}
361
362void AArch64MCLFIRewriter::emitBranch(unsigned Opcode, MCRegister Target,
363 MCStreamer &Out,
364 const MCSubtargetInfo &STI) {
365 emitInst(Inst: MCInstBuilder(Opcode).addReg(Reg: Target), Out, STI);
366}
367
368void AArch64MCLFIRewriter::emitPendingTLSDescCall(MCStreamer &Out,
369 const MCSubtargetInfo &STI) {
370 if (!PendingTLSDescCall)
371 return;
372 const MCExpr *Expr = PendingTLSDescCall;
373 PendingTLSDescCall = nullptr;
374 emitInst(Inst: MCInstBuilder(AArch64::TLSDESCCALL).addExpr(Val: Expr), Out, STI);
375}
376
377void AArch64MCLFIRewriter::emitMov(MCRegister Dest, MCRegister Src,
378 MCStreamer &Out,
379 const MCSubtargetInfo &STI) {
380 // orr Dest, xzr, Src
381 emitInst(Inst: MCInstBuilder(AArch64::ORRXrs)
382 .addReg(Reg: Dest)
383 .addReg(Reg: AArch64::XZR)
384 .addReg(Reg: Src)
385 .addImm(Val: 0),
386 Out, STI);
387}
388
389void AArch64MCLFIRewriter::emitAddImm(MCRegister Dest, MCRegister Src,
390 int64_t Imm, MCStreamer &Out,
391 const MCSubtargetInfo &STI) {
392 assert(std::abs(Imm) <= 4095);
393 // add Dest, Src, Imm (or sub Dest, Src, -Imm for negative offsets)
394 unsigned Opcode = Imm >= 0 ? AArch64::ADDXri : AArch64::SUBXri;
395 emitInst(Inst: MCInstBuilder(Opcode)
396 .addReg(Reg: Dest)
397 .addReg(Reg: Src)
398 .addImm(Val: std::abs(i: Imm))
399 .addImm(Val: 0), // shift
400 Out, STI);
401}
402
403void AArch64MCLFIRewriter::emitAddReg(MCRegister Dest, MCRegister Src1,
404 MCRegister Src2, unsigned Shift,
405 MCStreamer &Out,
406 const MCSubtargetInfo &STI) {
407 // add Dest, Src1, Src2, lsl #Shift
408 emitInst(Inst: MCInstBuilder(AArch64::ADDXrs)
409 .addReg(Reg: Dest)
410 .addReg(Reg: Src1)
411 .addReg(Reg: Src2)
412 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: Shift)),
413 Out, STI);
414}
415
416void AArch64MCLFIRewriter::emitAddRegExtend(MCRegister Dest, MCRegister Src1,
417 MCRegister Src2,
418 AArch64_AM::ShiftExtendType ExtType,
419 unsigned Shift, MCStreamer &Out,
420 const MCSubtargetInfo &STI) {
421 // add Dest, Src1, Src2, ExtType #Shift
422 unsigned Opcode = ExtType == AArch64_AM::SXTX || ExtType == AArch64_AM::UXTX
423 ? AArch64::ADDXrx64
424 : AArch64::ADDXrx;
425 emitInst(Inst: MCInstBuilder(Opcode).addReg(Reg: Dest).addReg(Reg: Src1).addReg(Reg: Src2).addImm(
426 Val: AArch64_AM::getArithExtendImm(ET: ExtType, Imm: Shift)),
427 Out, STI);
428}
429
430void AArch64MCLFIRewriter::emitMemRoW(unsigned Opcode, const MCOperand &DataOp,
431 MCRegister BaseReg, MCStreamer &Out,
432 const MCSubtargetInfo &STI) {
433 // Op DataOp, [LFIBaseReg, W(BaseReg), uxtw]
434 emitInst(Inst: MCInstBuilder(Opcode)
435 .addOperand(Op: DataOp)
436 .addReg(Reg: LFIBaseReg)
437 .addReg(Reg: getWRegFromXReg(Reg: BaseReg))
438 .addImm(Val: 0) // S bit = 0 (UXTW).
439 .addImm(Val: 0), // Shift amount = 0 (unscaled).
440 Out, STI);
441}
442
443// {br,blr} xN
444// ->
445// add x28, x27, wN, uxtw
446// {br,blr} x28
447void AArch64MCLFIRewriter::rewriteIndirectBranch(const MCInst &Inst,
448 MCStreamer &Out,
449 const MCSubtargetInfo &STI) {
450 assert(Inst.getNumOperands() >= 1 && Inst.getOperand(0).isReg() &&
451 "expected register operand");
452 MCRegister BranchReg = Inst.getOperand(i: 0).getReg();
453
454 // Guard the branch target through X28.
455 emitAddMask(Dest: LFIAddrReg, Src: BranchReg, Out, STI);
456
457 emitPendingTLSDescCall(Out, STI);
458
459 emitBranch(Opcode: Inst.getOpcode(), Target: LFIAddrReg, Out, STI);
460}
461
462// ret xN (where xN != x30)
463// ->
464// add x28, x27, wN, uxtw
465// ret x28
466//
467// ret (x30) is safe since x30 is always within the sandbox.
468void AArch64MCLFIRewriter::rewriteReturn(const MCInst &Inst, MCStreamer &Out,
469 const MCSubtargetInfo &STI) {
470 assert(Inst.getNumOperands() >= 1 && Inst.getOperand(0).isReg() &&
471 "expected register operand");
472 // RET through LR is safe since LR is always within sandbox.
473 if (Inst.getOperand(i: 0).getReg() != AArch64::LR)
474 rewriteIndirectBranch(Inst, Out, STI);
475 else
476 emitInst(Inst, Out, STI);
477}
478
479// modify x30
480// ->
481// modify x30
482// add x30, x27, w30, uxtw (deferred)
483void AArch64MCLFIRewriter::rewriteLRModification(const MCInst &Inst,
484 MCStreamer &Out,
485 const MCSubtargetInfo &STI) {
486 if (!isFakeMemAccess(Inst) &&
487 (mayLoad(Inst) || mayStore(Inst) || mayPrefetch(Inst)))
488 rewriteLoadStore(Inst, Out, STI);
489 else
490 emitInst(Inst, Out, STI);
491
492 // Defer the LR guard until the next control-flow instruction or label. This
493 // keeps a signed return address intact so that an authentication instruction
494 // can run before the mask destroys the PAC bits.
495 DeferredLRGuard = true;
496}
497
498// retaa / retab
499// ->
500// autiasp / autibsp
501// add x30, x27, w30, uxtw
502// ret
503void AArch64MCLFIRewriter::rewriteAuthenticatedReturn(
504 const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) {
505 emitInst(Inst: MCInstBuilder(Inst.getOpcode() == AArch64::RETAA ? AArch64::AUTIASP
506 : AArch64::AUTIBSP),
507 Out, STI);
508
509 emitAddMask(Dest: AArch64::LR, Src: AArch64::LR, Out, STI);
510 emitBranch(Opcode: AArch64::RET, Target: AArch64::LR, Out, STI);
511}
512
513// {braa,brab,braaz,brabz} xN[, xM] (blra* for calls)
514// ->
515// {autia,autib,autiza,autizb} xN[, xM]
516// add x28, x27, wN, uxtw
517// {br,blr} x28
518void AArch64MCLFIRewriter::rewriteAuthenticatedBranchOrCall(
519 const MCInst &Inst, unsigned BranchOpcode, MCStreamer &Out,
520 const MCSubtargetInfo &STI) {
521 MCRegister TargetReg = Inst.getOperand(i: 0).getReg();
522
523 // Select the authentication opcode for the target register.
524 unsigned AuthOpcode;
525 switch (Inst.getOpcode()) {
526 case AArch64::BRAA:
527 case AArch64::BLRAA:
528 AuthOpcode = AArch64::AUTIA;
529 break;
530 case AArch64::BRAB:
531 case AArch64::BLRAB:
532 AuthOpcode = AArch64::AUTIB;
533 break;
534 case AArch64::BRAAZ:
535 case AArch64::BLRAAZ:
536 AuthOpcode = AArch64::AUTIZA;
537 break;
538 case AArch64::BRABZ:
539 case AArch64::BLRABZ:
540 AuthOpcode = AArch64::AUTIZB;
541 break;
542 default:
543 llvm_unreachable("unexpected authenticated branch/call opcode");
544 }
545
546 MCInstBuilder Auth(AuthOpcode);
547 Auth.addReg(Reg: TargetReg); // dst
548 Auth.addReg(Reg: TargetReg); // src (tied to dst)
549 if (AuthOpcode == AArch64::AUTIA || AuthOpcode == AArch64::AUTIB)
550 Auth.addOperand(Op: Inst.getOperand(i: 1)); // modifier
551 emitInst(Inst: Auth, Out, STI);
552
553 // Guard the authenticated target and branch/call through x28.
554 emitAddMask(Dest: LFIAddrReg, Src: TargetReg, Out, STI);
555 emitBranch(Opcode: BranchOpcode, Target: LFIAddrReg, Out, STI);
556}
557
558// svc #0
559// ->
560// mov x26, x30
561// ldur x30, [x27, #-8]
562// blr x30
563// add x30, x27, w26, uxtw
564void AArch64MCLFIRewriter::rewriteSyscall(const MCInst &, MCStreamer &Out,
565 const MCSubtargetInfo &STI) {
566 // Save LR to scratch.
567 emitMov(Dest: LFIScratchReg, Src: AArch64::LR, Out, STI);
568
569 // Load syscall handler address from negative offset from sandbox base.
570 emitInst(Inst: MCInstBuilder(AArch64::LDURXi)
571 .addReg(Reg: AArch64::LR)
572 .addReg(Reg: LFIBaseReg)
573 .addImm(Val: LFISyscallOffset),
574 Out, STI);
575
576 // Call the runtime.
577 emitBranch(Opcode: AArch64::BLR, Target: AArch64::LR, Out, STI);
578
579 // Restore LR with guard.
580 emitAddMask(Dest: AArch64::LR, Src: LFIScratchReg, Out, STI);
581}
582
583// mrs xN, tpidr_el0
584// ->
585// ldr xN, [x25, #16]
586void AArch64MCLFIRewriter::rewriteTPRead(const MCInst &Inst, MCStreamer &Out,
587 const MCSubtargetInfo &STI) {
588 MCRegister DestReg = Inst.getOperand(i: 0).getReg();
589
590 emitInst(Inst: MCInstBuilder(AArch64::LDRXui)
591 .addReg(Reg: DestReg)
592 .addReg(Reg: LFICtxReg)
593 .addImm(Val: LFITPOffset),
594 Out, STI);
595}
596
597// msr tpidr_el0, xN
598// ->
599// str xN, [x25, #16]
600void AArch64MCLFIRewriter::rewriteTPWrite(const MCInst &Inst, MCStreamer &Out,
601 const MCSubtargetInfo &STI) {
602 MCRegister SrcReg = Inst.getOperand(i: 1).getReg();
603
604 emitInst(Inst: MCInstBuilder(AArch64::STRXui)
605 .addReg(Reg: SrcReg)
606 .addReg(Reg: LFICtxReg)
607 .addImm(Val: LFITPOffset),
608 Out, STI);
609}
610
611bool AArch64MCLFIRewriter::rewriteLoadStoreRoW(const MCInst &Inst,
612 MCStreamer &Out,
613 const MCSubtargetInfo &STI) {
614 unsigned Op = Inst.getOpcode();
615 unsigned MemOp;
616
617 // Case 1: Indexed load/store with zero immediate offset.
618 // ldr xN, [xM, #0] -> ldr xN, [x27, wM, uxtw]
619 if ((MemOp = convertVariantToRoW(Op, ExpectedMode: AArch64::LFI_AM_Ui)) !=
620 AArch64::INSTRUCTION_LIST_END) {
621 MCRegister BaseReg = Inst.getOperand(i: 1).getReg();
622 if (BaseReg == AArch64::SP)
623 return false;
624 const MCOperand &OffsetOp = Inst.getOperand(i: 2);
625 if (OffsetOp.isImm() && OffsetOp.getImm() == 0) {
626 emitMemRoW(Opcode: MemOp, DataOp: Inst.getOperand(i: 0), BaseReg, Out, STI);
627 return true;
628 }
629 return false;
630 }
631
632 // Case 2: Pre-index load/store with writeback.
633 // ldr xN, [xM, #imm]! -> add xM, xM, #imm; ldr xN, [x27, wM, uxtw]
634 if ((MemOp = convertVariantToRoW(Op, ExpectedMode: AArch64::LFI_AM_Pre)) !=
635 AArch64::INSTRUCTION_LIST_END) {
636 MCRegister BaseReg = Inst.getOperand(i: 2).getReg();
637 if (BaseReg == AArch64::SP)
638 return false;
639 int64_t Imm = Inst.getOperand(i: 3).getImm();
640 emitAddImm(Dest: BaseReg, Src: BaseReg, Imm, Out, STI);
641 emitMemRoW(Opcode: MemOp, DataOp: Inst.getOperand(i: 1), BaseReg, Out, STI);
642 return true;
643 }
644
645 // Case 3: Post-index load/store.
646 // ldr xN, [xM], #imm -> ldr xN, [x27, wM, uxtw]; add xM, xM, #imm
647 if ((MemOp = convertVariantToRoW(Op, ExpectedMode: AArch64::LFI_AM_Post)) !=
648 AArch64::INSTRUCTION_LIST_END) {
649 MCRegister BaseReg = Inst.getOperand(i: 2).getReg();
650 if (BaseReg == AArch64::SP)
651 return false;
652 int64_t Imm = Inst.getOperand(i: 3).getImm();
653 emitMemRoW(Opcode: MemOp, DataOp: Inst.getOperand(i: 1), BaseReg, Out, STI);
654 emitAddImm(Dest: BaseReg, Src: BaseReg, Imm, Out, STI);
655 return true;
656 }
657
658 // Case 4: Register-offset-X load/store.
659 // ldr xN, [xM1, xM2] -> add x26, xM1, xM2; ldr xN, [x27, w26, uxtw]
660 //
661 // In this case, even if xM1 is SP we must do a full rewrite, since an
662 // arbitrary register value is being added as the offset.
663 unsigned Shift;
664 if ((MemOp = convertRoXToRoW(Op, Shift)) != AArch64::INSTRUCTION_LIST_END) {
665 MCRegister Reg1 = Inst.getOperand(i: 1).getReg();
666 MCRegister Reg2 = Inst.getOperand(i: 2).getReg();
667 int64_t Extend = Inst.getOperand(i: 3).getImm();
668 int64_t IsShift = Inst.getOperand(i: 4).getImm();
669
670 if (!IsShift)
671 Shift = 0;
672
673 if (Extend)
674 emitAddRegExtend(Dest: LFIScratchReg, Src1: Reg1, Src2: Reg2, ExtType: AArch64_AM::SXTX, Shift, Out,
675 STI);
676 else
677 emitAddReg(Dest: LFIScratchReg, Src1: Reg1, Src2: Reg2, Shift, Out, STI);
678 emitMemRoW(Opcode: MemOp, DataOp: Inst.getOperand(i: 0), BaseReg: LFIScratchReg, Out, STI);
679 return true;
680 }
681
682 // Case 5: Register-offset-W load/store.
683 // ldr xN, [xM1, wM2, uxtw] -> add x26, xM1, wM2, uxtw;
684 // ldr xN, [x27, w26, uxtw]
685 if (getRoWShift(Op, Shift)) {
686 MCRegister Reg1 = Inst.getOperand(i: 1).getReg();
687 MCRegister Reg2 = Inst.getOperand(i: 2).getReg();
688 int64_t S = Inst.getOperand(i: 3).getImm();
689 int64_t IsShift = Inst.getOperand(i: 4).getImm();
690
691 if (!IsShift)
692 Shift = 0;
693
694 if (S)
695 emitAddRegExtend(Dest: LFIScratchReg, Src1: Reg1, Src2: Reg2, ExtType: AArch64_AM::SXTW, Shift, Out,
696 STI);
697 else
698 emitAddRegExtend(Dest: LFIScratchReg, Src1: Reg1, Src2: Reg2, ExtType: AArch64_AM::UXTW, Shift, Out,
699 STI);
700 emitMemRoW(Opcode: Op, DataOp: Inst.getOperand(i: 0), BaseReg: LFIScratchReg, Out, STI);
701 return true;
702 }
703
704 return false;
705}
706
707void AArch64MCLFIRewriter::rewriteLoadStoreBase(const MCInst &Inst,
708 MCStreamer &Out,
709 const MCSubtargetInfo &STI) {
710 unsigned Opcode = Inst.getOpcode();
711 const AArch64::MemInfoEntry *Info = AArch64::lookupMemInfoByOpcode(Inst: Opcode);
712
713 if (!Info) {
714 warning(Inst, Msg: "unknown addressing mode for memory instruction in LFI");
715 return emitInst(Inst, Out, STI);
716 }
717
718 if (Info->IsLiteral)
719 return error(Inst, Msg: "PC-relative literal loads are not supported in LFI");
720
721 MCRegister BaseReg = Inst.getOperand(i: Info->BaseIdx).getReg();
722
723 // Stack accesses don't need address sandboxing, except when sp is modified
724 // with a non-zero register post-index operand.
725 bool BaseIsSP = BaseReg == AArch64::SP;
726 if (BaseIsSP) {
727 if (!Info->HasOffset || !Inst.getOperand(i: Info->OffsetIdx).isReg())
728 return emitInst(Inst, Out, STI);
729 MCRegister OffReg = Inst.getOperand(i: Info->OffsetIdx).getReg();
730 if (OffReg == AArch64::XZR || OffReg == AArch64::WZR)
731 return emitInst(Inst, Out, STI);
732 }
733
734 // Guard the base register, unless it is SP.
735 if (!BaseIsSP)
736 emitAddMask(Dest: LFIAddrReg, Src: BaseReg, Out, STI);
737
738 if (!Info->IsPrePost) {
739 // Non-pre/post instruction: replace the base register operand.
740 MCInst NewInst = replaceRegAt(Inst, Idx: Info->BaseIdx, NewReg: LFIAddrReg);
741 emitInst(Inst: NewInst, Out, STI);
742 return;
743 }
744
745 bool IsPre = false;
746 bool IsNoOffset = false;
747 unsigned BaseOpcode = convertPrePostToBase(Op: Opcode, IsPre, IsNoOffset);
748
749 if (BaseOpcode == AArch64::INSTRUCTION_LIST_END)
750 return error(Inst, Msg: "unhandled pre/post-index instruction in LFI rewriter");
751
752 // Demote pre/post-index to base indexed form.
753 MCInstBuilder NewInst(BaseOpcode);
754 NewInst.setLoc(Inst.getLoc());
755
756 // Skip writeback operand (operand 0) and copy data operands up to base.
757 for (int I = 1; I < Info->BaseIdx; ++I)
758 NewInst.addOperand(Op: Inst.getOperand(i: I));
759
760 // Add the access base register (LFIAddrReg or SP).
761 NewInst.addReg(Reg: BaseIsSP ? AArch64::SP : LFIAddrReg);
762
763 // For pre-index, include the offset; for post-index, use zero.
764 if (IsPre && Info->HasOffset)
765 NewInst.addOperand(Op: Inst.getOperand(i: Info->OffsetIdx));
766 else if (!IsNoOffset)
767 NewInst.addImm(Val: 0);
768
769 emitInst(Inst: NewInst, Out, STI);
770
771 if (!Info->HasOffset)
772 return;
773
774 // Update the base register with the offset. If the base is SP, a register
775 // offset must be sandboxed (the result is otherwise unbounded), and ADDXrs
776 // cannot take SP, so the extended-register form via the scratch register is
777 // used.
778 const MCOperand &OffsetOp = Inst.getOperand(i: Info->OffsetIdx);
779 if (OffsetOp.isImm()) {
780 // Pair pre/post immediates are scaled by element size; other pre/post
781 // forms (scalar, SIMD) use the raw immediate (scale = 1).
782 int64_t Scale = 1;
783 if (const auto *E = AArch64::lookupPairVariantByOpcode(Inst: Opcode))
784 Scale = E->Scale;
785 int64_t Offset = OffsetOp.getImm() * Scale;
786 emitAddImm(Dest: BaseReg, Src: BaseReg, Imm: Offset, Out, STI);
787 } else if (OffsetOp.isReg()) {
788 // SIMD post-index uses a register offset (XZR for natural offset).
789 MCRegister OffReg = OffsetOp.getReg();
790 if (OffReg == AArch64::XZR) {
791 if (const auto *E = AArch64::lookupSIMDPostByOpcode(Inst: Opcode))
792 emitAddImm(Dest: BaseReg, Src: BaseReg, Imm: E->NaturalOffset, Out, STI);
793 } else if (OffReg != AArch64::WZR) {
794 if (BaseIsSP) {
795 emitAddRegExtend(Dest: LFIScratchReg, Src1: AArch64::SP, Src2: OffReg, ExtType: AArch64_AM::UXTX,
796 Shift: 0, Out, STI);
797 emitAddMask(Dest: AArch64::SP, Src: LFIScratchReg, Out, STI);
798 } else {
799 emitAddReg(Dest: BaseReg, Src1: BaseReg, Src2: OffReg, Shift: 0, Out, STI);
800 }
801 }
802 }
803}
804
805void AArch64MCLFIRewriter::rewriteLoadStore(const MCInst &Inst, MCStreamer &Out,
806 const MCSubtargetInfo &STI) {
807 bool IsStore = mayStore(Inst);
808 bool IsLoad = mayLoad(Inst) || mayPrefetch(Inst);
809
810 bool SkipLoads = STI.hasFeature(Feature: AArch64::FeatureNoLFILoads);
811 bool SkipStores = STI.hasFeature(Feature: AArch64::FeatureNoLFIStores);
812
813 if ((!IsLoad || SkipLoads) && (!IsStore || SkipStores))
814 return emitInst(Inst, Out, STI);
815
816 if (rewriteLoadStoreRoW(Inst, Out, STI))
817 return;
818
819 rewriteLoadStoreBase(Inst, Out, STI);
820}
821
822// modify sp
823// ->
824// modify x26
825// add sp, x27, w26, uxtw
826void AArch64MCLFIRewriter::rewriteSPModification(const MCInst &Inst,
827 MCStreamer &Out,
828 const MCSubtargetInfo &STI) {
829 // Route through rewriteLRModification or rewriteLoadStore for memory
830 // accesses. Those helpers automatically handle dangerous stack modifications
831 // that can happen via register post-index.
832 if (mayLoad(Inst) || mayStore(Inst)) {
833 if (mayModifyRegister(Inst, Reg: AArch64::LR))
834 return rewriteLRModification(Inst, Out, STI);
835 return rewriteLoadStore(Inst, Out, STI);
836 }
837
838 // No stack sandboxing if sandboxing is disabled for both loads and stores.
839 bool SkipLoads = STI.hasFeature(Feature: AArch64::FeatureNoLFILoads);
840 bool SkipStores = STI.hasFeature(Feature: AArch64::FeatureNoLFIStores);
841 if (SkipLoads && SkipStores)
842 return emitInst(Inst, Out, STI);
843
844 // Special case: mov sp, xN -> add sp, x27, wN, uxtw
845 if (Inst.getOpcode() == AArch64::ADDXri && Inst.getOperand(i: 2).getImm() == 0 &&
846 Inst.getOperand(i: 3).getImm() == 0)
847 return emitAddMask(Dest: AArch64::SP, Src: Inst.getOperand(i: 1).getReg(), Out, STI);
848
849 // Redirect SP modification destination to scratch, then sandbox.
850 MCInst ModInst = replaceRegAt(Inst, Idx: 0, NewReg: LFIScratchReg);
851 emitInst(Inst: ModInst, Out, STI);
852 emitAddMask(Dest: AArch64::SP, Src: LFIScratchReg, Out, STI);
853}
854
855// {dc,ic} <op>, xN
856// ->
857// add x28, x27, wN, uxtw
858// {dc,ic} <op>, x28
859void AArch64MCLFIRewriter::rewriteVASysOp(const MCInst &Inst, MCStreamer &Out,
860 const MCSubtargetInfo &STI) {
861 MCRegister AddrReg = Inst.getOperand(i: 4).getReg();
862
863 emitAddMask(Dest: LFIAddrReg, Src: AddrReg, Out, STI);
864
865 emitInst(Inst: MCInstBuilder(AArch64::SYSxt)
866 .addOperand(Op: Inst.getOperand(i: 0))
867 .addOperand(Op: Inst.getOperand(i: 1))
868 .addOperand(Op: Inst.getOperand(i: 2))
869 .addOperand(Op: Inst.getOperand(i: 3))
870 .addReg(Reg: LFIAddrReg),
871 Out, STI);
872}
873
874// NOTE: when adding new rewrites, the size estimates in
875// AArch64InstrInfo::getLFIInstSizeInBytes must be updated to match.
876void AArch64MCLFIRewriter::doRewriteInst(const MCInst &Inst, MCStreamer &Out,
877 const MCSubtargetInfo &STI) {
878 if (Inst.getOpcode() == AArch64::TLSDESCCALL) {
879 PendingTLSDescCall = Inst.getOperand(i: 0).getExpr();
880 return;
881 }
882
883 // Reserved register modification is an error.
884 if (MCRegister Reg = mayModifyReserved(Inst)) {
885 error(Inst, Msg: Twine("illegal modification of reserved LFI register ") +
886 RegInfo->getName(RegNo: Reg));
887 return;
888 }
889
890 // System instructions.
891 if (isSyscall(Inst))
892 return rewriteSyscall(Inst, Out, STI);
893
894 if (isTPRead(Inst))
895 return rewriteTPRead(Inst, Out, STI);
896
897 if (isTPWrite(Inst))
898 return rewriteTPWrite(Inst, Out, STI);
899
900 if (isPrivilegedTPAccess(Inst)) {
901 error(Inst, Msg: "illegal access to privileged thread pointer register");
902 return;
903 }
904
905 if (isVASysOp(Inst))
906 return rewriteVASysOp(Inst, Out, STI);
907
908 if (isExceptionReturn(Opcode: Inst.getOpcode())) {
909 error(Inst, Msg: "exception returns are not supported by LFI");
910 return;
911 }
912
913 // PAC authenticated returns expand to authenticate + guarded RET. The
914 // expansion emits its own LR guard, so discard any deferred guard: masking
915 // before the authentication would corrupt the signed return address.
916 if (isAuthenticatedReturn(Opcode: Inst.getOpcode())) {
917 DeferredLRGuard = false;
918 return rewriteAuthenticatedReturn(Inst, Out, STI);
919 }
920
921 // Flush a deferred LR guard before any control-flow instruction, so that a
922 // modified LR is sandboxed before it can be used to transfer control.
923 if (DeferredLRGuard && (isReturn(Inst) || isIndirectBranch(Inst) ||
924 isCall(Inst) || isBranch(Inst))) {
925 emitAddMask(Dest: AArch64::LR, Src: AArch64::LR, Out, STI);
926 DeferredLRGuard = false;
927 }
928
929 // PAC authenticated branches/calls expand to authenticate + guarded branch.
930 if (isAuthenticatedBranch(Opcode: Inst.getOpcode()))
931 return rewriteAuthenticatedBranchOrCall(Inst, BranchOpcode: AArch64::BR, Out, STI);
932 if (isAuthenticatedCall(Opcode: Inst.getOpcode()))
933 return rewriteAuthenticatedBranchOrCall(Inst, BranchOpcode: AArch64::BLR, Out, STI);
934
935 // Control flow.
936 switch (Inst.getOpcode()) {
937 case AArch64::RET:
938 return rewriteReturn(Inst, Out, STI);
939 case AArch64::BR:
940 case AArch64::BLR:
941 return rewriteIndirectBranch(Inst, Out, STI);
942 }
943
944 // Register modifications that require sandboxing.
945 if (mayModifySP(Inst))
946 return rewriteSPModification(Inst, Out, STI);
947
948 // Link register modification. This covers explicit writes to x30 as well as
949 // PAC instructions that write LR in place, which define LR implicitly.
950 if (explicitlyModifiesRegister(Inst, Reg: AArch64::LR) || pacWritesLR(Inst))
951 return rewriteLRModification(Inst, Out, STI);
952
953 // Memory access.
954 if (!isFakeMemAccess(Inst) &&
955 (mayLoad(Inst) || mayStore(Inst) || mayPrefetch(Inst)))
956 return rewriteLoadStore(Inst, Out, STI);
957
958 emitInst(Inst, Out, STI);
959}
960
961// This function is made available to the size estimator so that it can
962// classify Pre/Post-index instructions.
963bool llvm::isLFIPrePostMemAccess(unsigned Opcode) {
964 if (convertVariantToRoW(Op: Opcode, ExpectedMode: AArch64::LFI_AM_Pre) !=
965 AArch64::INSTRUCTION_LIST_END)
966 return true;
967 if (convertVariantToRoW(Op: Opcode, ExpectedMode: AArch64::LFI_AM_Post) !=
968 AArch64::INSTRUCTION_LIST_END)
969 return true;
970 bool IsPre, IsNoOffset;
971 if (convertPrePostToBase(Op: Opcode, IsPre, IsNoOffset) !=
972 AArch64::INSTRUCTION_LIST_END)
973 return true;
974 return false;
975}
976
977bool AArch64MCLFIRewriter::rewriteInst(const MCInst &Inst, MCStreamer &Out,
978 const MCSubtargetInfo &STI) {
979 // Invalidate guard state if the rewriter was manually disabled.
980 if (!Enabled)
981 ActiveGuardReg = std::nullopt;
982
983 // This recursion guard prevents rewrite-recursion when we emit instructions
984 // from inside the rewriter (such instructions should not be rewritten).
985 if (!Enabled || Guard)
986 return false;
987 Guard = true;
988
989 // Record the subtarget so a deferred LR guard can be emitted from
990 // onLabel/finish, which are not given an MCSubtargetInfo.
991 LastSTI = &STI;
992
993 doRewriteInst(Inst, Out, STI);
994
995 Guard = false;
996 return true;
997}
998