1//===- AArch64MCLFIRewriter.cpp ---------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the AArch64MCLFIRewriter class, the AArch64 specific
10// subclass of MCLFIRewriter.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AArch64MCLFIRewriter.h"
15#include "AArch64AddressingModes.h"
16#include "MCTargetDesc/AArch64MCTargetDesc.h"
17#include "Utils/AArch64BaseInfo.h"
18
19#include "llvm/ADT/Twine.h"
20#include "llvm/MC/MCInst.h"
21#include "llvm/MC/MCInstBuilder.h"
22#include "llvm/MC/MCInstrDesc.h"
23#include "llvm/MC/MCInstrInfo.h"
24#include "llvm/MC/MCStreamer.h"
25#include "llvm/MC/MCSubtargetInfo.h"
26#include "llvm/MC/MCSymbol.h"
27
28using namespace llvm;
29
30namespace llvm::AArch64 {
31struct LFIVariantEntry {
32 unsigned Inst;
33 uint8_t AddrMode;
34 uint8_t Log2Size;
35 unsigned RoWInst;
36};
37struct PairVariantEntry {
38 unsigned Inst;
39 bool IsPre;
40 uint8_t Scale;
41 unsigned BaseInst;
42};
43struct SIMDPostEntry {
44 unsigned Inst;
45 uint8_t NaturalOffset;
46 unsigned BaseInst;
47};
48struct MemInfoEntry {
49 unsigned Inst;
50 uint8_t BaseIdx;
51 uint8_t OffsetIdx;
52 bool HasOffset;
53 bool IsPrePost;
54 bool IsLiteral;
55};
56
57// LFI addressing-mode codes (must match AArch64LFI.td's LFI_AM_* defs).
58enum LFIAddrMode : uint8_t {
59 LFI_AM_Ui = 0,
60 LFI_AM_RoW = 1,
61 LFI_AM_RoX = 2,
62 LFI_AM_Pre = 3,
63 LFI_AM_Post = 4,
64};
65
66#define GET_LFIVariantTable_DECL
67#define GET_PairVariantTable_DECL
68#define GET_SIMDPostTable_DECL
69#define GET_MemInfoTable_DECL
70#define GET_LFIVariantTable_IMPL
71#define GET_PairVariantTable_IMPL
72#define GET_SIMDPostTable_IMPL
73#define GET_MemInfoTable_IMPL
74// The LFI tables defined in AArch64LFI.td are emitted into this file alongside
75// the system operand tables (single -gen-searchable-tables output).
76#include "AArch64GenSystemOperands.inc"
77} // namespace llvm::AArch64
78
79// LFI reserved registers.
80static constexpr MCRegister LFIBaseReg = AArch64::X27;
81static constexpr MCRegister LFIAddrReg = AArch64::X28;
82static constexpr MCRegister LFIScratchReg = AArch64::X26;
83static constexpr MCRegister LFICtxReg = AArch64::X25;
84
85// Offset into the context register block (pointed to by LFICtxReg) where the
86// thread pointer is stored. This is a scaled offset (multiplied by 8 for
87// 64-bit loads), so a value of 2 means an actual byte offset of 16.
88static constexpr unsigned LFITPOffset = 2;
89
90// Byte offset from the sandbox base register where the syscall handler address
91// is stored (negative because it is below the sandbox base).
92static constexpr int LFISyscallOffset = -8;
93
94static bool isSyscall(const MCInst &Inst) {
95 return Inst.getOpcode() == AArch64::SVC;
96}
97
98static bool isPrivilegedTP(int64_t Reg) {
99 return Reg == AArch64SysReg::TPIDR_EL1 || Reg == AArch64SysReg::TPIDR_EL2 ||
100 Reg == AArch64SysReg::TPIDR_EL3;
101}
102
103static bool isTPRead(const MCInst &Inst) {
104 return Inst.getOpcode() == AArch64::MRS &&
105 Inst.getOperand(i: 1).getImm() == AArch64SysReg::TPIDR_EL0;
106}
107
108static bool isTPWrite(const MCInst &Inst) {
109 return Inst.getOpcode() == AArch64::MSR &&
110 Inst.getOperand(i: 0).getImm() == AArch64SysReg::TPIDR_EL0;
111}
112
113static bool isPrivilegedTPAccess(const MCInst &Inst) {
114 if (Inst.getOpcode() == AArch64::MRS)
115 return isPrivilegedTP(Reg: Inst.getOperand(i: 1).getImm());
116 if (Inst.getOpcode() == AArch64::MSR)
117 return isPrivilegedTP(Reg: Inst.getOperand(i: 0).getImm());
118 return false;
119}
120
121// Classification functions are limited to Armv8.1-A. Instructions outside of
122// this subset are not guaranteed to be rewritten and as a result may fail LFI
123// verification after compilation.
124
125// Instructions that have mayLoad/mayStore set in TableGen but don't actually
126// perform memory accesses.
127static bool isFakeMemAccess(const MCInst &Inst) {
128 switch (Inst.getOpcode()) {
129 case AArch64::CLREX:
130 case AArch64::DMB:
131 case AArch64::DSB:
132 case AArch64::ISB:
133 case AArch64::HINT:
134 // The range of sub-architectures supported by LFI do not include any load
135 // or store instructions in the HINT space.
136 return true;
137 default:
138 return false;
139 }
140}
141
142static bool mayPrefetch(const MCInst &Inst) {
143 switch (Inst.getOpcode()) {
144 case AArch64::PRFMl:
145 case AArch64::PRFMroW:
146 case AArch64::PRFMroX:
147 case AArch64::PRFMui:
148 case AArch64::PRFUMi:
149 return true;
150 default:
151 return false;
152 }
153}
154
155static bool isAuthenticatedBranch(unsigned Opcode) {
156 switch (Opcode) {
157 case AArch64::BRAA:
158 case AArch64::BRAAZ:
159 case AArch64::BRAB:
160 case AArch64::BRABZ:
161 return true;
162 default:
163 return false;
164 }
165}
166
167static bool isAuthenticatedCall(unsigned Opcode) {
168 switch (Opcode) {
169 case AArch64::BLRAA:
170 case AArch64::BLRAAZ:
171 case AArch64::BLRAB:
172 case AArch64::BLRABZ:
173 return true;
174 default:
175 return false;
176 }
177}
178
179static bool isAuthenticatedReturn(unsigned Opcode) {
180 return Opcode == AArch64::RETAA || Opcode == AArch64::RETAB;
181}
182
183static bool isExceptionReturn(unsigned Opcode) {
184 return Opcode == AArch64::ERET || Opcode == AArch64::ERETAA ||
185 Opcode == AArch64::ERETAB;
186}
187
188static bool pacWritesLR(const MCInst &Inst) {
189 switch (Inst.getOpcode()) {
190 case AArch64::AUTIASP:
191 case AArch64::AUTIBSP:
192 case AArch64::AUTIAZ:
193 case AArch64::AUTIBZ:
194 case AArch64::XPACLRI:
195 return true;
196 default:
197 return false;
198 }
199}
200
201// User-mode DC/IC instructions that take a virtual address operand. Encoded as
202// SYSxt with op1=3, Cn=7, op2=1 where the Cm field selects the operation.
203static bool isVASysOp(const MCInst &Inst) {
204 if (Inst.getOpcode() != AArch64::SYSxt)
205 return false;
206 if (Inst.getOperand(i: 0).getImm() != 3 || Inst.getOperand(i: 1).getImm() != 7 ||
207 Inst.getOperand(i: 3).getImm() != 1)
208 return false;
209 switch (Inst.getOperand(i: 2).getImm()) {
210 case 4: // DC ZVA
211 case 5: // IC IVAU
212 case 10: // DC CVAC
213 case 11: // DC CVAU
214 case 12: // DC CVAP
215 case 13: // DC CVADP
216 case 14: // DC CIVAC
217 return true;
218 default:
219 return false;
220 }
221}
222
223static MCInst replaceRegAt(const MCInst &Inst, unsigned Idx,
224 MCRegister NewReg) {
225 MCInst New = Inst;
226 assert(New.getOperand(Idx).isReg());
227 New.getOperand(i: Idx).setReg(NewReg);
228 return New;
229}
230
231// AArch64 load/store opcode suffixes used throughout this file:
232// Ui: Unsigned immediate offset, scaled by access size: [Xn, #imm].
233// RoW: Register offset with 32-bit W register: [Xn, Wm, uxtw #shift].
234// RoX: Register offset with 64-bit X register: [Xn, Xm, lsl #shift].
235
236// Scalar load/store variant lookup. If Op is a scalar mem instruction with
237// addressing mode ExpectedMode, returns the RoW variant of the same family.
238// Returns INSTRUCTION_LIST_END otherwise.
239static unsigned convertVariantToRoW(unsigned Op, unsigned ExpectedMode) {
240 const AArch64::LFIVariantEntry *E = AArch64::lookupLFIVariantByOpcode(Inst: Op);
241 if (!E || E->AddrMode != ExpectedMode)
242 return AArch64::INSTRUCTION_LIST_END;
243 return E->RoWInst;
244}
245
246static unsigned convertRoXToRoW(unsigned Op, unsigned &Shift) {
247 Shift = 0;
248 const AArch64::LFIVariantEntry *E = AArch64::lookupLFIVariantByOpcode(Inst: Op);
249 if (!E || E->AddrMode != AArch64::LFI_AM_RoX)
250 return AArch64::INSTRUCTION_LIST_END;
251 Shift = E->Log2Size;
252 return E->RoWInst;
253}
254
255static bool getRoWShift(unsigned Op, unsigned &Shift) {
256 Shift = 0;
257 const AArch64::LFIVariantEntry *E = AArch64::lookupLFIVariantByOpcode(Inst: Op);
258 if (!E || E->AddrMode != AArch64::LFI_AM_RoW)
259 return false;
260 Shift = E->Log2Size;
261 return true;
262}
263
264// Pre/post-index conversion to base form. Both LDP/STP pair pre/post forms and
265// SIMD post-index forms come from generated lookup tables. The pair table sets
266// IsPre to distinguish pre-index from post-index. The SIMD table is
267// post-index-only so IsNoOffset is set to indicate the demoted base form takes
268// no immediate offset.
269static unsigned convertPrePostToBase(unsigned Op, bool &IsPre,
270 bool &IsNoOffset) {
271 IsPre = false;
272 IsNoOffset = false;
273 if (const auto *E = AArch64::lookupPairVariantByOpcode(Inst: Op)) {
274 IsPre = E->IsPre;
275 return E->BaseInst;
276 }
277 if (const auto *E = AArch64::lookupSIMDPostByOpcode(Inst: Op)) {
278 IsNoOffset = true;
279 return E->BaseInst;
280 }
281 return AArch64::INSTRUCTION_LIST_END;
282}
283
284bool AArch64MCLFIRewriter::mayModifySP(const MCInst &Inst) const {
285 return mayModifyRegister(Inst, Reg: AArch64::SP);
286}
287
288MCRegister AArch64MCLFIRewriter::mayModifyReserved(const MCInst &Inst) const {
289 for (MCRegister Reg : {LFIAddrReg, LFIBaseReg, LFICtxReg}) {
290 if (mayModifyRegister(Inst, Reg))
291 return Reg;
292 }
293 return {};
294}
295
296void AArch64MCLFIRewriter::onLabel(const MCSymbol *Symbol, MCStreamer &Out) {
297 if (Guard)
298 return;
299
300 // Flush a deferred LR guard before the label, since the label is a potential
301 // branch target and code reached through it may use LR for control flow.
302 if (DeferredLRGuard && LastSTI && !Symbol->isTemporary()) {
303 emitAddMask(Dest: AArch64::LR, Src: AArch64::LR, Out, STI: *LastSTI);
304 DeferredLRGuard = false;
305 }
306
307 // Invalidate guard state since the label is a potential branch target.
308 ActiveGuardReg = std::nullopt;
309}
310
311void AArch64MCLFIRewriter::finish(MCStreamer &Out) {
312 // Flush a deferred LR guard at the end of the stream.
313 if (DeferredLRGuard && LastSTI) {
314 emitAddMask(Dest: AArch64::LR, Src: AArch64::LR, Out, STI: *LastSTI);
315 DeferredLRGuard = false;
316 }
317}
318
319void AArch64MCLFIRewriter::emitInst(const MCInst &Inst, MCStreamer &Out,
320 const MCSubtargetInfo &STI) {
321 // Invalidate the active guard if this instruction modifies the guarded
322 // register, modifies x28 itself, or may affect control flow.
323 if (ActiveGuardReg) {
324 const MCInstrDesc &Desc = InstInfo->get(Opcode: Inst.getOpcode());
325 if (Desc.mayAffectControlFlow(MI: Inst, RI: *RegInfo) ||
326 mayModifyRegister(Inst, Reg: *ActiveGuardReg) ||
327 mayModifyRegister(Inst, Reg: getWRegFromXReg(Reg: *ActiveGuardReg)) ||
328 mayModifyRegister(Inst, Reg: LFIAddrReg))
329 ActiveGuardReg = std::nullopt;
330 }
331
332 Out.emitInstruction(Inst, STI);
333}
334
335void AArch64MCLFIRewriter::emitAddMask(MCRegister Dest, MCRegister Src,
336 MCStreamer &Out,
337 const MCSubtargetInfo &STI) {
338 // If x28 already holds the guarded value of Src, this guard is redundant and
339 // can be skipped.
340 if (CLOpts.lfi_guard_elim && Dest == LFIAddrReg && ActiveGuardReg == Src)
341 return;
342
343 // add Dest, LFIBaseReg, W(Src), uxtw
344 emitInst(Inst: MCInstBuilder(AArch64::ADDXrx)
345 .addReg(Reg: Dest)
346 .addReg(Reg: LFIBaseReg)
347 .addReg(Reg: getWRegFromXReg(Reg: Src))
348 .addImm(Val: AArch64_AM::getArithExtendImm(ET: AArch64_AM::UXTW, Imm: 0)),
349 Out, STI);
350
351 // Record Src as the new active guard.
352 if (Dest == LFIAddrReg)
353 ActiveGuardReg = Src;
354}
355
356void AArch64MCLFIRewriter::emitBranch(unsigned Opcode, MCRegister Target,
357 MCStreamer &Out,
358 const MCSubtargetInfo &STI) {
359 emitInst(Inst: MCInstBuilder(Opcode).addReg(Reg: Target), Out, STI);
360}
361
362void AArch64MCLFIRewriter::emitPendingTLSDescCall(MCStreamer &Out,
363 const MCSubtargetInfo &STI) {
364 if (!PendingTLSDescCall)
365 return;
366 const MCExpr *Expr = PendingTLSDescCall;
367 PendingTLSDescCall = nullptr;
368 emitInst(Inst: MCInstBuilder(AArch64::TLSDESCCALL).addExpr(Val: Expr), Out, STI);
369}
370
371void AArch64MCLFIRewriter::emitMov(MCRegister Dest, MCRegister Src,
372 MCStreamer &Out,
373 const MCSubtargetInfo &STI) {
374 // orr Dest, xzr, Src
375 emitInst(Inst: MCInstBuilder(AArch64::ORRXrs)
376 .addReg(Reg: Dest)
377 .addReg(Reg: AArch64::XZR)
378 .addReg(Reg: Src)
379 .addImm(Val: 0),
380 Out, STI);
381}
382
383void AArch64MCLFIRewriter::emitAddImm(MCRegister Dest, MCRegister Src,
384 int64_t Imm, MCStreamer &Out,
385 const MCSubtargetInfo &STI) {
386 assert(std::abs(Imm) <= 4095);
387 // add Dest, Src, Imm (or sub Dest, Src, -Imm for negative offsets)
388 unsigned Opcode = Imm >= 0 ? AArch64::ADDXri : AArch64::SUBXri;
389 emitInst(Inst: MCInstBuilder(Opcode)
390 .addReg(Reg: Dest)
391 .addReg(Reg: Src)
392 .addImm(Val: std::abs(i: Imm))
393 .addImm(Val: 0), // shift
394 Out, STI);
395}
396
397void AArch64MCLFIRewriter::emitAddReg(MCRegister Dest, MCRegister Src1,
398 MCRegister Src2, unsigned Shift,
399 MCStreamer &Out,
400 const MCSubtargetInfo &STI) {
401 // add Dest, Src1, Src2, lsl #Shift
402 emitInst(Inst: MCInstBuilder(AArch64::ADDXrs)
403 .addReg(Reg: Dest)
404 .addReg(Reg: Src1)
405 .addReg(Reg: Src2)
406 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: Shift)),
407 Out, STI);
408}
409
410void AArch64MCLFIRewriter::emitAddRegExtend(MCRegister Dest, MCRegister Src1,
411 MCRegister Src2,
412 AArch64_AM::ShiftExtendType ExtType,
413 unsigned Shift, MCStreamer &Out,
414 const MCSubtargetInfo &STI) {
415 // add Dest, Src1, Src2, ExtType #Shift
416 unsigned Opcode = ExtType == AArch64_AM::SXTX || ExtType == AArch64_AM::UXTX
417 ? AArch64::ADDXrx64
418 : AArch64::ADDXrx;
419 emitInst(Inst: MCInstBuilder(Opcode).addReg(Reg: Dest).addReg(Reg: Src1).addReg(Reg: Src2).addImm(
420 Val: AArch64_AM::getArithExtendImm(ET: ExtType, Imm: Shift)),
421 Out, STI);
422}
423
424void AArch64MCLFIRewriter::emitMemRoW(unsigned Opcode, const MCOperand &DataOp,
425 MCRegister BaseReg, MCStreamer &Out,
426 const MCSubtargetInfo &STI) {
427 // Op DataOp, [LFIBaseReg, W(BaseReg), uxtw]
428 emitInst(Inst: MCInstBuilder(Opcode)
429 .addOperand(Op: DataOp)
430 .addReg(Reg: LFIBaseReg)
431 .addReg(Reg: getWRegFromXReg(Reg: BaseReg))
432 .addImm(Val: 0) // S bit = 0 (UXTW).
433 .addImm(Val: 0), // Shift amount = 0 (unscaled).
434 Out, STI);
435}
436
437// {br,blr} xN
438// ->
439// add x28, x27, wN, uxtw
440// {br,blr} x28
441void AArch64MCLFIRewriter::rewriteIndirectBranch(const MCInst &Inst,
442 MCStreamer &Out,
443 const MCSubtargetInfo &STI) {
444 assert(Inst.getNumOperands() >= 1 && Inst.getOperand(0).isReg() &&
445 "expected register operand");
446 MCRegister BranchReg = Inst.getOperand(i: 0).getReg();
447
448 // Guard the branch target through X28.
449 emitAddMask(Dest: LFIAddrReg, Src: BranchReg, Out, STI);
450
451 emitPendingTLSDescCall(Out, STI);
452
453 emitBranch(Opcode: Inst.getOpcode(), Target: LFIAddrReg, Out, STI);
454}
455
456// ret xN (where xN != x30)
457// ->
458// add x28, x27, wN, uxtw
459// ret x28
460//
461// ret (x30) is safe since x30 is always within the sandbox.
462void AArch64MCLFIRewriter::rewriteReturn(const MCInst &Inst, MCStreamer &Out,
463 const MCSubtargetInfo &STI) {
464 assert(Inst.getNumOperands() >= 1 && Inst.getOperand(0).isReg() &&
465 "expected register operand");
466 // RET through LR is safe since LR is always within sandbox.
467 if (Inst.getOperand(i: 0).getReg() != AArch64::LR)
468 rewriteIndirectBranch(Inst, Out, STI);
469 else
470 emitInst(Inst, Out, STI);
471}
472
473// modify x30
474// ->
475// modify x30
476// add x30, x27, w30, uxtw (deferred)
477void AArch64MCLFIRewriter::rewriteLRModification(const MCInst &Inst,
478 MCStreamer &Out,
479 const MCSubtargetInfo &STI) {
480 if (!isFakeMemAccess(Inst) &&
481 (mayLoad(Inst) || mayStore(Inst) || mayPrefetch(Inst)))
482 rewriteLoadStore(Inst, Out, STI);
483 else
484 emitInst(Inst, Out, STI);
485
486 // Defer the LR guard until the next control-flow instruction or label. This
487 // keeps a signed return address intact so that an authentication instruction
488 // can run before the mask destroys the PAC bits.
489 DeferredLRGuard = true;
490}
491
492// retaa / retab
493// ->
494// autiasp / autibsp
495// add x30, x27, w30, uxtw
496// ret
497void AArch64MCLFIRewriter::rewriteAuthenticatedReturn(
498 const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) {
499 emitInst(Inst: MCInstBuilder(Inst.getOpcode() == AArch64::RETAA ? AArch64::AUTIASP
500 : AArch64::AUTIBSP),
501 Out, STI);
502
503 emitAddMask(Dest: AArch64::LR, Src: AArch64::LR, Out, STI);
504 emitBranch(Opcode: AArch64::RET, Target: AArch64::LR, Out, STI);
505}
506
507// {braa,brab,braaz,brabz} xN[, xM] (blra* for calls)
508// ->
509// {autia,autib,autiza,autizb} xN[, xM]
510// add x28, x27, wN, uxtw
511// {br,blr} x28
512void AArch64MCLFIRewriter::rewriteAuthenticatedBranchOrCall(
513 const MCInst &Inst, unsigned BranchOpcode, MCStreamer &Out,
514 const MCSubtargetInfo &STI) {
515 MCRegister TargetReg = Inst.getOperand(i: 0).getReg();
516
517 // Select the authentication opcode for the target register.
518 unsigned AuthOpcode;
519 switch (Inst.getOpcode()) {
520 case AArch64::BRAA:
521 case AArch64::BLRAA:
522 AuthOpcode = AArch64::AUTIA;
523 break;
524 case AArch64::BRAB:
525 case AArch64::BLRAB:
526 AuthOpcode = AArch64::AUTIB;
527 break;
528 case AArch64::BRAAZ:
529 case AArch64::BLRAAZ:
530 AuthOpcode = AArch64::AUTIZA;
531 break;
532 case AArch64::BRABZ:
533 case AArch64::BLRABZ:
534 AuthOpcode = AArch64::AUTIZB;
535 break;
536 default:
537 llvm_unreachable("unexpected authenticated branch/call opcode");
538 }
539
540 MCInstBuilder Auth(AuthOpcode);
541 Auth.addReg(Reg: TargetReg); // dst
542 Auth.addReg(Reg: TargetReg); // src (tied to dst)
543 if (AuthOpcode == AArch64::AUTIA || AuthOpcode == AArch64::AUTIB)
544 Auth.addOperand(Op: Inst.getOperand(i: 1)); // modifier
545 emitInst(Inst: Auth, Out, STI);
546
547 // Guard the authenticated target and branch/call through x28.
548 emitAddMask(Dest: LFIAddrReg, Src: TargetReg, Out, STI);
549 emitBranch(Opcode: BranchOpcode, Target: LFIAddrReg, Out, STI);
550}
551
552// svc #0
553// ->
554// mov x26, x30
555// ldur x30, [x27, #-8]
556// blr x30
557// add x30, x27, w26, uxtw
558void AArch64MCLFIRewriter::rewriteSyscall(const MCInst &, MCStreamer &Out,
559 const MCSubtargetInfo &STI) {
560 // Save LR to scratch.
561 emitMov(Dest: LFIScratchReg, Src: AArch64::LR, Out, STI);
562
563 // Load syscall handler address from negative offset from sandbox base.
564 emitInst(Inst: MCInstBuilder(AArch64::LDURXi)
565 .addReg(Reg: AArch64::LR)
566 .addReg(Reg: LFIBaseReg)
567 .addImm(Val: LFISyscallOffset),
568 Out, STI);
569
570 // Call the runtime.
571 emitBranch(Opcode: AArch64::BLR, Target: AArch64::LR, Out, STI);
572
573 // Restore LR with guard.
574 emitAddMask(Dest: AArch64::LR, Src: LFIScratchReg, Out, STI);
575}
576
577// mrs xN, tpidr_el0
578// ->
579// ldr xN, [x25, #16]
580void AArch64MCLFIRewriter::rewriteTPRead(const MCInst &Inst, MCStreamer &Out,
581 const MCSubtargetInfo &STI) {
582 MCRegister DestReg = Inst.getOperand(i: 0).getReg();
583
584 emitInst(Inst: MCInstBuilder(AArch64::LDRXui)
585 .addReg(Reg: DestReg)
586 .addReg(Reg: LFICtxReg)
587 .addImm(Val: LFITPOffset),
588 Out, STI);
589}
590
591// msr tpidr_el0, xN
592// ->
593// str xN, [x25, #16]
594void AArch64MCLFIRewriter::rewriteTPWrite(const MCInst &Inst, MCStreamer &Out,
595 const MCSubtargetInfo &STI) {
596 MCRegister SrcReg = Inst.getOperand(i: 1).getReg();
597
598 emitInst(Inst: MCInstBuilder(AArch64::STRXui)
599 .addReg(Reg: SrcReg)
600 .addReg(Reg: LFICtxReg)
601 .addImm(Val: LFITPOffset),
602 Out, STI);
603}
604
605bool AArch64MCLFIRewriter::rewriteLoadStoreRoW(const MCInst &Inst,
606 MCStreamer &Out,
607 const MCSubtargetInfo &STI) {
608 unsigned Op = Inst.getOpcode();
609 unsigned MemOp;
610
611 // Case 1: Indexed load/store with zero immediate offset.
612 // ldr xN, [xM, #0] -> ldr xN, [x27, wM, uxtw]
613 if ((MemOp = convertVariantToRoW(Op, ExpectedMode: AArch64::LFI_AM_Ui)) !=
614 AArch64::INSTRUCTION_LIST_END) {
615 MCRegister BaseReg = Inst.getOperand(i: 1).getReg();
616 if (BaseReg == AArch64::SP)
617 return false;
618 const MCOperand &OffsetOp = Inst.getOperand(i: 2);
619 if (OffsetOp.isImm() && OffsetOp.getImm() == 0) {
620 emitMemRoW(Opcode: MemOp, DataOp: Inst.getOperand(i: 0), BaseReg, Out, STI);
621 return true;
622 }
623 return false;
624 }
625
626 // Case 2: Pre-index load/store with writeback.
627 // ldr xN, [xM, #imm]! -> add xM, xM, #imm; ldr xN, [x27, wM, uxtw]
628 if ((MemOp = convertVariantToRoW(Op, ExpectedMode: AArch64::LFI_AM_Pre)) !=
629 AArch64::INSTRUCTION_LIST_END) {
630 MCRegister BaseReg = Inst.getOperand(i: 2).getReg();
631 if (BaseReg == AArch64::SP)
632 return false;
633 int64_t Imm = Inst.getOperand(i: 3).getImm();
634 emitAddImm(Dest: BaseReg, Src: BaseReg, Imm, Out, STI);
635 emitMemRoW(Opcode: MemOp, DataOp: Inst.getOperand(i: 1), BaseReg, Out, STI);
636 return true;
637 }
638
639 // Case 3: Post-index load/store.
640 // ldr xN, [xM], #imm -> ldr xN, [x27, wM, uxtw]; add xM, xM, #imm
641 if ((MemOp = convertVariantToRoW(Op, ExpectedMode: AArch64::LFI_AM_Post)) !=
642 AArch64::INSTRUCTION_LIST_END) {
643 MCRegister BaseReg = Inst.getOperand(i: 2).getReg();
644 if (BaseReg == AArch64::SP)
645 return false;
646 int64_t Imm = Inst.getOperand(i: 3).getImm();
647 emitMemRoW(Opcode: MemOp, DataOp: Inst.getOperand(i: 1), BaseReg, Out, STI);
648 emitAddImm(Dest: BaseReg, Src: BaseReg, Imm, Out, STI);
649 return true;
650 }
651
652 // Case 4: Register-offset-X load/store.
653 // ldr xN, [xM1, xM2] -> add x26, xM1, xM2; ldr xN, [x27, w26, uxtw]
654 //
655 // In this case, even if xM1 is SP we must do a full rewrite, since an
656 // arbitrary register value is being added as the offset.
657 unsigned Shift;
658 if ((MemOp = convertRoXToRoW(Op, Shift)) != AArch64::INSTRUCTION_LIST_END) {
659 MCRegister Reg1 = Inst.getOperand(i: 1).getReg();
660 MCRegister Reg2 = Inst.getOperand(i: 2).getReg();
661 int64_t Extend = Inst.getOperand(i: 3).getImm();
662 int64_t IsShift = Inst.getOperand(i: 4).getImm();
663
664 if (!IsShift)
665 Shift = 0;
666
667 if (Extend)
668 emitAddRegExtend(Dest: LFIScratchReg, Src1: Reg1, Src2: Reg2, ExtType: AArch64_AM::SXTX, Shift, Out,
669 STI);
670 else
671 emitAddReg(Dest: LFIScratchReg, Src1: Reg1, Src2: Reg2, Shift, Out, STI);
672 emitMemRoW(Opcode: MemOp, DataOp: Inst.getOperand(i: 0), BaseReg: LFIScratchReg, Out, STI);
673 return true;
674 }
675
676 // Case 5: Register-offset-W load/store.
677 // ldr xN, [xM1, wM2, uxtw] -> add x26, xM1, wM2, uxtw;
678 // ldr xN, [x27, w26, uxtw]
679 if (getRoWShift(Op, Shift)) {
680 MCRegister Reg1 = Inst.getOperand(i: 1).getReg();
681 MCRegister Reg2 = Inst.getOperand(i: 2).getReg();
682 int64_t S = Inst.getOperand(i: 3).getImm();
683 int64_t IsShift = Inst.getOperand(i: 4).getImm();
684
685 if (!IsShift)
686 Shift = 0;
687
688 if (S)
689 emitAddRegExtend(Dest: LFIScratchReg, Src1: Reg1, Src2: Reg2, ExtType: AArch64_AM::SXTW, Shift, Out,
690 STI);
691 else
692 emitAddRegExtend(Dest: LFIScratchReg, Src1: Reg1, Src2: Reg2, ExtType: AArch64_AM::UXTW, Shift, Out,
693 STI);
694 emitMemRoW(Opcode: Op, DataOp: Inst.getOperand(i: 0), BaseReg: LFIScratchReg, Out, STI);
695 return true;
696 }
697
698 return false;
699}
700
701void AArch64MCLFIRewriter::rewriteLoadStoreBase(const MCInst &Inst,
702 MCStreamer &Out,
703 const MCSubtargetInfo &STI) {
704 unsigned Opcode = Inst.getOpcode();
705 const AArch64::MemInfoEntry *Info = AArch64::lookupMemInfoByOpcode(Inst: Opcode);
706
707 if (!Info) {
708 warning(Inst, Msg: "unknown addressing mode for memory instruction in LFI");
709 return emitInst(Inst, Out, STI);
710 }
711
712 if (Info->IsLiteral)
713 return error(Inst, Msg: "PC-relative literal loads are not supported in LFI");
714
715 MCRegister BaseReg = Inst.getOperand(i: Info->BaseIdx).getReg();
716
717 // Stack accesses don't need address sandboxing, except when sp is modified
718 // with a non-zero register post-index operand.
719 bool BaseIsSP = BaseReg == AArch64::SP;
720 if (BaseIsSP) {
721 if (!Info->HasOffset || !Inst.getOperand(i: Info->OffsetIdx).isReg())
722 return emitInst(Inst, Out, STI);
723 MCRegister OffReg = Inst.getOperand(i: Info->OffsetIdx).getReg();
724 if (OffReg == AArch64::XZR || OffReg == AArch64::WZR)
725 return emitInst(Inst, Out, STI);
726 }
727
728 // Guard the base register, unless it is SP.
729 if (!BaseIsSP)
730 emitAddMask(Dest: LFIAddrReg, Src: BaseReg, Out, STI);
731
732 if (!Info->IsPrePost) {
733 // Non-pre/post instruction: replace the base register operand.
734 MCInst NewInst = replaceRegAt(Inst, Idx: Info->BaseIdx, NewReg: LFIAddrReg);
735 emitInst(Inst: NewInst, Out, STI);
736 return;
737 }
738
739 bool IsPre = false;
740 bool IsNoOffset = false;
741 unsigned BaseOpcode = convertPrePostToBase(Op: Opcode, IsPre, IsNoOffset);
742
743 if (BaseOpcode == AArch64::INSTRUCTION_LIST_END)
744 return error(Inst, Msg: "unhandled pre/post-index instruction in LFI rewriter");
745
746 // Demote pre/post-index to base indexed form.
747 MCInstBuilder NewInst(BaseOpcode);
748 NewInst.setLoc(Inst.getLoc());
749
750 // Skip writeback operand (operand 0) and copy data operands up to base.
751 for (int I = 1; I < Info->BaseIdx; ++I)
752 NewInst.addOperand(Op: Inst.getOperand(i: I));
753
754 // Add the access base register (LFIAddrReg or SP).
755 NewInst.addReg(Reg: BaseIsSP ? AArch64::SP : LFIAddrReg);
756
757 // For pre-index, include the offset; for post-index, use zero.
758 if (IsPre && Info->HasOffset)
759 NewInst.addOperand(Op: Inst.getOperand(i: Info->OffsetIdx));
760 else if (!IsNoOffset)
761 NewInst.addImm(Val: 0);
762
763 emitInst(Inst: NewInst, Out, STI);
764
765 if (!Info->HasOffset)
766 return;
767
768 // Update the base register with the offset. If the base is SP, a register
769 // offset must be sandboxed (the result is otherwise unbounded), and ADDXrs
770 // cannot take SP, so the extended-register form via the scratch register is
771 // used.
772 const MCOperand &OffsetOp = Inst.getOperand(i: Info->OffsetIdx);
773 if (OffsetOp.isImm()) {
774 // Pair pre/post immediates are scaled by element size; other pre/post
775 // forms (scalar, SIMD) use the raw immediate (scale = 1).
776 int64_t Scale = 1;
777 if (const auto *E = AArch64::lookupPairVariantByOpcode(Inst: Opcode))
778 Scale = E->Scale;
779 int64_t Offset = OffsetOp.getImm() * Scale;
780 emitAddImm(Dest: BaseReg, Src: BaseReg, Imm: Offset, Out, STI);
781 } else if (OffsetOp.isReg()) {
782 // SIMD post-index uses a register offset (XZR for natural offset).
783 MCRegister OffReg = OffsetOp.getReg();
784 if (OffReg == AArch64::XZR) {
785 if (const auto *E = AArch64::lookupSIMDPostByOpcode(Inst: Opcode))
786 emitAddImm(Dest: BaseReg, Src: BaseReg, Imm: E->NaturalOffset, Out, STI);
787 } else if (OffReg != AArch64::WZR) {
788 if (BaseIsSP) {
789 emitAddRegExtend(Dest: LFIScratchReg, Src1: AArch64::SP, Src2: OffReg, ExtType: AArch64_AM::UXTX,
790 Shift: 0, Out, STI);
791 emitAddMask(Dest: AArch64::SP, Src: LFIScratchReg, Out, STI);
792 } else {
793 emitAddReg(Dest: BaseReg, Src1: BaseReg, Src2: OffReg, Shift: 0, Out, STI);
794 }
795 }
796 }
797}
798
799void AArch64MCLFIRewriter::rewriteLoadStore(const MCInst &Inst, MCStreamer &Out,
800 const MCSubtargetInfo &STI) {
801 bool IsStore = mayStore(Inst);
802 bool IsLoad = mayLoad(Inst) || mayPrefetch(Inst);
803
804 bool SkipLoads = STI.hasFeature(Feature: AArch64::FeatureNoLFILoads);
805 bool SkipStores = STI.hasFeature(Feature: AArch64::FeatureNoLFIStores);
806
807 if ((!IsLoad || SkipLoads) && (!IsStore || SkipStores))
808 return emitInst(Inst, Out, STI);
809
810 if (rewriteLoadStoreRoW(Inst, Out, STI))
811 return;
812
813 rewriteLoadStoreBase(Inst, Out, STI);
814}
815
816// modify sp
817// ->
818// modify x26
819// add sp, x27, w26, uxtw
820void AArch64MCLFIRewriter::rewriteSPModification(const MCInst &Inst,
821 MCStreamer &Out,
822 const MCSubtargetInfo &STI) {
823 // Route through rewriteLRModification or rewriteLoadStore for memory
824 // accesses. Those helpers automatically handle dangerous stack modifications
825 // that can happen via register post-index.
826 if (mayLoad(Inst) || mayStore(Inst)) {
827 if (mayModifyRegister(Inst, Reg: AArch64::LR))
828 return rewriteLRModification(Inst, Out, STI);
829 return rewriteLoadStore(Inst, Out, STI);
830 }
831
832 // No stack sandboxing if sandboxing is disabled for both loads and stores.
833 bool SkipLoads = STI.hasFeature(Feature: AArch64::FeatureNoLFILoads);
834 bool SkipStores = STI.hasFeature(Feature: AArch64::FeatureNoLFIStores);
835 if (SkipLoads && SkipStores)
836 return emitInst(Inst, Out, STI);
837
838 // Special case: mov sp, xN -> add sp, x27, wN, uxtw
839 if (Inst.getOpcode() == AArch64::ADDXri && Inst.getOperand(i: 2).getImm() == 0 &&
840 Inst.getOperand(i: 3).getImm() == 0)
841 return emitAddMask(Dest: AArch64::SP, Src: Inst.getOperand(i: 1).getReg(), Out, STI);
842
843 // Redirect SP modification destination to scratch, then sandbox.
844 MCInst ModInst = replaceRegAt(Inst, Idx: 0, NewReg: LFIScratchReg);
845 emitInst(Inst: ModInst, Out, STI);
846 emitAddMask(Dest: AArch64::SP, Src: LFIScratchReg, Out, STI);
847}
848
849// {dc,ic} <op>, xN
850// ->
851// add x28, x27, wN, uxtw
852// {dc,ic} <op>, x28
853void AArch64MCLFIRewriter::rewriteVASysOp(const MCInst &Inst, MCStreamer &Out,
854 const MCSubtargetInfo &STI) {
855 MCRegister AddrReg = Inst.getOperand(i: 4).getReg();
856
857 emitAddMask(Dest: LFIAddrReg, Src: AddrReg, Out, STI);
858
859 emitInst(Inst: MCInstBuilder(AArch64::SYSxt)
860 .addOperand(Op: Inst.getOperand(i: 0))
861 .addOperand(Op: Inst.getOperand(i: 1))
862 .addOperand(Op: Inst.getOperand(i: 2))
863 .addOperand(Op: Inst.getOperand(i: 3))
864 .addReg(Reg: LFIAddrReg),
865 Out, STI);
866}
867
868// NOTE: when adding new rewrites, the size estimates in
869// AArch64InstrInfo::getLFIInstSizeInBytes must be updated to match.
870void AArch64MCLFIRewriter::doRewriteInst(const MCInst &Inst, MCStreamer &Out,
871 const MCSubtargetInfo &STI) {
872 if (Inst.getOpcode() == AArch64::TLSDESCCALL) {
873 PendingTLSDescCall = Inst.getOperand(i: 0).getExpr();
874 return;
875 }
876
877 // Reserved register modification is an error.
878 if (MCRegister Reg = mayModifyReserved(Inst)) {
879 error(Inst, Msg: Twine("illegal modification of reserved LFI register ") +
880 RegInfo->getName(RegNo: Reg));
881 return;
882 }
883
884 // System instructions.
885 if (isSyscall(Inst))
886 return rewriteSyscall(Inst, Out, STI);
887
888 if (isTPRead(Inst))
889 return rewriteTPRead(Inst, Out, STI);
890
891 if (isTPWrite(Inst))
892 return rewriteTPWrite(Inst, Out, STI);
893
894 if (isPrivilegedTPAccess(Inst)) {
895 error(Inst, Msg: "illegal access to privileged thread pointer register");
896 return;
897 }
898
899 if (isVASysOp(Inst))
900 return rewriteVASysOp(Inst, Out, STI);
901
902 if (isExceptionReturn(Opcode: Inst.getOpcode())) {
903 error(Inst, Msg: "exception returns are not supported by LFI");
904 return;
905 }
906
907 // PAC authenticated returns expand to authenticate + guarded RET. The
908 // expansion emits its own LR guard, so discard any deferred guard: masking
909 // before the authentication would corrupt the signed return address.
910 if (isAuthenticatedReturn(Opcode: Inst.getOpcode())) {
911 DeferredLRGuard = false;
912 return rewriteAuthenticatedReturn(Inst, Out, STI);
913 }
914
915 // Flush a deferred LR guard before any control-flow instruction, so that a
916 // modified LR is sandboxed before it can be used to transfer control.
917 if (DeferredLRGuard && (isReturn(Inst) || isIndirectBranch(Inst) ||
918 isCall(Inst) || isBranch(Inst))) {
919 emitAddMask(Dest: AArch64::LR, Src: AArch64::LR, Out, STI);
920 DeferredLRGuard = false;
921 }
922
923 // PAC authenticated branches/calls expand to authenticate + guarded branch.
924 if (isAuthenticatedBranch(Opcode: Inst.getOpcode()))
925 return rewriteAuthenticatedBranchOrCall(Inst, BranchOpcode: AArch64::BR, Out, STI);
926 if (isAuthenticatedCall(Opcode: Inst.getOpcode()))
927 return rewriteAuthenticatedBranchOrCall(Inst, BranchOpcode: AArch64::BLR, Out, STI);
928
929 // Control flow.
930 switch (Inst.getOpcode()) {
931 case AArch64::RET:
932 return rewriteReturn(Inst, Out, STI);
933 case AArch64::BR:
934 case AArch64::BLR:
935 return rewriteIndirectBranch(Inst, Out, STI);
936 }
937
938 // Register modifications that require sandboxing.
939 if (mayModifySP(Inst))
940 return rewriteSPModification(Inst, Out, STI);
941
942 // Link register modification. This covers explicit writes to x30 as well as
943 // PAC instructions that write LR in place, which define LR implicitly.
944 if (explicitlyModifiesRegister(Inst, Reg: AArch64::LR) || pacWritesLR(Inst))
945 return rewriteLRModification(Inst, Out, STI);
946
947 // Memory access.
948 if (!isFakeMemAccess(Inst) &&
949 (mayLoad(Inst) || mayStore(Inst) || mayPrefetch(Inst)))
950 return rewriteLoadStore(Inst, Out, STI);
951
952 emitInst(Inst, Out, STI);
953}
954
955// This function is made available to the size estimator so that it can
956// classify Pre/Post-index instructions.
957bool llvm::isLFIPrePostMemAccess(unsigned Opcode) {
958 if (convertVariantToRoW(Op: Opcode, ExpectedMode: AArch64::LFI_AM_Pre) !=
959 AArch64::INSTRUCTION_LIST_END)
960 return true;
961 if (convertVariantToRoW(Op: Opcode, ExpectedMode: AArch64::LFI_AM_Post) !=
962 AArch64::INSTRUCTION_LIST_END)
963 return true;
964 bool IsPre, IsNoOffset;
965 if (convertPrePostToBase(Op: Opcode, IsPre, IsNoOffset) !=
966 AArch64::INSTRUCTION_LIST_END)
967 return true;
968 return false;
969}
970
971bool AArch64MCLFIRewriter::rewriteInst(const MCInst &Inst, MCStreamer &Out,
972 const MCSubtargetInfo &STI) {
973 // Invalidate guard state if the rewriter was manually disabled.
974 if (!Enabled)
975 ActiveGuardReg = std::nullopt;
976
977 // This recursion guard prevents rewrite-recursion when we emit instructions
978 // from inside the rewriter (such instructions should not be rewritten).
979 if (!Enabled || Guard)
980 return false;
981 Guard = true;
982
983 // Record the subtarget so a deferred LR guard can be emitted from
984 // onLabel/finish, which are not given an MCSubtargetInfo.
985 LastSTI = &STI;
986
987 doRewriteInst(Inst, Out, STI);
988
989 Guard = false;
990 return true;
991}
992