1//===- AArch64MCLFIRewriter.cpp ---------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the AArch64MCLFIRewriter class, the AArch64 specific
10// subclass of MCLFIRewriter.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AArch64MCLFIRewriter.h"
15#include "AArch64AddressingModes.h"
16#include "MCTargetDesc/AArch64MCTargetDesc.h"
17#include "Utils/AArch64BaseInfo.h"
18
19#include "llvm/ADT/Twine.h"
20#include "llvm/MC/MCInst.h"
21#include "llvm/MC/MCInstrDesc.h"
22#include "llvm/MC/MCInstrInfo.h"
23#include "llvm/MC/MCStreamer.h"
24#include "llvm/MC/MCSubtargetInfo.h"
25#include "llvm/Support/CommandLine.h"
26
27using namespace llvm;
28
29static cl::opt<bool>
30 LFIGuardElim("aarch64-lfi-guard-elim", cl::Hidden,
31 cl::desc("Enable the LFI guard elimination optimization"),
32 cl::init(Val: true));
33
34namespace llvm::AArch64 {
35struct LFIVariantEntry {
36 unsigned Inst;
37 uint8_t AddrMode;
38 uint8_t Log2Size;
39 unsigned RoWInst;
40};
41struct PairVariantEntry {
42 unsigned Inst;
43 bool IsPre;
44 uint8_t Scale;
45 unsigned BaseInst;
46};
47struct SIMDPostEntry {
48 unsigned Inst;
49 uint8_t NaturalOffset;
50 unsigned BaseInst;
51};
52struct MemInfoEntry {
53 unsigned Inst;
54 uint8_t BaseIdx;
55 uint8_t OffsetIdx;
56 bool HasOffset;
57 bool IsPrePost;
58 bool IsLiteral;
59};
60
61// LFI addressing-mode codes (must match AArch64LFI.td's LFI_AM_* defs).
62enum LFIAddrMode : uint8_t {
63 LFI_AM_Ui = 0,
64 LFI_AM_RoW = 1,
65 LFI_AM_RoX = 2,
66 LFI_AM_Pre = 3,
67 LFI_AM_Post = 4,
68};
69
70#define GET_LFIVariantTable_DECL
71#define GET_PairVariantTable_DECL
72#define GET_SIMDPostTable_DECL
73#define GET_MemInfoTable_DECL
74#define GET_LFIVariantTable_IMPL
75#define GET_PairVariantTable_IMPL
76#define GET_SIMDPostTable_IMPL
77#define GET_MemInfoTable_IMPL
78// The LFI tables defined in AArch64LFI.td are emitted into this file alongside
79// the system operand tables (single -gen-searchable-tables output).
80#include "AArch64GenSystemOperands.inc"
81} // namespace llvm::AArch64
82
83// LFI reserved registers.
84static constexpr MCRegister LFIBaseReg = AArch64::X27;
85static constexpr MCRegister LFIAddrReg = AArch64::X28;
86static constexpr MCRegister LFIScratchReg = AArch64::X26;
87static constexpr MCRegister LFICtxReg = AArch64::X25;
88
89// Offset into the context register block (pointed to by LFICtxReg) where the
90// thread pointer is stored. This is a scaled offset (multiplied by 8 for
91// 64-bit loads), so a value of 2 means an actual byte offset of 16.
92static constexpr unsigned LFITPOffset = 2;
93
94// Byte offset from the sandbox base register where the syscall handler address
95// is stored (negative because it is below the sandbox base).
96static constexpr int LFISyscallOffset = -8;
97
98static bool isSyscall(const MCInst &Inst) {
99 return Inst.getOpcode() == AArch64::SVC;
100}
101
102static bool isPrivilegedTP(int64_t Reg) {
103 return Reg == AArch64SysReg::TPIDR_EL1 || Reg == AArch64SysReg::TPIDR_EL2 ||
104 Reg == AArch64SysReg::TPIDR_EL3;
105}
106
107static bool isTPRead(const MCInst &Inst) {
108 return Inst.getOpcode() == AArch64::MRS &&
109 Inst.getOperand(i: 1).getImm() == AArch64SysReg::TPIDR_EL0;
110}
111
112static bool isTPWrite(const MCInst &Inst) {
113 return Inst.getOpcode() == AArch64::MSR &&
114 Inst.getOperand(i: 0).getImm() == AArch64SysReg::TPIDR_EL0;
115}
116
117static bool isPrivilegedTPAccess(const MCInst &Inst) {
118 if (Inst.getOpcode() == AArch64::MRS)
119 return isPrivilegedTP(Reg: Inst.getOperand(i: 1).getImm());
120 if (Inst.getOpcode() == AArch64::MSR)
121 return isPrivilegedTP(Reg: Inst.getOperand(i: 0).getImm());
122 return false;
123}
124
125// Classification functions are limited to Armv8.1-A. Instructions outside of
126// this subset are not guaranteed to be rewritten and as a result may fail LFI
127// verification after compilation.
128
129// Instructions that have mayLoad/mayStore set in TableGen but don't actually
130// perform memory accesses.
131static bool isFakeMemAccess(const MCInst &Inst) {
132 switch (Inst.getOpcode()) {
133 case AArch64::CLREX:
134 case AArch64::DMB:
135 case AArch64::DSB:
136 case AArch64::ISB:
137 case AArch64::HINT:
138 // The range of sub-architectures supported by LFI do not include any load
139 // or store instructions in the HINT space.
140 return true;
141 default:
142 return false;
143 }
144}
145
146static bool mayPrefetch(const MCInst &Inst) {
147 switch (Inst.getOpcode()) {
148 case AArch64::PRFMl:
149 case AArch64::PRFMroW:
150 case AArch64::PRFMroX:
151 case AArch64::PRFMui:
152 case AArch64::PRFUMi:
153 return true;
154 default:
155 return false;
156 }
157}
158
159static bool isAuthenticatedBranch(unsigned Opcode) {
160 switch (Opcode) {
161 case AArch64::BRAA:
162 case AArch64::BRAAZ:
163 case AArch64::BRAB:
164 case AArch64::BRABZ:
165 return true;
166 default:
167 return false;
168 }
169}
170
171static bool isAuthenticatedCall(unsigned Opcode) {
172 switch (Opcode) {
173 case AArch64::BLRAA:
174 case AArch64::BLRAAZ:
175 case AArch64::BLRAB:
176 case AArch64::BLRABZ:
177 return true;
178 default:
179 return false;
180 }
181}
182
183static bool isAuthenticatedReturn(unsigned Opcode) {
184 return Opcode == AArch64::RETAA || Opcode == AArch64::RETAB;
185}
186
187static bool isExceptionReturn(unsigned Opcode) {
188 return Opcode == AArch64::ERET || Opcode == AArch64::ERETAA ||
189 Opcode == AArch64::ERETAB;
190}
191
192static bool pacWritesLR(const MCInst &Inst) {
193 switch (Inst.getOpcode()) {
194 case AArch64::AUTIASP:
195 case AArch64::AUTIBSP:
196 case AArch64::AUTIAZ:
197 case AArch64::AUTIBZ:
198 case AArch64::XPACLRI:
199 return true;
200 default:
201 return false;
202 }
203}
204
205// User-mode DC/IC instructions that take a virtual address operand. Encoded as
206// SYSxt with op1=3, Cn=7, op2=1 where the Cm field selects the operation.
207static bool isVASysOp(const MCInst &Inst) {
208 if (Inst.getOpcode() != AArch64::SYSxt)
209 return false;
210 if (Inst.getOperand(i: 0).getImm() != 3 || Inst.getOperand(i: 1).getImm() != 7 ||
211 Inst.getOperand(i: 3).getImm() != 1)
212 return false;
213 switch (Inst.getOperand(i: 2).getImm()) {
214 case 4: // DC ZVA
215 case 5: // IC IVAU
216 case 10: // DC CVAC
217 case 11: // DC CVAU
218 case 12: // DC CVAP
219 case 13: // DC CVADP
220 case 14: // DC CIVAC
221 return true;
222 default:
223 return false;
224 }
225}
226
227static MCInst replaceRegAt(const MCInst &Inst, unsigned Idx,
228 MCRegister NewReg) {
229 MCInst New = Inst;
230 assert(New.getOperand(Idx).isReg());
231 New.getOperand(i: Idx).setReg(NewReg);
232 return New;
233}
234
235// AArch64 load/store opcode suffixes used throughout this file:
236// Ui: Unsigned immediate offset, scaled by access size: [Xn, #imm].
237// RoW: Register offset with 32-bit W register: [Xn, Wm, uxtw #shift].
238// RoX: Register offset with 64-bit X register: [Xn, Xm, lsl #shift].
239
240// Scalar load/store variant lookup. If Op is a scalar mem instruction with
241// addressing mode ExpectedMode, returns the RoW variant of the same family.
242// Returns INSTRUCTION_LIST_END otherwise.
243static unsigned convertVariantToRoW(unsigned Op, unsigned ExpectedMode) {
244 const AArch64::LFIVariantEntry *E = AArch64::lookupLFIVariantByOpcode(Inst: Op);
245 if (!E || E->AddrMode != ExpectedMode)
246 return AArch64::INSTRUCTION_LIST_END;
247 return E->RoWInst;
248}
249
250static unsigned convertRoXToRoW(unsigned Op, unsigned &Shift) {
251 Shift = 0;
252 const AArch64::LFIVariantEntry *E = AArch64::lookupLFIVariantByOpcode(Inst: Op);
253 if (!E || E->AddrMode != AArch64::LFI_AM_RoX)
254 return AArch64::INSTRUCTION_LIST_END;
255 Shift = E->Log2Size;
256 return E->RoWInst;
257}
258
259static bool getRoWShift(unsigned Op, unsigned &Shift) {
260 Shift = 0;
261 const AArch64::LFIVariantEntry *E = AArch64::lookupLFIVariantByOpcode(Inst: Op);
262 if (!E || E->AddrMode != AArch64::LFI_AM_RoW)
263 return false;
264 Shift = E->Log2Size;
265 return true;
266}
267
268// Pre/post-index conversion to base form. Both LDP/STP pair pre/post forms and
269// SIMD post-index forms come from generated lookup tables. The pair table sets
270// IsPre to distinguish pre-index from post-index. The SIMD table is
271// post-index-only so IsNoOffset is set to indicate the demoted base form takes
272// no immediate offset.
273static unsigned convertPrePostToBase(unsigned Op, bool &IsPre,
274 bool &IsNoOffset) {
275 IsPre = false;
276 IsNoOffset = false;
277 if (const auto *E = AArch64::lookupPairVariantByOpcode(Inst: Op)) {
278 IsPre = E->IsPre;
279 return E->BaseInst;
280 }
281 if (const auto *E = AArch64::lookupSIMDPostByOpcode(Inst: Op)) {
282 IsNoOffset = true;
283 return E->BaseInst;
284 }
285 return AArch64::INSTRUCTION_LIST_END;
286}
287
288bool AArch64MCLFIRewriter::mayModifySP(const MCInst &Inst) const {
289 return mayModifyRegister(Inst, Reg: AArch64::SP);
290}
291
292MCRegister AArch64MCLFIRewriter::mayModifyReserved(const MCInst &Inst) const {
293 for (MCRegister Reg : {LFIAddrReg, LFIBaseReg, LFICtxReg}) {
294 if (mayModifyRegister(Inst, Reg))
295 return Reg;
296 }
297 return {};
298}
299
300void AArch64MCLFIRewriter::onLabel(const MCSymbol *, MCStreamer &Out) {
301 if (Guard)
302 return;
303
304 // Flush a deferred LR guard before the label, since the label is a potential
305 // branch target and code reached through it may use LR for control flow.
306 if (DeferredLRGuard && LastSTI) {
307 emitAddMask(Dest: AArch64::LR, Src: AArch64::LR, Out, STI: *LastSTI);
308 DeferredLRGuard = false;
309 }
310
311 // Invalidate guard state since the label is a potential branch target.
312 ActiveGuardReg = std::nullopt;
313}
314
315void AArch64MCLFIRewriter::finish(MCStreamer &Out) {
316 // Flush a deferred LR guard at the end of the stream.
317 if (DeferredLRGuard && LastSTI) {
318 emitAddMask(Dest: AArch64::LR, Src: AArch64::LR, Out, STI: *LastSTI);
319 DeferredLRGuard = false;
320 }
321}
322
323void AArch64MCLFIRewriter::emitInst(const MCInst &Inst, MCStreamer &Out,
324 const MCSubtargetInfo &STI) {
325 // Invalidate the active guard if this instruction modifies the guarded
326 // register, modifies x28 itself, or may affect control flow.
327 if (ActiveGuardReg) {
328 const MCInstrDesc &Desc = InstInfo->get(Opcode: Inst.getOpcode());
329 if (Desc.mayAffectControlFlow(MI: Inst, RI: *RegInfo) ||
330 mayModifyRegister(Inst, Reg: *ActiveGuardReg) ||
331 mayModifyRegister(Inst, Reg: getWRegFromXReg(Reg: *ActiveGuardReg)) ||
332 mayModifyRegister(Inst, Reg: LFIAddrReg))
333 ActiveGuardReg = std::nullopt;
334 }
335
336 Out.emitInstruction(Inst, STI);
337}
338
339void AArch64MCLFIRewriter::emitAddMask(MCRegister Dest, MCRegister Src,
340 MCStreamer &Out,
341 const MCSubtargetInfo &STI) {
342 // If x28 already holds the guarded value of Src, this guard is redundant and
343 // can be skipped.
344 if (LFIGuardElim && Dest == LFIAddrReg && ActiveGuardReg == Src)
345 return;
346
347 // add Dest, LFIBaseReg, W(Src), uxtw
348 MCInst Inst;
349 Inst.setOpcode(AArch64::ADDXrx);
350 Inst.addOperand(Op: MCOperand::createReg(Reg: Dest));
351 Inst.addOperand(Op: MCOperand::createReg(Reg: LFIBaseReg));
352 Inst.addOperand(Op: MCOperand::createReg(Reg: getWRegFromXReg(Reg: Src)));
353 Inst.addOperand(
354 Op: MCOperand::createImm(Val: AArch64_AM::getArithExtendImm(ET: AArch64_AM::UXTW, Imm: 0)));
355 emitInst(Inst, Out, STI);
356
357 // Record Src as the new active guard.
358 if (Dest == LFIAddrReg)
359 ActiveGuardReg = Src;
360}
361
362void AArch64MCLFIRewriter::emitBranch(unsigned Opcode, MCRegister Target,
363 MCStreamer &Out,
364 const MCSubtargetInfo &STI) {
365 MCInst Branch;
366 Branch.setOpcode(Opcode);
367 Branch.addOperand(Op: MCOperand::createReg(Reg: Target));
368 emitInst(Inst: Branch, Out, STI);
369}
370
371void AArch64MCLFIRewriter::emitPendingTLSDescCall(MCStreamer &Out,
372 const MCSubtargetInfo &STI) {
373 if (!PendingTLSDescCall)
374 return;
375 MCInst Marker;
376 Marker.setOpcode(AArch64::TLSDESCCALL);
377 Marker.addOperand(Op: MCOperand::createExpr(Val: PendingTLSDescCall));
378 PendingTLSDescCall = nullptr;
379 emitInst(Inst: Marker, Out, STI);
380}
381
382void AArch64MCLFIRewriter::emitMov(MCRegister Dest, MCRegister Src,
383 MCStreamer &Out,
384 const MCSubtargetInfo &STI) {
385 // orr Dest, xzr, Src
386 MCInst Inst;
387 Inst.setOpcode(AArch64::ORRXrs);
388 Inst.addOperand(Op: MCOperand::createReg(Reg: Dest));
389 Inst.addOperand(Op: MCOperand::createReg(Reg: AArch64::XZR));
390 Inst.addOperand(Op: MCOperand::createReg(Reg: Src));
391 Inst.addOperand(Op: MCOperand::createImm(Val: 0));
392 emitInst(Inst, Out, STI);
393}
394
395void AArch64MCLFIRewriter::emitAddImm(MCRegister Dest, MCRegister Src,
396 int64_t Imm, MCStreamer &Out,
397 const MCSubtargetInfo &STI) {
398 assert(std::abs(Imm) <= 4095);
399 MCInst Inst;
400 if (Imm >= 0) {
401 // add Dest, Src, Imm
402 Inst.setOpcode(AArch64::ADDXri);
403 Inst.addOperand(Op: MCOperand::createReg(Reg: Dest));
404 Inst.addOperand(Op: MCOperand::createReg(Reg: Src));
405 Inst.addOperand(Op: MCOperand::createImm(Val: Imm));
406 Inst.addOperand(Op: MCOperand::createImm(Val: 0)); // shift
407 } else {
408 // sub Dest, Src, -Imm
409 Inst.setOpcode(AArch64::SUBXri);
410 Inst.addOperand(Op: MCOperand::createReg(Reg: Dest));
411 Inst.addOperand(Op: MCOperand::createReg(Reg: Src));
412 Inst.addOperand(Op: MCOperand::createImm(Val: -Imm));
413 Inst.addOperand(Op: MCOperand::createImm(Val: 0)); // shift
414 }
415 emitInst(Inst, Out, STI);
416}
417
418void AArch64MCLFIRewriter::emitAddReg(MCRegister Dest, MCRegister Src1,
419 MCRegister Src2, unsigned Shift,
420 MCStreamer &Out,
421 const MCSubtargetInfo &STI) {
422 // add Dest, Src1, Src2, lsl #Shift
423 MCInst Inst;
424 Inst.setOpcode(AArch64::ADDXrs);
425 Inst.addOperand(Op: MCOperand::createReg(Reg: Dest));
426 Inst.addOperand(Op: MCOperand::createReg(Reg: Src1));
427 Inst.addOperand(Op: MCOperand::createReg(Reg: Src2));
428 Inst.addOperand(
429 Op: MCOperand::createImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: Shift)));
430 emitInst(Inst, Out, STI);
431}
432
433void AArch64MCLFIRewriter::emitAddRegExtend(MCRegister Dest, MCRegister Src1,
434 MCRegister Src2,
435 AArch64_AM::ShiftExtendType ExtType,
436 unsigned Shift, MCStreamer &Out,
437 const MCSubtargetInfo &STI) {
438 // add Dest, Src1, Src2, ExtType #Shift
439 MCInst Inst;
440 if (ExtType == AArch64_AM::SXTX || ExtType == AArch64_AM::UXTX)
441 Inst.setOpcode(AArch64::ADDXrx64);
442 else
443 Inst.setOpcode(AArch64::ADDXrx);
444 Inst.addOperand(Op: MCOperand::createReg(Reg: Dest));
445 Inst.addOperand(Op: MCOperand::createReg(Reg: Src1));
446 Inst.addOperand(Op: MCOperand::createReg(Reg: Src2));
447 Inst.addOperand(
448 Op: MCOperand::createImm(Val: AArch64_AM::getArithExtendImm(ET: ExtType, Imm: Shift)));
449 emitInst(Inst, Out, STI);
450}
451
452void AArch64MCLFIRewriter::emitMemRoW(unsigned Opcode, const MCOperand &DataOp,
453 MCRegister BaseReg, MCStreamer &Out,
454 const MCSubtargetInfo &STI) {
455 // Op DataOp, [LFIBaseReg, W(BaseReg), uxtw]
456 MCInst Inst;
457 Inst.setOpcode(Opcode);
458 Inst.addOperand(Op: DataOp);
459 Inst.addOperand(Op: MCOperand::createReg(Reg: LFIBaseReg));
460 Inst.addOperand(Op: MCOperand::createReg(Reg: getWRegFromXReg(Reg: BaseReg)));
461 Inst.addOperand(Op: MCOperand::createImm(Val: 0)); // S bit = 0 (UXTW).
462 Inst.addOperand(Op: MCOperand::createImm(Val: 0)); // Shift amount = 0 (unscaled).
463 emitInst(Inst, Out, STI);
464}
465
466// {br,blr} xN
467// ->
468// add x28, x27, wN, uxtw
469// {br,blr} x28
470void AArch64MCLFIRewriter::rewriteIndirectBranch(const MCInst &Inst,
471 MCStreamer &Out,
472 const MCSubtargetInfo &STI) {
473 assert(Inst.getNumOperands() >= 1 && Inst.getOperand(0).isReg() &&
474 "expected register operand");
475 MCRegister BranchReg = Inst.getOperand(i: 0).getReg();
476
477 // Guard the branch target through X28.
478 emitAddMask(Dest: LFIAddrReg, Src: BranchReg, Out, STI);
479
480 emitPendingTLSDescCall(Out, STI);
481
482 emitBranch(Opcode: Inst.getOpcode(), Target: LFIAddrReg, Out, STI);
483}
484
485// ret xN (where xN != x30)
486// ->
487// add x28, x27, wN, uxtw
488// ret x28
489//
490// ret (x30) is safe since x30 is always within the sandbox.
491void AArch64MCLFIRewriter::rewriteReturn(const MCInst &Inst, MCStreamer &Out,
492 const MCSubtargetInfo &STI) {
493 assert(Inst.getNumOperands() >= 1 && Inst.getOperand(0).isReg() &&
494 "expected register operand");
495 // RET through LR is safe since LR is always within sandbox.
496 if (Inst.getOperand(i: 0).getReg() != AArch64::LR)
497 rewriteIndirectBranch(Inst, Out, STI);
498 else
499 emitInst(Inst, Out, STI);
500}
501
502// modify x30
503// ->
504// modify x30
505// add x30, x27, w30, uxtw (deferred)
506void AArch64MCLFIRewriter::rewriteLRModification(const MCInst &Inst,
507 MCStreamer &Out,
508 const MCSubtargetInfo &STI) {
509 if (!isFakeMemAccess(Inst) &&
510 (mayLoad(Inst) || mayStore(Inst) || mayPrefetch(Inst)))
511 rewriteLoadStore(Inst, Out, STI);
512 else
513 emitInst(Inst, Out, STI);
514
515 // Defer the LR guard until the next control-flow instruction or label. This
516 // keeps a signed return address intact so that an authentication instruction
517 // can run before the mask destroys the PAC bits.
518 DeferredLRGuard = true;
519}
520
521// retaa / retab
522// ->
523// autiasp / autibsp
524// add x30, x27, w30, uxtw
525// ret
526void AArch64MCLFIRewriter::rewriteAuthenticatedReturn(
527 const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) {
528 MCInst Auth;
529 Auth.setOpcode(Inst.getOpcode() == AArch64::RETAA ? AArch64::AUTIASP
530 : AArch64::AUTIBSP);
531 emitInst(Inst: Auth, Out, STI);
532
533 emitAddMask(Dest: AArch64::LR, Src: AArch64::LR, Out, STI);
534 emitBranch(Opcode: AArch64::RET, Target: AArch64::LR, Out, STI);
535}
536
537// {braa,brab,braaz,brabz} xN[, xM] (blra* for calls)
538// ->
539// {autia,autib,autiza,autizb} xN[, xM]
540// add x28, x27, wN, uxtw
541// {br,blr} x28
542void AArch64MCLFIRewriter::rewriteAuthenticatedBranchOrCall(
543 const MCInst &Inst, unsigned BranchOpcode, MCStreamer &Out,
544 const MCSubtargetInfo &STI) {
545 MCRegister TargetReg = Inst.getOperand(i: 0).getReg();
546
547 // Select the authentication opcode for the target register.
548 MCInst Auth;
549 switch (Inst.getOpcode()) {
550 case AArch64::BRAA:
551 case AArch64::BLRAA:
552 Auth.setOpcode(AArch64::AUTIA);
553 break;
554 case AArch64::BRAB:
555 case AArch64::BLRAB:
556 Auth.setOpcode(AArch64::AUTIB);
557 break;
558 case AArch64::BRAAZ:
559 case AArch64::BLRAAZ:
560 Auth.setOpcode(AArch64::AUTIZA);
561 break;
562 case AArch64::BRABZ:
563 case AArch64::BLRABZ:
564 Auth.setOpcode(AArch64::AUTIZB);
565 break;
566 default:
567 llvm_unreachable("unexpected authenticated branch/call opcode");
568 }
569
570 Auth.addOperand(Op: MCOperand::createReg(Reg: TargetReg)); // dst
571 Auth.addOperand(Op: MCOperand::createReg(Reg: TargetReg)); // src (tied to dst)
572 if (Auth.getOpcode() == AArch64::AUTIA || Auth.getOpcode() == AArch64::AUTIB)
573 Auth.addOperand(Op: Inst.getOperand(i: 1)); // modifier
574 emitInst(Inst: Auth, Out, STI);
575
576 // Guard the authenticated target and branch/call through x28.
577 emitAddMask(Dest: LFIAddrReg, Src: TargetReg, Out, STI);
578 emitBranch(Opcode: BranchOpcode, Target: LFIAddrReg, Out, STI);
579}
580
581// svc #0
582// ->
583// mov x26, x30
584// ldur x30, [x27, #-8]
585// blr x30
586// add x30, x27, w26, uxtw
587void AArch64MCLFIRewriter::rewriteSyscall(const MCInst &, MCStreamer &Out,
588 const MCSubtargetInfo &STI) {
589 // Save LR to scratch.
590 emitMov(Dest: LFIScratchReg, Src: AArch64::LR, Out, STI);
591
592 // Load syscall handler address from negative offset from sandbox base.
593 MCInst Load;
594 Load.setOpcode(AArch64::LDURXi);
595 Load.addOperand(Op: MCOperand::createReg(Reg: AArch64::LR));
596 Load.addOperand(Op: MCOperand::createReg(Reg: LFIBaseReg));
597 Load.addOperand(Op: MCOperand::createImm(Val: LFISyscallOffset));
598 emitInst(Inst: Load, Out, STI);
599
600 // Call the runtime.
601 emitBranch(Opcode: AArch64::BLR, Target: AArch64::LR, Out, STI);
602
603 // Restore LR with guard.
604 emitAddMask(Dest: AArch64::LR, Src: LFIScratchReg, Out, STI);
605}
606
607// mrs xN, tpidr_el0
608// ->
609// ldr xN, [x25, #16]
610void AArch64MCLFIRewriter::rewriteTPRead(const MCInst &Inst, MCStreamer &Out,
611 const MCSubtargetInfo &STI) {
612 MCRegister DestReg = Inst.getOperand(i: 0).getReg();
613
614 MCInst Load;
615 Load.setOpcode(AArch64::LDRXui);
616 Load.addOperand(Op: MCOperand::createReg(Reg: DestReg));
617 Load.addOperand(Op: MCOperand::createReg(Reg: LFICtxReg));
618 Load.addOperand(Op: MCOperand::createImm(Val: LFITPOffset));
619 emitInst(Inst: Load, Out, STI);
620}
621
622// msr tpidr_el0, xN
623// ->
624// str xN, [x25, #16]
625void AArch64MCLFIRewriter::rewriteTPWrite(const MCInst &Inst, MCStreamer &Out,
626 const MCSubtargetInfo &STI) {
627 MCRegister SrcReg = Inst.getOperand(i: 1).getReg();
628
629 MCInst Store;
630 Store.setOpcode(AArch64::STRXui);
631 Store.addOperand(Op: MCOperand::createReg(Reg: SrcReg));
632 Store.addOperand(Op: MCOperand::createReg(Reg: LFICtxReg));
633 Store.addOperand(Op: MCOperand::createImm(Val: LFITPOffset));
634 emitInst(Inst: Store, Out, STI);
635}
636
637bool AArch64MCLFIRewriter::rewriteLoadStoreRoW(const MCInst &Inst,
638 MCStreamer &Out,
639 const MCSubtargetInfo &STI) {
640 unsigned Op = Inst.getOpcode();
641 unsigned MemOp;
642
643 // Case 1: Indexed load/store with zero immediate offset.
644 // ldr xN, [xM, #0] -> ldr xN, [x27, wM, uxtw]
645 if ((MemOp = convertVariantToRoW(Op, ExpectedMode: AArch64::LFI_AM_Ui)) !=
646 AArch64::INSTRUCTION_LIST_END) {
647 MCRegister BaseReg = Inst.getOperand(i: 1).getReg();
648 if (BaseReg == AArch64::SP)
649 return false;
650 const MCOperand &OffsetOp = Inst.getOperand(i: 2);
651 if (OffsetOp.isImm() && OffsetOp.getImm() == 0) {
652 emitMemRoW(Opcode: MemOp, DataOp: Inst.getOperand(i: 0), BaseReg, Out, STI);
653 return true;
654 }
655 return false;
656 }
657
658 // Case 2: Pre-index load/store with writeback.
659 // ldr xN, [xM, #imm]! -> add xM, xM, #imm; ldr xN, [x27, wM, uxtw]
660 if ((MemOp = convertVariantToRoW(Op, ExpectedMode: AArch64::LFI_AM_Pre)) !=
661 AArch64::INSTRUCTION_LIST_END) {
662 MCRegister BaseReg = Inst.getOperand(i: 2).getReg();
663 if (BaseReg == AArch64::SP)
664 return false;
665 int64_t Imm = Inst.getOperand(i: 3).getImm();
666 emitAddImm(Dest: BaseReg, Src: BaseReg, Imm, Out, STI);
667 emitMemRoW(Opcode: MemOp, DataOp: Inst.getOperand(i: 1), BaseReg, Out, STI);
668 return true;
669 }
670
671 // Case 3: Post-index load/store.
672 // ldr xN, [xM], #imm -> ldr xN, [x27, wM, uxtw]; add xM, xM, #imm
673 if ((MemOp = convertVariantToRoW(Op, ExpectedMode: AArch64::LFI_AM_Post)) !=
674 AArch64::INSTRUCTION_LIST_END) {
675 MCRegister BaseReg = Inst.getOperand(i: 2).getReg();
676 if (BaseReg == AArch64::SP)
677 return false;
678 int64_t Imm = Inst.getOperand(i: 3).getImm();
679 emitMemRoW(Opcode: MemOp, DataOp: Inst.getOperand(i: 1), BaseReg, Out, STI);
680 emitAddImm(Dest: BaseReg, Src: BaseReg, Imm, Out, STI);
681 return true;
682 }
683
684 // Case 4: Register-offset-X load/store.
685 // ldr xN, [xM1, xM2] -> add x26, xM1, xM2; ldr xN, [x27, w26, uxtw]
686 //
687 // In this case, even if xM1 is SP we must do a full rewrite, since an
688 // arbitrary register value is being added as the offset.
689 unsigned Shift;
690 if ((MemOp = convertRoXToRoW(Op, Shift)) != AArch64::INSTRUCTION_LIST_END) {
691 MCRegister Reg1 = Inst.getOperand(i: 1).getReg();
692 MCRegister Reg2 = Inst.getOperand(i: 2).getReg();
693 int64_t Extend = Inst.getOperand(i: 3).getImm();
694 int64_t IsShift = Inst.getOperand(i: 4).getImm();
695
696 if (!IsShift)
697 Shift = 0;
698
699 if (Extend)
700 emitAddRegExtend(Dest: LFIScratchReg, Src1: Reg1, Src2: Reg2, ExtType: AArch64_AM::SXTX, Shift, Out,
701 STI);
702 else
703 emitAddReg(Dest: LFIScratchReg, Src1: Reg1, Src2: Reg2, Shift, Out, STI);
704 emitMemRoW(Opcode: MemOp, DataOp: Inst.getOperand(i: 0), BaseReg: LFIScratchReg, Out, STI);
705 return true;
706 }
707
708 // Case 5: Register-offset-W load/store.
709 // ldr xN, [xM1, wM2, uxtw] -> add x26, xM1, wM2, uxtw;
710 // ldr xN, [x27, w26, uxtw]
711 if (getRoWShift(Op, Shift)) {
712 MCRegister Reg1 = Inst.getOperand(i: 1).getReg();
713 MCRegister Reg2 = Inst.getOperand(i: 2).getReg();
714 int64_t S = Inst.getOperand(i: 3).getImm();
715 int64_t IsShift = Inst.getOperand(i: 4).getImm();
716
717 if (!IsShift)
718 Shift = 0;
719
720 if (S)
721 emitAddRegExtend(Dest: LFIScratchReg, Src1: Reg1, Src2: Reg2, ExtType: AArch64_AM::SXTW, Shift, Out,
722 STI);
723 else
724 emitAddRegExtend(Dest: LFIScratchReg, Src1: Reg1, Src2: Reg2, ExtType: AArch64_AM::UXTW, Shift, Out,
725 STI);
726 emitMemRoW(Opcode: Op, DataOp: Inst.getOperand(i: 0), BaseReg: LFIScratchReg, Out, STI);
727 return true;
728 }
729
730 return false;
731}
732
733void AArch64MCLFIRewriter::rewriteLoadStoreBase(const MCInst &Inst,
734 MCStreamer &Out,
735 const MCSubtargetInfo &STI) {
736 unsigned Opcode = Inst.getOpcode();
737 const AArch64::MemInfoEntry *Info = AArch64::lookupMemInfoByOpcode(Inst: Opcode);
738
739 if (!Info) {
740 warning(Inst, Msg: "unknown addressing mode for memory instruction in LFI");
741 return emitInst(Inst, Out, STI);
742 }
743
744 if (Info->IsLiteral)
745 return error(Inst, Msg: "PC-relative literal loads are not supported in LFI");
746
747 MCRegister BaseReg = Inst.getOperand(i: Info->BaseIdx).getReg();
748
749 // Stack accesses don't need address sandboxing, except when sp is modified
750 // with a non-zero register post-index operand.
751 bool BaseIsSP = BaseReg == AArch64::SP;
752 if (BaseIsSP) {
753 if (!Info->HasOffset || !Inst.getOperand(i: Info->OffsetIdx).isReg())
754 return emitInst(Inst, Out, STI);
755 MCRegister OffReg = Inst.getOperand(i: Info->OffsetIdx).getReg();
756 if (OffReg == AArch64::XZR || OffReg == AArch64::WZR)
757 return emitInst(Inst, Out, STI);
758 }
759
760 // Guard the base register, unless it is SP.
761 if (!BaseIsSP)
762 emitAddMask(Dest: LFIAddrReg, Src: BaseReg, Out, STI);
763
764 if (!Info->IsPrePost) {
765 // Non-pre/post instruction: replace the base register operand.
766 MCInst NewInst = replaceRegAt(Inst, Idx: Info->BaseIdx, NewReg: LFIAddrReg);
767 emitInst(Inst: NewInst, Out, STI);
768 return;
769 }
770
771 bool IsPre = false;
772 bool IsNoOffset = false;
773 unsigned BaseOpcode = convertPrePostToBase(Op: Opcode, IsPre, IsNoOffset);
774
775 if (BaseOpcode == AArch64::INSTRUCTION_LIST_END)
776 return error(Inst, Msg: "unhandled pre/post-index instruction in LFI rewriter");
777
778 // Demote pre/post-index to base indexed form.
779 MCInst NewInst;
780 NewInst.setOpcode(BaseOpcode);
781 NewInst.setLoc(Inst.getLoc());
782
783 // Skip writeback operand (operand 0) and copy data operands up to base.
784 for (int I = 1; I < Info->BaseIdx; ++I)
785 NewInst.addOperand(Op: Inst.getOperand(i: I));
786
787 // Add the access base register (LFIAddrReg or SP).
788 MCRegister AccessBase = BaseIsSP ? AArch64::SP : LFIAddrReg;
789 NewInst.addOperand(Op: MCOperand::createReg(Reg: AccessBase));
790
791 // For pre-index, include the offset; for post-index, use zero.
792 if (IsPre && Info->HasOffset)
793 NewInst.addOperand(Op: Inst.getOperand(i: Info->OffsetIdx));
794 else if (!IsNoOffset)
795 NewInst.addOperand(Op: MCOperand::createImm(Val: 0));
796
797 emitInst(Inst: NewInst, Out, STI);
798
799 if (!Info->HasOffset)
800 return;
801
802 // Update the base register with the offset. If the base is SP, a register
803 // offset must be sandboxed (the result is otherwise unbounded), and ADDXrs
804 // cannot take SP, so the extended-register form via the scratch register is
805 // used.
806 const MCOperand &OffsetOp = Inst.getOperand(i: Info->OffsetIdx);
807 if (OffsetOp.isImm()) {
808 // Pair pre/post immediates are scaled by element size; other pre/post
809 // forms (scalar, SIMD) use the raw immediate (scale = 1).
810 int64_t Scale = 1;
811 if (const auto *E = AArch64::lookupPairVariantByOpcode(Inst: Opcode))
812 Scale = E->Scale;
813 int64_t Offset = OffsetOp.getImm() * Scale;
814 emitAddImm(Dest: BaseReg, Src: BaseReg, Imm: Offset, Out, STI);
815 } else if (OffsetOp.isReg()) {
816 // SIMD post-index uses a register offset (XZR for natural offset).
817 MCRegister OffReg = OffsetOp.getReg();
818 if (OffReg == AArch64::XZR) {
819 if (const auto *E = AArch64::lookupSIMDPostByOpcode(Inst: Opcode))
820 emitAddImm(Dest: BaseReg, Src: BaseReg, Imm: E->NaturalOffset, Out, STI);
821 } else if (OffReg != AArch64::WZR) {
822 if (BaseIsSP) {
823 emitAddRegExtend(Dest: LFIScratchReg, Src1: AArch64::SP, Src2: OffReg, ExtType: AArch64_AM::UXTX,
824 Shift: 0, Out, STI);
825 emitAddMask(Dest: AArch64::SP, Src: LFIScratchReg, Out, STI);
826 } else {
827 emitAddReg(Dest: BaseReg, Src1: BaseReg, Src2: OffReg, Shift: 0, Out, STI);
828 }
829 }
830 }
831}
832
833void AArch64MCLFIRewriter::rewriteLoadStore(const MCInst &Inst, MCStreamer &Out,
834 const MCSubtargetInfo &STI) {
835 bool IsStore = mayStore(Inst);
836 bool IsLoad = mayLoad(Inst) || mayPrefetch(Inst);
837
838 bool SkipLoads = STI.hasFeature(Feature: AArch64::FeatureNoLFILoads);
839 bool SkipStores = STI.hasFeature(Feature: AArch64::FeatureNoLFIStores);
840
841 if ((!IsLoad || SkipLoads) && (!IsStore || SkipStores))
842 return emitInst(Inst, Out, STI);
843
844 if (rewriteLoadStoreRoW(Inst, Out, STI))
845 return;
846
847 rewriteLoadStoreBase(Inst, Out, STI);
848}
849
850// modify sp
851// ->
852// modify x26
853// add sp, x27, w26, uxtw
854void AArch64MCLFIRewriter::rewriteSPModification(const MCInst &Inst,
855 MCStreamer &Out,
856 const MCSubtargetInfo &STI) {
857 // Route through rewriteLRModification or rewriteLoadStore for memory
858 // accesses. Those helpers automatically handle dangerous stack modifications
859 // that can happen via register post-index.
860 if (mayLoad(Inst) || mayStore(Inst)) {
861 if (mayModifyRegister(Inst, Reg: AArch64::LR))
862 return rewriteLRModification(Inst, Out, STI);
863 return rewriteLoadStore(Inst, Out, STI);
864 }
865
866 // No stack sandboxing if sandboxing is disabled for both loads and stores.
867 bool SkipLoads = STI.hasFeature(Feature: AArch64::FeatureNoLFILoads);
868 bool SkipStores = STI.hasFeature(Feature: AArch64::FeatureNoLFIStores);
869 if (SkipLoads && SkipStores)
870 return emitInst(Inst, Out, STI);
871
872 // Special case: mov sp, xN -> add sp, x27, wN, uxtw
873 if (Inst.getOpcode() == AArch64::ADDXri && Inst.getOperand(i: 2).getImm() == 0 &&
874 Inst.getOperand(i: 3).getImm() == 0)
875 return emitAddMask(Dest: AArch64::SP, Src: Inst.getOperand(i: 1).getReg(), Out, STI);
876
877 // Redirect SP modification destination to scratch, then sandbox.
878 MCInst ModInst = replaceRegAt(Inst, Idx: 0, NewReg: LFIScratchReg);
879 emitInst(Inst: ModInst, Out, STI);
880 emitAddMask(Dest: AArch64::SP, Src: LFIScratchReg, Out, STI);
881}
882
883// {dc,ic} <op>, xN
884// ->
885// add x28, x27, wN, uxtw
886// {dc,ic} <op>, x28
887void AArch64MCLFIRewriter::rewriteVASysOp(const MCInst &Inst, MCStreamer &Out,
888 const MCSubtargetInfo &STI) {
889 MCRegister AddrReg = Inst.getOperand(i: 4).getReg();
890
891 emitAddMask(Dest: LFIAddrReg, Src: AddrReg, Out, STI);
892
893 MCInst NewInst;
894 NewInst.setOpcode(AArch64::SYSxt);
895 NewInst.addOperand(Op: Inst.getOperand(i: 0));
896 NewInst.addOperand(Op: Inst.getOperand(i: 1));
897 NewInst.addOperand(Op: Inst.getOperand(i: 2));
898 NewInst.addOperand(Op: Inst.getOperand(i: 3));
899 NewInst.addOperand(Op: MCOperand::createReg(Reg: LFIAddrReg));
900 emitInst(Inst: NewInst, Out, STI);
901}
902
903// NOTE: when adding new rewrites, the size estimates in
904// AArch64InstrInfo::getLFIInstSizeInBytes must be updated to match.
905void AArch64MCLFIRewriter::doRewriteInst(const MCInst &Inst, MCStreamer &Out,
906 const MCSubtargetInfo &STI) {
907 if (Inst.getOpcode() == AArch64::TLSDESCCALL) {
908 PendingTLSDescCall = Inst.getOperand(i: 0).getExpr();
909 return;
910 }
911
912 // Reserved register modification is an error.
913 if (MCRegister Reg = mayModifyReserved(Inst)) {
914 error(Inst, Msg: Twine("illegal modification of reserved LFI register ") +
915 RegInfo->getName(RegNo: Reg));
916 return;
917 }
918
919 // System instructions.
920 if (isSyscall(Inst))
921 return rewriteSyscall(Inst, Out, STI);
922
923 if (isTPRead(Inst))
924 return rewriteTPRead(Inst, Out, STI);
925
926 if (isTPWrite(Inst))
927 return rewriteTPWrite(Inst, Out, STI);
928
929 if (isPrivilegedTPAccess(Inst)) {
930 error(Inst, Msg: "illegal access to privileged thread pointer register");
931 return;
932 }
933
934 if (isVASysOp(Inst))
935 return rewriteVASysOp(Inst, Out, STI);
936
937 if (isExceptionReturn(Opcode: Inst.getOpcode())) {
938 error(Inst, Msg: "exception returns are not supported by LFI");
939 return;
940 }
941
942 // PAC authenticated returns expand to authenticate + guarded RET. The
943 // expansion emits its own LR guard, so discard any deferred guard: masking
944 // before the authentication would corrupt the signed return address.
945 if (isAuthenticatedReturn(Opcode: Inst.getOpcode())) {
946 DeferredLRGuard = false;
947 return rewriteAuthenticatedReturn(Inst, Out, STI);
948 }
949
950 // Flush a deferred LR guard before any control-flow instruction, so that a
951 // modified LR is sandboxed before it can be used to transfer control.
952 if (DeferredLRGuard && (isReturn(Inst) || isIndirectBranch(Inst) ||
953 isCall(Inst) || isBranch(Inst))) {
954 emitAddMask(Dest: AArch64::LR, Src: AArch64::LR, Out, STI);
955 DeferredLRGuard = false;
956 }
957
958 // PAC authenticated branches/calls expand to authenticate + guarded branch.
959 if (isAuthenticatedBranch(Opcode: Inst.getOpcode()))
960 return rewriteAuthenticatedBranchOrCall(Inst, BranchOpcode: AArch64::BR, Out, STI);
961 if (isAuthenticatedCall(Opcode: Inst.getOpcode()))
962 return rewriteAuthenticatedBranchOrCall(Inst, BranchOpcode: AArch64::BLR, Out, STI);
963
964 // Control flow.
965 switch (Inst.getOpcode()) {
966 case AArch64::RET:
967 return rewriteReturn(Inst, Out, STI);
968 case AArch64::BR:
969 case AArch64::BLR:
970 return rewriteIndirectBranch(Inst, Out, STI);
971 }
972
973 // Register modifications that require sandboxing.
974 if (mayModifySP(Inst))
975 return rewriteSPModification(Inst, Out, STI);
976
977 // Link register modification. This covers explicit writes to x30 as well as
978 // PAC instructions that write LR in place, which define LR implicitly.
979 if (explicitlyModifiesRegister(Inst, Reg: AArch64::LR) || pacWritesLR(Inst))
980 return rewriteLRModification(Inst, Out, STI);
981
982 // Memory access.
983 if (!isFakeMemAccess(Inst) &&
984 (mayLoad(Inst) || mayStore(Inst) || mayPrefetch(Inst)))
985 return rewriteLoadStore(Inst, Out, STI);
986
987 emitInst(Inst, Out, STI);
988}
989
990// This function is made available to the size estimator so that it can
991// classify Pre/Post-index instructions.
992bool llvm::isLFIPrePostMemAccess(unsigned Opcode) {
993 if (convertVariantToRoW(Op: Opcode, ExpectedMode: AArch64::LFI_AM_Pre) !=
994 AArch64::INSTRUCTION_LIST_END)
995 return true;
996 if (convertVariantToRoW(Op: Opcode, ExpectedMode: AArch64::LFI_AM_Post) !=
997 AArch64::INSTRUCTION_LIST_END)
998 return true;
999 bool IsPre, IsNoOffset;
1000 if (convertPrePostToBase(Op: Opcode, IsPre, IsNoOffset) !=
1001 AArch64::INSTRUCTION_LIST_END)
1002 return true;
1003 return false;
1004}
1005
1006bool AArch64MCLFIRewriter::rewriteInst(const MCInst &Inst, MCStreamer &Out,
1007 const MCSubtargetInfo &STI) {
1008 // Invalidate guard state if the rewriter was manually disabled.
1009 if (!Enabled)
1010 ActiveGuardReg = std::nullopt;
1011
1012 // This recursion guard prevents rewrite-recursion when we emit instructions
1013 // from inside the rewriter (such instructions should not be rewritten).
1014 if (!Enabled || Guard)
1015 return false;
1016 Guard = true;
1017
1018 // Record the subtarget so a deferred LR guard can be emitted from
1019 // onLabel/finish, which are not given an MCSubtargetInfo.
1020 LastSTI = &STI;
1021
1022 doRewriteInst(Inst, Out, STI);
1023
1024 Guard = false;
1025 return true;
1026}
1027