1//===- AArch64MCLFIRewriter.cpp ---------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the AArch64MCLFIRewriter class, the AArch64 specific
10// subclass of MCLFIRewriter.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AArch64MCLFIRewriter.h"
15#include "AArch64AddressingModes.h"
16#include "MCTargetDesc/AArch64MCTargetDesc.h"
17#include "Utils/AArch64BaseInfo.h"
18
19#include "llvm/ADT/Twine.h"
20#include "llvm/MC/MCInst.h"
21#include "llvm/MC/MCInstrDesc.h"
22#include "llvm/MC/MCInstrInfo.h"
23#include "llvm/MC/MCStreamer.h"
24#include "llvm/MC/MCSubtargetInfo.h"
25
26using namespace llvm;
27
28namespace llvm::AArch64 {
29struct LFIVariantEntry {
30 unsigned Inst;
31 uint8_t AddrMode;
32 uint8_t Log2Size;
33 unsigned RoWInst;
34};
35struct PairVariantEntry {
36 unsigned Inst;
37 bool IsPre;
38 uint8_t Scale;
39 unsigned BaseInst;
40};
41struct SIMDPostEntry {
42 unsigned Inst;
43 uint8_t NaturalOffset;
44 unsigned BaseInst;
45};
46struct MemInfoEntry {
47 unsigned Inst;
48 uint8_t BaseIdx;
49 uint8_t OffsetIdx;
50 bool HasOffset;
51 bool IsPrePost;
52 bool IsLiteral;
53};
54
55// LFI addressing-mode codes (must match AArch64LFI.td's LFI_AM_* defs).
56enum LFIAddrMode : uint8_t {
57 LFI_AM_Ui = 0,
58 LFI_AM_RoW = 1,
59 LFI_AM_RoX = 2,
60 LFI_AM_Pre = 3,
61 LFI_AM_Post = 4,
62};
63
64#define GET_LFIVariantTable_DECL
65#define GET_PairVariantTable_DECL
66#define GET_SIMDPostTable_DECL
67#define GET_MemInfoTable_DECL
68#define GET_LFIVariantTable_IMPL
69#define GET_PairVariantTable_IMPL
70#define GET_SIMDPostTable_IMPL
71#define GET_MemInfoTable_IMPL
72// The LFI tables defined in AArch64LFI.td are emitted into this file alongside
73// the system operand tables (single -gen-searchable-tables output).
74#include "AArch64GenSystemOperands.inc"
75} // namespace llvm::AArch64
76
77// LFI reserved registers.
78static constexpr MCRegister LFIBaseReg = AArch64::X27;
79static constexpr MCRegister LFIAddrReg = AArch64::X28;
80static constexpr MCRegister LFIScratchReg = AArch64::X26;
81static constexpr MCRegister LFICtxReg = AArch64::X25;
82
83// Offset into the context register block (pointed to by LFICtxReg) where the
84// thread pointer is stored. This is a scaled offset (multiplied by 8 for
85// 64-bit loads), so a value of 2 means an actual byte offset of 16.
86static constexpr unsigned LFITPOffset = 2;
87
88// Byte offset from the sandbox base register where the syscall handler address
89// is stored (negative because it is below the sandbox base).
90static constexpr int LFISyscallOffset = -8;
91
92static bool isSyscall(const MCInst &Inst) {
93 return Inst.getOpcode() == AArch64::SVC;
94}
95
96static bool isPrivilegedTP(int64_t Reg) {
97 return Reg == AArch64SysReg::TPIDR_EL1 || Reg == AArch64SysReg::TPIDR_EL2 ||
98 Reg == AArch64SysReg::TPIDR_EL3;
99}
100
101static bool isTPRead(const MCInst &Inst) {
102 return Inst.getOpcode() == AArch64::MRS &&
103 Inst.getOperand(i: 1).getImm() == AArch64SysReg::TPIDR_EL0;
104}
105
106static bool isTPWrite(const MCInst &Inst) {
107 return Inst.getOpcode() == AArch64::MSR &&
108 Inst.getOperand(i: 0).getImm() == AArch64SysReg::TPIDR_EL0;
109}
110
111static bool isPrivilegedTPAccess(const MCInst &Inst) {
112 if (Inst.getOpcode() == AArch64::MRS)
113 return isPrivilegedTP(Reg: Inst.getOperand(i: 1).getImm());
114 if (Inst.getOpcode() == AArch64::MSR)
115 return isPrivilegedTP(Reg: Inst.getOperand(i: 0).getImm());
116 return false;
117}
118
119// Classification functions are limited to Armv8.1-A. Instructions outside of
120// this subset are not guaranteed to be rewritten and as a result may fail LFI
121// verification after compilation.
122
123// Instructions that have mayLoad/mayStore set in TableGen but don't actually
124// perform memory accesses.
125static bool isFakeMemAccess(const MCInst &Inst) {
126 switch (Inst.getOpcode()) {
127 case AArch64::CLREX:
128 case AArch64::DMB:
129 case AArch64::DSB:
130 case AArch64::ISB:
131 case AArch64::HINT:
132 // The range of sub-architectures supported by LFI do not include any load
133 // or store instructions in the HINT space.
134 return true;
135 default:
136 return false;
137 }
138}
139
140static bool mayPrefetch(const MCInst &Inst) {
141 switch (Inst.getOpcode()) {
142 case AArch64::PRFMl:
143 case AArch64::PRFMroW:
144 case AArch64::PRFMroX:
145 case AArch64::PRFMui:
146 case AArch64::PRFUMi:
147 return true;
148 default:
149 return false;
150 }
151}
152
153// User-mode DC/IC instructions that take a virtual address operand. Encoded as
154// SYSxt with op1=3, Cn=7, op2=1 where the Cm field selects the operation.
155static bool isVASysOp(const MCInst &Inst) {
156 if (Inst.getOpcode() != AArch64::SYSxt)
157 return false;
158 if (Inst.getOperand(i: 0).getImm() != 3 || Inst.getOperand(i: 1).getImm() != 7 ||
159 Inst.getOperand(i: 3).getImm() != 1)
160 return false;
161 switch (Inst.getOperand(i: 2).getImm()) {
162 case 4: // DC ZVA
163 case 5: // IC IVAU
164 case 10: // DC CVAC
165 case 11: // DC CVAU
166 case 12: // DC CVAP
167 case 13: // DC CVADP
168 case 14: // DC CIVAC
169 return true;
170 default:
171 return false;
172 }
173}
174
175static MCInst replaceRegAt(const MCInst &Inst, unsigned Idx,
176 MCRegister NewReg) {
177 MCInst New = Inst;
178 assert(New.getOperand(Idx).isReg());
179 New.getOperand(i: Idx).setReg(NewReg);
180 return New;
181}
182
183// AArch64 load/store opcode suffixes used throughout this file:
184// Ui: Unsigned immediate offset, scaled by access size: [Xn, #imm].
185// RoW: Register offset with 32-bit W register: [Xn, Wm, uxtw #shift].
186// RoX: Register offset with 64-bit X register: [Xn, Xm, lsl #shift].
187
188// Scalar load/store variant lookup. If Op is a scalar mem instruction with
189// addressing mode ExpectedMode, returns the RoW variant of the same family.
190// Returns INSTRUCTION_LIST_END otherwise.
191static unsigned convertVariantToRoW(unsigned Op, unsigned ExpectedMode) {
192 const AArch64::LFIVariantEntry *E = AArch64::lookupLFIVariantByOpcode(Inst: Op);
193 if (!E || E->AddrMode != ExpectedMode)
194 return AArch64::INSTRUCTION_LIST_END;
195 return E->RoWInst;
196}
197
198static unsigned convertRoXToRoW(unsigned Op, unsigned &Shift) {
199 Shift = 0;
200 const AArch64::LFIVariantEntry *E = AArch64::lookupLFIVariantByOpcode(Inst: Op);
201 if (!E || E->AddrMode != AArch64::LFI_AM_RoX)
202 return AArch64::INSTRUCTION_LIST_END;
203 Shift = E->Log2Size;
204 return E->RoWInst;
205}
206
207static bool getRoWShift(unsigned Op, unsigned &Shift) {
208 Shift = 0;
209 const AArch64::LFIVariantEntry *E = AArch64::lookupLFIVariantByOpcode(Inst: Op);
210 if (!E || E->AddrMode != AArch64::LFI_AM_RoW)
211 return false;
212 Shift = E->Log2Size;
213 return true;
214}
215
216// Pre/post-index conversion to base form. Both LDP/STP pair pre/post forms and
217// SIMD post-index forms come from generated lookup tables. The pair table sets
218// IsPre to distinguish pre-index from post-index. The SIMD table is
219// post-index-only so IsNoOffset is set to indicate the demoted base form takes
220// no immediate offset.
221static unsigned convertPrePostToBase(unsigned Op, bool &IsPre,
222 bool &IsNoOffset) {
223 IsPre = false;
224 IsNoOffset = false;
225 if (const auto *E = AArch64::lookupPairVariantByOpcode(Inst: Op)) {
226 IsPre = E->IsPre;
227 return E->BaseInst;
228 }
229 if (const auto *E = AArch64::lookupSIMDPostByOpcode(Inst: Op)) {
230 IsNoOffset = true;
231 return E->BaseInst;
232 }
233 return AArch64::INSTRUCTION_LIST_END;
234}
235
236bool AArch64MCLFIRewriter::mayModifySP(const MCInst &Inst) const {
237 return mayModifyRegister(Inst, Reg: AArch64::SP);
238}
239
240MCRegister AArch64MCLFIRewriter::mayModifyReserved(const MCInst &Inst) const {
241 for (MCRegister Reg : {LFIAddrReg, LFIBaseReg, LFICtxReg}) {
242 if (mayModifyRegister(Inst, Reg))
243 return Reg;
244 }
245 return {};
246}
247
248void AArch64MCLFIRewriter::emitInst(const MCInst &Inst, MCStreamer &Out,
249 const MCSubtargetInfo &STI) {
250 Out.emitInstruction(Inst, STI);
251}
252
253void AArch64MCLFIRewriter::emitAddMask(MCRegister Dest, MCRegister Src,
254 MCStreamer &Out,
255 const MCSubtargetInfo &STI) {
256 // add Dest, LFIBaseReg, W(Src), uxtw
257 MCInst Inst;
258 Inst.setOpcode(AArch64::ADDXrx);
259 Inst.addOperand(Op: MCOperand::createReg(Reg: Dest));
260 Inst.addOperand(Op: MCOperand::createReg(Reg: LFIBaseReg));
261 Inst.addOperand(Op: MCOperand::createReg(Reg: getWRegFromXReg(Reg: Src)));
262 Inst.addOperand(
263 Op: MCOperand::createImm(Val: AArch64_AM::getArithExtendImm(ET: AArch64_AM::UXTW, Imm: 0)));
264 emitInst(Inst, Out, STI);
265}
266
267void AArch64MCLFIRewriter::emitBranch(unsigned Opcode, MCRegister Target,
268 MCStreamer &Out,
269 const MCSubtargetInfo &STI) {
270 MCInst Branch;
271 Branch.setOpcode(Opcode);
272 Branch.addOperand(Op: MCOperand::createReg(Reg: Target));
273 emitInst(Inst: Branch, Out, STI);
274}
275
276void AArch64MCLFIRewriter::emitPendingTLSDescCall(MCStreamer &Out,
277 const MCSubtargetInfo &STI) {
278 if (!PendingTLSDescCall)
279 return;
280 MCInst Marker;
281 Marker.setOpcode(AArch64::TLSDESCCALL);
282 Marker.addOperand(Op: MCOperand::createExpr(Val: PendingTLSDescCall));
283 PendingTLSDescCall = nullptr;
284 emitInst(Inst: Marker, Out, STI);
285}
286
287void AArch64MCLFIRewriter::emitMov(MCRegister Dest, MCRegister Src,
288 MCStreamer &Out,
289 const MCSubtargetInfo &STI) {
290 // orr Dest, xzr, Src
291 MCInst Inst;
292 Inst.setOpcode(AArch64::ORRXrs);
293 Inst.addOperand(Op: MCOperand::createReg(Reg: Dest));
294 Inst.addOperand(Op: MCOperand::createReg(Reg: AArch64::XZR));
295 Inst.addOperand(Op: MCOperand::createReg(Reg: Src));
296 Inst.addOperand(Op: MCOperand::createImm(Val: 0));
297 emitInst(Inst, Out, STI);
298}
299
300void AArch64MCLFIRewriter::emitAddImm(MCRegister Dest, MCRegister Src,
301 int64_t Imm, MCStreamer &Out,
302 const MCSubtargetInfo &STI) {
303 assert(std::abs(Imm) <= 4095);
304 MCInst Inst;
305 if (Imm >= 0) {
306 // add Dest, Src, Imm
307 Inst.setOpcode(AArch64::ADDXri);
308 Inst.addOperand(Op: MCOperand::createReg(Reg: Dest));
309 Inst.addOperand(Op: MCOperand::createReg(Reg: Src));
310 Inst.addOperand(Op: MCOperand::createImm(Val: Imm));
311 Inst.addOperand(Op: MCOperand::createImm(Val: 0)); // shift
312 } else {
313 // sub Dest, Src, -Imm
314 Inst.setOpcode(AArch64::SUBXri);
315 Inst.addOperand(Op: MCOperand::createReg(Reg: Dest));
316 Inst.addOperand(Op: MCOperand::createReg(Reg: Src));
317 Inst.addOperand(Op: MCOperand::createImm(Val: -Imm));
318 Inst.addOperand(Op: MCOperand::createImm(Val: 0)); // shift
319 }
320 emitInst(Inst, Out, STI);
321}
322
323void AArch64MCLFIRewriter::emitAddReg(MCRegister Dest, MCRegister Src1,
324 MCRegister Src2, unsigned Shift,
325 MCStreamer &Out,
326 const MCSubtargetInfo &STI) {
327 // add Dest, Src1, Src2, lsl #Shift
328 MCInst Inst;
329 Inst.setOpcode(AArch64::ADDXrs);
330 Inst.addOperand(Op: MCOperand::createReg(Reg: Dest));
331 Inst.addOperand(Op: MCOperand::createReg(Reg: Src1));
332 Inst.addOperand(Op: MCOperand::createReg(Reg: Src2));
333 Inst.addOperand(
334 Op: MCOperand::createImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: Shift)));
335 emitInst(Inst, Out, STI);
336}
337
338void AArch64MCLFIRewriter::emitAddRegExtend(MCRegister Dest, MCRegister Src1,
339 MCRegister Src2,
340 AArch64_AM::ShiftExtendType ExtType,
341 unsigned Shift, MCStreamer &Out,
342 const MCSubtargetInfo &STI) {
343 // add Dest, Src1, Src2, ExtType #Shift
344 MCInst Inst;
345 if (ExtType == AArch64_AM::SXTX || ExtType == AArch64_AM::UXTX)
346 Inst.setOpcode(AArch64::ADDXrx64);
347 else
348 Inst.setOpcode(AArch64::ADDXrx);
349 Inst.addOperand(Op: MCOperand::createReg(Reg: Dest));
350 Inst.addOperand(Op: MCOperand::createReg(Reg: Src1));
351 Inst.addOperand(Op: MCOperand::createReg(Reg: Src2));
352 Inst.addOperand(
353 Op: MCOperand::createImm(Val: AArch64_AM::getArithExtendImm(ET: ExtType, Imm: Shift)));
354 emitInst(Inst, Out, STI);
355}
356
357void AArch64MCLFIRewriter::emitMemRoW(unsigned Opcode, const MCOperand &DataOp,
358 MCRegister BaseReg, MCStreamer &Out,
359 const MCSubtargetInfo &STI) {
360 // Op DataOp, [LFIBaseReg, W(BaseReg), uxtw]
361 MCInst Inst;
362 Inst.setOpcode(Opcode);
363 Inst.addOperand(Op: DataOp);
364 Inst.addOperand(Op: MCOperand::createReg(Reg: LFIBaseReg));
365 Inst.addOperand(Op: MCOperand::createReg(Reg: getWRegFromXReg(Reg: BaseReg)));
366 Inst.addOperand(Op: MCOperand::createImm(Val: 0)); // S bit = 0 (UXTW).
367 Inst.addOperand(Op: MCOperand::createImm(Val: 0)); // Shift amount = 0 (unscaled).
368 emitInst(Inst, Out, STI);
369}
370
371// {br,blr} xN
372// ->
373// add x28, x27, wN, uxtw
374// {br,blr} x28
375void AArch64MCLFIRewriter::rewriteIndirectBranch(const MCInst &Inst,
376 MCStreamer &Out,
377 const MCSubtargetInfo &STI) {
378 assert(Inst.getNumOperands() >= 1 && Inst.getOperand(0).isReg() &&
379 "expected register operand");
380 MCRegister BranchReg = Inst.getOperand(i: 0).getReg();
381
382 // Guard the branch target through X28.
383 emitAddMask(Dest: LFIAddrReg, Src: BranchReg, Out, STI);
384
385 emitPendingTLSDescCall(Out, STI);
386
387 emitBranch(Opcode: Inst.getOpcode(), Target: LFIAddrReg, Out, STI);
388}
389
390// ret xN (where xN != x30)
391// ->
392// add x28, x27, wN, uxtw
393// ret x28
394//
395// ret (x30) is safe since x30 is always within the sandbox.
396void AArch64MCLFIRewriter::rewriteReturn(const MCInst &Inst, MCStreamer &Out,
397 const MCSubtargetInfo &STI) {
398 assert(Inst.getNumOperands() >= 1 && Inst.getOperand(0).isReg() &&
399 "expected register operand");
400 // RET through LR is safe since LR is always within sandbox.
401 if (Inst.getOperand(i: 0).getReg() != AArch64::LR)
402 rewriteIndirectBranch(Inst, Out, STI);
403 else
404 emitInst(Inst, Out, STI);
405}
406
407// modify x30
408// ->
409// modify x30
410// add x30, x27, w30, uxtw
411void AArch64MCLFIRewriter::rewriteLRModification(const MCInst &Inst,
412 MCStreamer &Out,
413 const MCSubtargetInfo &STI) {
414 if (!isFakeMemAccess(Inst) &&
415 (mayLoad(Inst) || mayStore(Inst) || mayPrefetch(Inst)))
416 rewriteLoadStore(Inst, Out, STI);
417 else
418 emitInst(Inst, Out, STI);
419 emitAddMask(Dest: AArch64::LR, Src: AArch64::LR, Out, STI);
420}
421
422// svc #0
423// ->
424// mov x26, x30
425// ldur x30, [x27, #-8]
426// blr x30
427// add x30, x27, w26, uxtw
428void AArch64MCLFIRewriter::rewriteSyscall(const MCInst &, MCStreamer &Out,
429 const MCSubtargetInfo &STI) {
430 // Save LR to scratch.
431 emitMov(Dest: LFIScratchReg, Src: AArch64::LR, Out, STI);
432
433 // Load syscall handler address from negative offset from sandbox base.
434 MCInst Load;
435 Load.setOpcode(AArch64::LDURXi);
436 Load.addOperand(Op: MCOperand::createReg(Reg: AArch64::LR));
437 Load.addOperand(Op: MCOperand::createReg(Reg: LFIBaseReg));
438 Load.addOperand(Op: MCOperand::createImm(Val: LFISyscallOffset));
439 emitInst(Inst: Load, Out, STI);
440
441 // Call the runtime.
442 emitBranch(Opcode: AArch64::BLR, Target: AArch64::LR, Out, STI);
443
444 // Restore LR with guard.
445 emitAddMask(Dest: AArch64::LR, Src: LFIScratchReg, Out, STI);
446}
447
448// mrs xN, tpidr_el0
449// ->
450// ldr xN, [x25, #16]
451void AArch64MCLFIRewriter::rewriteTPRead(const MCInst &Inst, MCStreamer &Out,
452 const MCSubtargetInfo &STI) {
453 MCRegister DestReg = Inst.getOperand(i: 0).getReg();
454
455 MCInst Load;
456 Load.setOpcode(AArch64::LDRXui);
457 Load.addOperand(Op: MCOperand::createReg(Reg: DestReg));
458 Load.addOperand(Op: MCOperand::createReg(Reg: LFICtxReg));
459 Load.addOperand(Op: MCOperand::createImm(Val: LFITPOffset));
460 emitInst(Inst: Load, Out, STI);
461}
462
463// msr tpidr_el0, xN
464// ->
465// str xN, [x25, #16]
466void AArch64MCLFIRewriter::rewriteTPWrite(const MCInst &Inst, MCStreamer &Out,
467 const MCSubtargetInfo &STI) {
468 MCRegister SrcReg = Inst.getOperand(i: 1).getReg();
469
470 MCInst Store;
471 Store.setOpcode(AArch64::STRXui);
472 Store.addOperand(Op: MCOperand::createReg(Reg: SrcReg));
473 Store.addOperand(Op: MCOperand::createReg(Reg: LFICtxReg));
474 Store.addOperand(Op: MCOperand::createImm(Val: LFITPOffset));
475 emitInst(Inst: Store, Out, STI);
476}
477
478bool AArch64MCLFIRewriter::rewriteLoadStoreRoW(const MCInst &Inst,
479 MCStreamer &Out,
480 const MCSubtargetInfo &STI) {
481 unsigned Op = Inst.getOpcode();
482 unsigned MemOp;
483
484 // Case 1: Indexed load/store with zero immediate offset.
485 // ldr xN, [xM, #0] -> ldr xN, [x27, wM, uxtw]
486 if ((MemOp = convertVariantToRoW(Op, ExpectedMode: AArch64::LFI_AM_Ui)) !=
487 AArch64::INSTRUCTION_LIST_END) {
488 MCRegister BaseReg = Inst.getOperand(i: 1).getReg();
489 if (BaseReg == AArch64::SP)
490 return false;
491 const MCOperand &OffsetOp = Inst.getOperand(i: 2);
492 if (OffsetOp.isImm() && OffsetOp.getImm() == 0) {
493 emitMemRoW(Opcode: MemOp, DataOp: Inst.getOperand(i: 0), BaseReg, Out, STI);
494 return true;
495 }
496 return false;
497 }
498
499 // Case 2: Pre-index load/store with writeback.
500 // ldr xN, [xM, #imm]! -> add xM, xM, #imm; ldr xN, [x27, wM, uxtw]
501 if ((MemOp = convertVariantToRoW(Op, ExpectedMode: AArch64::LFI_AM_Pre)) !=
502 AArch64::INSTRUCTION_LIST_END) {
503 MCRegister BaseReg = Inst.getOperand(i: 2).getReg();
504 if (BaseReg == AArch64::SP)
505 return false;
506 int64_t Imm = Inst.getOperand(i: 3).getImm();
507 emitAddImm(Dest: BaseReg, Src: BaseReg, Imm, Out, STI);
508 emitMemRoW(Opcode: MemOp, DataOp: Inst.getOperand(i: 1), BaseReg, Out, STI);
509 return true;
510 }
511
512 // Case 3: Post-index load/store.
513 // ldr xN, [xM], #imm -> ldr xN, [x27, wM, uxtw]; add xM, xM, #imm
514 if ((MemOp = convertVariantToRoW(Op, ExpectedMode: AArch64::LFI_AM_Post)) !=
515 AArch64::INSTRUCTION_LIST_END) {
516 MCRegister BaseReg = Inst.getOperand(i: 2).getReg();
517 if (BaseReg == AArch64::SP)
518 return false;
519 int64_t Imm = Inst.getOperand(i: 3).getImm();
520 emitMemRoW(Opcode: MemOp, DataOp: Inst.getOperand(i: 1), BaseReg, Out, STI);
521 emitAddImm(Dest: BaseReg, Src: BaseReg, Imm, Out, STI);
522 return true;
523 }
524
525 // Case 4: Register-offset-X load/store.
526 // ldr xN, [xM1, xM2] -> add x26, xM1, xM2; ldr xN, [x27, w26, uxtw]
527 //
528 // In this case, even if xM1 is SP we must do a full rewrite, since an
529 // arbitrary register value is being added as the offset.
530 unsigned Shift;
531 if ((MemOp = convertRoXToRoW(Op, Shift)) != AArch64::INSTRUCTION_LIST_END) {
532 MCRegister Reg1 = Inst.getOperand(i: 1).getReg();
533 MCRegister Reg2 = Inst.getOperand(i: 2).getReg();
534 int64_t Extend = Inst.getOperand(i: 3).getImm();
535 int64_t IsShift = Inst.getOperand(i: 4).getImm();
536
537 if (!IsShift)
538 Shift = 0;
539
540 if (Extend)
541 emitAddRegExtend(Dest: LFIScratchReg, Src1: Reg1, Src2: Reg2, ExtType: AArch64_AM::SXTX, Shift, Out,
542 STI);
543 else
544 emitAddReg(Dest: LFIScratchReg, Src1: Reg1, Src2: Reg2, Shift, Out, STI);
545 emitMemRoW(Opcode: MemOp, DataOp: Inst.getOperand(i: 0), BaseReg: LFIScratchReg, Out, STI);
546 return true;
547 }
548
549 // Case 5: Register-offset-W load/store.
550 // ldr xN, [xM1, wM2, uxtw] -> add x26, xM1, wM2, uxtw;
551 // ldr xN, [x27, w26, uxtw]
552 if (getRoWShift(Op, Shift)) {
553 MCRegister Reg1 = Inst.getOperand(i: 1).getReg();
554 MCRegister Reg2 = Inst.getOperand(i: 2).getReg();
555 int64_t S = Inst.getOperand(i: 3).getImm();
556 int64_t IsShift = Inst.getOperand(i: 4).getImm();
557
558 if (!IsShift)
559 Shift = 0;
560
561 if (S)
562 emitAddRegExtend(Dest: LFIScratchReg, Src1: Reg1, Src2: Reg2, ExtType: AArch64_AM::SXTW, Shift, Out,
563 STI);
564 else
565 emitAddRegExtend(Dest: LFIScratchReg, Src1: Reg1, Src2: Reg2, ExtType: AArch64_AM::UXTW, Shift, Out,
566 STI);
567 emitMemRoW(Opcode: Op, DataOp: Inst.getOperand(i: 0), BaseReg: LFIScratchReg, Out, STI);
568 return true;
569 }
570
571 return false;
572}
573
574void AArch64MCLFIRewriter::rewriteLoadStoreBase(const MCInst &Inst,
575 MCStreamer &Out,
576 const MCSubtargetInfo &STI) {
577 unsigned Opcode = Inst.getOpcode();
578 const AArch64::MemInfoEntry *Info = AArch64::lookupMemInfoByOpcode(Inst: Opcode);
579
580 if (!Info) {
581 warning(Inst, Msg: "unknown addressing mode for memory instruction in LFI");
582 return emitInst(Inst, Out, STI);
583 }
584
585 if (Info->IsLiteral)
586 return error(Inst, Msg: "PC-relative literal loads are not supported in LFI");
587
588 MCRegister BaseReg = Inst.getOperand(i: Info->BaseIdx).getReg();
589
590 // Stack accesses don't need address sandboxing, except when sp is modified
591 // with a non-zero register post-index operand.
592 bool BaseIsSP = BaseReg == AArch64::SP;
593 if (BaseIsSP) {
594 if (!Info->HasOffset || !Inst.getOperand(i: Info->OffsetIdx).isReg())
595 return emitInst(Inst, Out, STI);
596 MCRegister OffReg = Inst.getOperand(i: Info->OffsetIdx).getReg();
597 if (OffReg == AArch64::XZR || OffReg == AArch64::WZR)
598 return emitInst(Inst, Out, STI);
599 }
600
601 // Guard the base register, unless it is SP.
602 if (!BaseIsSP)
603 emitAddMask(Dest: LFIAddrReg, Src: BaseReg, Out, STI);
604
605 if (!Info->IsPrePost) {
606 // Non-pre/post instruction: replace the base register operand.
607 MCInst NewInst = replaceRegAt(Inst, Idx: Info->BaseIdx, NewReg: LFIAddrReg);
608 emitInst(Inst: NewInst, Out, STI);
609 return;
610 }
611
612 bool IsPre = false;
613 bool IsNoOffset = false;
614 unsigned BaseOpcode = convertPrePostToBase(Op: Opcode, IsPre, IsNoOffset);
615
616 if (BaseOpcode == AArch64::INSTRUCTION_LIST_END)
617 return error(Inst, Msg: "unhandled pre/post-index instruction in LFI rewriter");
618
619 // Demote pre/post-index to base indexed form.
620 MCInst NewInst;
621 NewInst.setOpcode(BaseOpcode);
622 NewInst.setLoc(Inst.getLoc());
623
624 // Skip writeback operand (operand 0) and copy data operands up to base.
625 for (int I = 1; I < Info->BaseIdx; ++I)
626 NewInst.addOperand(Op: Inst.getOperand(i: I));
627
628 // Add the access base register (LFIAddrReg or SP).
629 MCRegister AccessBase = BaseIsSP ? AArch64::SP : LFIAddrReg;
630 NewInst.addOperand(Op: MCOperand::createReg(Reg: AccessBase));
631
632 // For pre-index, include the offset; for post-index, use zero.
633 if (IsPre && Info->HasOffset)
634 NewInst.addOperand(Op: Inst.getOperand(i: Info->OffsetIdx));
635 else if (!IsNoOffset)
636 NewInst.addOperand(Op: MCOperand::createImm(Val: 0));
637
638 emitInst(Inst: NewInst, Out, STI);
639
640 if (!Info->HasOffset)
641 return;
642
643 // Update the base register with the offset. If the base is SP, a register
644 // offset must be sandboxed (the result is otherwise unbounded), and ADDXrs
645 // cannot take SP, so the extended-register form via the scratch register is
646 // used.
647 const MCOperand &OffsetOp = Inst.getOperand(i: Info->OffsetIdx);
648 if (OffsetOp.isImm()) {
649 // Pair pre/post immediates are scaled by element size; other pre/post
650 // forms (scalar, SIMD) use the raw immediate (scale = 1).
651 int64_t Scale = 1;
652 if (const auto *E = AArch64::lookupPairVariantByOpcode(Inst: Opcode))
653 Scale = E->Scale;
654 int64_t Offset = OffsetOp.getImm() * Scale;
655 emitAddImm(Dest: BaseReg, Src: BaseReg, Imm: Offset, Out, STI);
656 } else if (OffsetOp.isReg()) {
657 // SIMD post-index uses a register offset (XZR for natural offset).
658 MCRegister OffReg = OffsetOp.getReg();
659 if (OffReg == AArch64::XZR) {
660 if (const auto *E = AArch64::lookupSIMDPostByOpcode(Inst: Opcode))
661 emitAddImm(Dest: BaseReg, Src: BaseReg, Imm: E->NaturalOffset, Out, STI);
662 } else if (OffReg != AArch64::WZR) {
663 if (BaseIsSP) {
664 emitAddRegExtend(Dest: LFIScratchReg, Src1: AArch64::SP, Src2: OffReg, ExtType: AArch64_AM::UXTX,
665 Shift: 0, Out, STI);
666 emitAddMask(Dest: AArch64::SP, Src: LFIScratchReg, Out, STI);
667 } else {
668 emitAddReg(Dest: BaseReg, Src1: BaseReg, Src2: OffReg, Shift: 0, Out, STI);
669 }
670 }
671 }
672}
673
674void AArch64MCLFIRewriter::rewriteLoadStore(const MCInst &Inst, MCStreamer &Out,
675 const MCSubtargetInfo &STI) {
676 bool IsStore = mayStore(Inst);
677 bool IsLoad = mayLoad(Inst) || mayPrefetch(Inst);
678
679 bool SkipLoads = STI.hasFeature(Feature: AArch64::FeatureNoLFILoads);
680 bool SkipStores = STI.hasFeature(Feature: AArch64::FeatureNoLFIStores);
681
682 if ((!IsLoad || SkipLoads) && (!IsStore || SkipStores))
683 return emitInst(Inst, Out, STI);
684
685 if (rewriteLoadStoreRoW(Inst, Out, STI))
686 return;
687
688 rewriteLoadStoreBase(Inst, Out, STI);
689}
690
691// modify sp
692// ->
693// modify x26
694// add sp, x27, w26, uxtw
695void AArch64MCLFIRewriter::rewriteSPModification(const MCInst &Inst,
696 MCStreamer &Out,
697 const MCSubtargetInfo &STI) {
698 // Route through rewriteLRModification or rewriteLoadStore for memory
699 // accesses. Those helpers automatically handle dangerous stack modifications
700 // that can happen via register post-index.
701 if (mayLoad(Inst) || mayStore(Inst)) {
702 if (mayModifyRegister(Inst, Reg: AArch64::LR))
703 return rewriteLRModification(Inst, Out, STI);
704 return rewriteLoadStore(Inst, Out, STI);
705 }
706
707 // No stack sandboxing if sandboxing is disabled for both loads and stores.
708 bool SkipLoads = STI.hasFeature(Feature: AArch64::FeatureNoLFILoads);
709 bool SkipStores = STI.hasFeature(Feature: AArch64::FeatureNoLFIStores);
710 if (SkipLoads && SkipStores)
711 return emitInst(Inst, Out, STI);
712
713 // Special case: mov sp, xN -> add sp, x27, wN, uxtw
714 if (Inst.getOpcode() == AArch64::ADDXri && Inst.getOperand(i: 2).getImm() == 0 &&
715 Inst.getOperand(i: 3).getImm() == 0)
716 return emitAddMask(Dest: AArch64::SP, Src: Inst.getOperand(i: 1).getReg(), Out, STI);
717
718 // Redirect SP modification destination to scratch, then sandbox.
719 MCInst ModInst = replaceRegAt(Inst, Idx: 0, NewReg: LFIScratchReg);
720 emitInst(Inst: ModInst, Out, STI);
721 emitAddMask(Dest: AArch64::SP, Src: LFIScratchReg, Out, STI);
722}
723
724// {dc,ic} <op>, xN
725// ->
726// add x28, x27, wN, uxtw
727// {dc,ic} <op>, x28
728void AArch64MCLFIRewriter::rewriteVASysOp(const MCInst &Inst, MCStreamer &Out,
729 const MCSubtargetInfo &STI) {
730 MCRegister AddrReg = Inst.getOperand(i: 4).getReg();
731
732 emitAddMask(Dest: LFIAddrReg, Src: AddrReg, Out, STI);
733
734 MCInst NewInst;
735 NewInst.setOpcode(AArch64::SYSxt);
736 NewInst.addOperand(Op: Inst.getOperand(i: 0));
737 NewInst.addOperand(Op: Inst.getOperand(i: 1));
738 NewInst.addOperand(Op: Inst.getOperand(i: 2));
739 NewInst.addOperand(Op: Inst.getOperand(i: 3));
740 NewInst.addOperand(Op: MCOperand::createReg(Reg: LFIAddrReg));
741 emitInst(Inst: NewInst, Out, STI);
742}
743
744// NOTE: when adding new rewrites, the size estimates in
745// AArch64InstrInfo::getLFIInstSizeInBytes must be updated to match.
746void AArch64MCLFIRewriter::doRewriteInst(const MCInst &Inst, MCStreamer &Out,
747 const MCSubtargetInfo &STI) {
748 if (Inst.getOpcode() == AArch64::TLSDESCCALL) {
749 PendingTLSDescCall = Inst.getOperand(i: 0).getExpr();
750 return;
751 }
752
753 // Reserved register modification is an error.
754 if (MCRegister Reg = mayModifyReserved(Inst)) {
755 error(Inst, Msg: Twine("illegal modification of reserved LFI register ") +
756 RegInfo->getName(RegNo: Reg));
757 return;
758 }
759
760 // System instructions.
761 if (isSyscall(Inst))
762 return rewriteSyscall(Inst, Out, STI);
763
764 if (isTPRead(Inst))
765 return rewriteTPRead(Inst, Out, STI);
766
767 if (isTPWrite(Inst))
768 return rewriteTPWrite(Inst, Out, STI);
769
770 if (isPrivilegedTPAccess(Inst)) {
771 error(Inst, Msg: "illegal access to privileged thread pointer register");
772 return;
773 }
774
775 if (isVASysOp(Inst))
776 return rewriteVASysOp(Inst, Out, STI);
777
778 // Control flow.
779 switch (Inst.getOpcode()) {
780 case AArch64::RET:
781 return rewriteReturn(Inst, Out, STI);
782 case AArch64::BR:
783 case AArch64::BLR:
784 return rewriteIndirectBranch(Inst, Out, STI);
785 }
786
787 // Register modifications that require sandboxing.
788 if (mayModifySP(Inst))
789 return rewriteSPModification(Inst, Out, STI);
790
791 // Link register modification.
792 if (explicitlyModifiesRegister(Inst, Reg: AArch64::LR))
793 return rewriteLRModification(Inst, Out, STI);
794
795 // Memory access.
796 if (!isFakeMemAccess(Inst) &&
797 (mayLoad(Inst) || mayStore(Inst) || mayPrefetch(Inst)))
798 return rewriteLoadStore(Inst, Out, STI);
799
800 emitInst(Inst, Out, STI);
801}
802
803// This function is made available to the size estimator so that it can
804// classify Pre/Post-index instructions.
805bool llvm::isLFIPrePostMemAccess(unsigned Opcode) {
806 if (convertVariantToRoW(Op: Opcode, ExpectedMode: AArch64::LFI_AM_Pre) !=
807 AArch64::INSTRUCTION_LIST_END)
808 return true;
809 if (convertVariantToRoW(Op: Opcode, ExpectedMode: AArch64::LFI_AM_Post) !=
810 AArch64::INSTRUCTION_LIST_END)
811 return true;
812 bool IsPre, IsNoOffset;
813 if (convertPrePostToBase(Op: Opcode, IsPre, IsNoOffset) !=
814 AArch64::INSTRUCTION_LIST_END)
815 return true;
816 return false;
817}
818
819bool AArch64MCLFIRewriter::rewriteInst(const MCInst &Inst, MCStreamer &Out,
820 const MCSubtargetInfo &STI) {
821 // The guard prevents rewrite-recursion when we emit instructions from inside
822 // the rewriter (such instructions should not be rewritten).
823 if (!Enabled || Guard)
824 return false;
825 Guard = true;
826
827 doRewriteInst(Inst, Out, STI);
828
829 Guard = false;
830 return true;
831}
832