1//===-- AArch64PointerAuth.cpp -- Harden code using PAuth ------------------==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AArch64PointerAuth.h"
10
11#include "AArch64.h"
12#include "AArch64FrameLowering.h"
13#include "AArch64InstrInfo.h"
14#include "AArch64MachineFunctionInfo.h"
15#include "AArch64RegisterInfo.h"
16#include "AArch64Subtarget.h"
17#include "MCTargetDesc/AArch64AddressingModes.h"
18#include "llvm/CodeGen/CFIInstBuilder.h"
19#include "llvm/CodeGen/MachineBasicBlock.h"
20#include "llvm/CodeGen/MachineInstrBuilder.h"
21#include "llvm/CodeGen/MachineModuleInfo.h"
22#include "llvm/CodeGen/RegisterScavenging.h"
23#include "llvm/IR/Module.h"
24
25using namespace llvm;
26using namespace llvm::AArch64PAuth;
27
28#define AARCH64_POINTER_AUTH_NAME "AArch64 Pointer Authentication"
29
30namespace {
31
32class AArch64PointerAuthImpl {
33public:
34 bool run(MachineFunction &MF);
35
36private:
37 const AArch64Subtarget *Subtarget = nullptr;
38 const AArch64InstrInfo *TII = nullptr;
39
40 void signLR(MachineFunction &MF, MachineBasicBlock::iterator MBBI) const;
41
42 void authenticateLR(MachineFunction &MF,
43 MachineBasicBlock::iterator MBBI) const;
44
45 bool emitSignReturnAddressHardening(MachineFunction &MF);
46};
47
48class AArch64PointerAuthLegacy : public MachineFunctionPass {
49public:
50 static char ID;
51
52 AArch64PointerAuthLegacy() : MachineFunctionPass(ID) {}
53
54 bool runOnMachineFunction(MachineFunction &MF) override;
55
56 StringRef getPassName() const override { return AARCH64_POINTER_AUTH_NAME; }
57};
58
59} // end anonymous namespace
60
61INITIALIZE_PASS(AArch64PointerAuthLegacy, "aarch64-ptrauth",
62 AARCH64_POINTER_AUTH_NAME, false, false)
63
64FunctionPass *llvm::createAArch64PointerAuthPass() {
65 return new AArch64PointerAuthLegacy();
66}
67
68char AArch64PointerAuthLegacy::ID = 0;
69
70static void emitEpiloguePACSymOffsetIntoReg(const TargetInstrInfo &TII,
71 MachineBasicBlock &MBB,
72 MachineBasicBlock::iterator I,
73 DebugLoc DL, MCSymbol *PACSym,
74 Register Reg) {
75 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AArch64::ADRP), DestReg: Reg)
76 .addSym(Sym: PACSym, TargetFlags: AArch64II::MO_PAGE)
77 .setMIFlag(MachineInstr::FrameDestroy);
78 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AArch64::ADDXri), DestReg: Reg)
79 .addReg(RegNo: Reg)
80 .addSym(Sym: PACSym, TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC)
81 .addImm(Val: 0)
82 .setMIFlag(MachineInstr::FrameDestroy);
83}
84
85// Wrap a given PAC instruction in CFI that describes it.
86//
87// Depending on the type of CFI required, we may need to emit the directive
88// either before or after the instruction, so that unwinders can correctly
89// interpret the location of the signing instruction.
90//
91// As a general rule, CFI opcodes describe the actions needed to recover the
92// register state leading up to a not-yet-retired instruction, with one
93// exception: .cfi_negate_ra_state_with_pc always comes before the paci[ab]sppc,
94// since the unwinder uses the location of the CFI itself to derive the address
95// of the signing instruction [1].
96// 1: https://github.com/llvm/llvm-project/pull/137795#issuecomment-2838779129
97template <typename BuildPACMIFn>
98static void decoratePACWithCFI(MachineBasicBlock &MBB,
99 MachineBasicBlock::iterator MBBI, bool EmitCFI,
100 BuildPACMIFn BuildPACMI) {
101 if (!EmitCFI) {
102 BuildPACMI();
103 return;
104 }
105
106 auto &MF = *MBB.getParent();
107 auto &MFnI = *MF.getInfo<AArch64FunctionInfo>();
108 CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
109 const Triple &TT = MF.getFunction().getParent()->getTargetTriple();
110 SetRAStateMode Mode = MF.getSubtarget<AArch64Subtarget>()
111 .getCLOpts()
112 .cfi_llvm_set_ra_sign_state;
113
114 if (MFnI.branchProtectionPAuthLR()) {
115 switch (Mode) {
116 case SetRAStateMode::Never:
117 CFIBuilder.buildNegateRAStateWithPC();
118 BuildPACMI();
119 break;
120 case SetRAStateMode::PAuthLR:
121 case SetRAStateMode::Always: {
122 BuildPACMI();
123 MCSymbol *PACSym = MFnI.getSigningInstrLabel();
124 assert(PACSym && "No PAC instruction to refer to");
125 CFIBuilder.buildSetRAState(State: 2, PACSym);
126 break;
127 }
128 }
129 } else {
130 switch (Mode) {
131 case SetRAStateMode::Never:
132 case SetRAStateMode::PAuthLR:
133 BuildPACMI();
134 if (!TT.isOSBinFormatMachO()) {
135 CFIBuilder.buildNegateRAState();
136 }
137 break;
138 case SetRAStateMode::Always:
139 BuildPACMI();
140 CFIBuilder.buildSetRAState(State: 1, PACSym: nullptr);
141 break;
142 }
143 }
144}
145
146static void emitAUTCFI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
147 bool EmitCFI) {
148 if (!EmitCFI)
149 return;
150
151 auto &MF = *MBB.getParent();
152 auto &MFnI = *MF.getInfo<AArch64FunctionInfo>();
153 CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameDestroy);
154 const Triple &TT = MF.getFunction().getParent()->getTargetTriple();
155 SetRAStateMode Mode = MF.getSubtarget<AArch64Subtarget>()
156 .getCLOpts()
157 .cfi_llvm_set_ra_sign_state;
158
159 if (MFnI.branchProtectionPAuthLR()) {
160 switch (Mode) {
161 case SetRAStateMode::Never:
162 // DW_CFA_AARCH64_negate_ra_state_with_pc is semantically broken for
163 // functions where shrinkwrapping places signing/authenticating pairs on
164 // distinct CFG paths.
165 //
166 // DWARF CFI is evaluated linearly over the byte stream, not along control
167 // flow edges. The toggle semantics of this directive therefore cannot
168 // faithfully represent the signed/unsigned RA state for all possible CFG
169 // paths. The added complexity versus DW_CFA_AARCH64_negate_ra_state is
170 // that an unwinder must also reconstruct the PC of the PACI[AB]SPPC in
171 // order to verify the signed LR, and that address is derived from the
172 // location of this directive in the linear CFI stream.
173 //
174 // The correct fix is to use DW_CFA_AARCH64_set_ra_state_with_pc, which
175 // sets the RA state and signing address absolutely rather than toggling
176 // them. An unwinder that supports this directive can reconstruct the
177 // correct state on any CFG path, regardless of how many
178 // signing/authenticating pairs exist in the function. However, not all
179 // unwinders support this directive, so we cannot rely on it exclusively.
180 //
181 // For unwinders that only support DW_CFA_AARCH64_negate_ra_state_with_pc,
182 // libunwind exploits a loophole: it records the address at the
183 // DW_CFA_AARCH64_negate_ra_state_with_pc site to authenticate the LR, but
184 // does not care that the CFI state remains "signed with pc" after
185 // authentication has occurred. This means we can safely omit the
186 // FrameDestroy emission of this directive, treating it solely as a marker
187 // for the signing site, as long as each function has at most one such
188 // signing location. That invariant holds today because shrinkwrapping
189 // does not yet hoist or sink PAuth_LR frame code across CFG join/split
190 // points; once it does, we must avoid those transformations on platforms
191 // that have this limitation.
192 //
193 // https://github.com/ARM-software/abi-aa/issues/327
194 // https://github.com/ARM-software/abi-aa/pull/346
195 break;
196 case SetRAStateMode::PAuthLR:
197 case SetRAStateMode::Always:
198 CFIBuilder.buildSetRAState(State: 0, PACSym: nullptr);
199 break;
200 }
201 } else if (!TT.isOSBinFormatMachO()) {
202 switch (Mode) {
203 case SetRAStateMode::Never:
204 case SetRAStateMode::PAuthLR:
205 CFIBuilder.buildNegateRAState();
206 break;
207 case SetRAStateMode::Always:
208 CFIBuilder.buildSetRAState(State: 0, PACSym: nullptr);
209 break;
210 }
211 }
212}
213
214static inline void emitMOVWithFrameDestroy(MachineBasicBlock &MBB,
215 MachineBasicBlock::iterator &MBBI,
216 DebugLoc DL,
217 const AArch64InstrInfo *TII,
218 Register Dst, Register Src) {
219 assert(&MBB == MBBI->getParent());
220 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ORRXrs), DestReg: Dst)
221 .addReg(RegNo: AArch64::XZR)
222 .addReg(RegNo: Src)
223 .addImm(Val: 0)
224 .setMIFlag(MachineInstr::FrameDestroy);
225}
226
227void AArch64PointerAuthImpl::signLR(MachineFunction &MF,
228 MachineBasicBlock::iterator MBBI) const {
229 auto &MFnI = *MF.getInfo<AArch64FunctionInfo>();
230 bool UseBKey = MFnI.shouldSignWithBKey();
231 bool EmitCFI = MFnI.needsDwarfUnwindInfo(MF);
232 bool NeedsWinCFI = MF.hasWinCFI();
233
234 MachineBasicBlock &MBB = *MBBI->getParent();
235
236 // Debug location must be unknown, see AArch64FrameLowering::emitPrologue.
237 DebugLoc DL;
238
239 if (UseBKey && !MF.getTarget().getTargetTriple().isOSBinFormatMachO()) {
240 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::EMITBKEY))
241 .setMIFlag(MachineInstr::FrameSetup);
242 }
243
244 // PAuthLR authentication instructions need to know the value of PC at the
245 // point of signing (PACI*).
246 if (MFnI.branchProtectionPAuthLR()) {
247 MCSymbol *PACSym = MF.getContext().createTempSymbol();
248 MFnI.setSigningInstrLabel(PACSym);
249 }
250
251 // No SEH opcode for this one; it doesn't materialize into an
252 // instruction on Windows.
253 if (MFnI.branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
254 decoratePACWithCFI(MBB, MBBI, EmitCFI, BuildPACMI: [&]() {
255 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
256 MCID: TII->get(Opcode: UseBKey ? AArch64::PACIBSPPC : AArch64::PACIASPPC))
257 .setMIFlag(MachineInstr::FrameSetup)
258 ->setPreInstrSymbol(MF, Symbol: MFnI.getSigningInstrLabel());
259 });
260 } else {
261 if (MFnI.branchProtectionPAuthLR()) {
262 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PACM))
263 .setMIFlag(MachineInstr::FrameSetup);
264 }
265 decoratePACWithCFI(MBB, MBBI, EmitCFI, BuildPACMI: [&]() {
266 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
267 MCID: TII->get(Opcode: UseBKey ? AArch64::PACIBSP : AArch64::PACIASP))
268 .setMIFlag(MachineInstr::FrameSetup)
269 ->setPreInstrSymbol(MF, Symbol: MFnI.getSigningInstrLabel());
270 });
271 }
272
273 if (!EmitCFI && NeedsWinCFI) {
274 assert(UseBKey &&
275 "Windows SEH PAC unwind info only supports B-key signing");
276 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_PACSignLR))
277 .setMIFlag(MachineInstr::FrameSetup);
278 }
279}
280
281void AArch64PointerAuthImpl::authenticateLR(
282 MachineFunction &MF, MachineBasicBlock::iterator MBBI) const {
283 const AArch64FunctionInfo *MFnI = MF.getInfo<AArch64FunctionInfo>();
284 bool UseBKey = MFnI->shouldSignWithBKey();
285 bool EmitAsyncCFI = MFnI->needsAsyncDwarfUnwindInfo(MF);
286 bool NeedsWinCFI = MF.hasWinCFI();
287
288 MachineBasicBlock &MBB = *MBBI->getParent();
289 DebugLoc DL = MBBI->getDebugLoc();
290 // MBBI points to a PAUTH_EPILOGUE instruction to be replaced and
291 // TI points to a terminator instruction that may or may not be combined.
292 // Note that inserting new instructions "before MBBI" and "before TI" is
293 // not the same because if ShadowCallStack is enabled, its instructions
294 // are placed between MBBI and TI.
295 MachineBasicBlock::iterator TI = MBB.getFirstInstrTerminator();
296
297 MCSymbol *PACSym = MFnI->getSigningInstrLabel();
298 auto &AFL = *static_cast<const AArch64FrameLowering *>(
299 MF.getSubtarget().getFrameLowering());
300 int64_t ArgumentStackToRestore = AFL.getArgumentStackToRestore(MF, MBB);
301
302 // The AUTIASP instruction assembles to a hint instruction before v8.3a so
303 // this instruction can safely be used for any v8a architecture.
304 // From v8.3a onwards there are optimised authenticate LR and return
305 // instructions, namely RETA{A,B}, that can be used instead. In this case
306 // the DW_CFA_AARCH64_negate_ra_state can't be emitted. Additionally,
307 // RET{A,B} requires the SP to match its incoming value on entry to the
308 // function.
309 //
310 // If the PAC-RET hardening based on load from the return address is
311 // enabled, fallback to the use of AUTIASP/AUTIBSP and RET.
312 bool TerminatorIsCombinable = std::next(x: MBBI) == TI && TI != MBB.end() &&
313 TI->getOpcode() == AArch64::RET &&
314 ArgumentStackToRestore == 0 &&
315 !MFnI->shouldHardenSignReturnAddress();
316
317 if (Subtarget->hasPAuth() && TerminatorIsCombinable && !NeedsWinCFI &&
318 !MF.getFunction().hasFnAttribute(Kind: Attribute::ShadowCallStack)) {
319 if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
320 assert(PACSym && "No PAC instruction to refer to");
321 BuildMI(BB&: MBB, I: TI, MIMD: DL,
322 MCID: TII->get(Opcode: UseBKey ? AArch64::RETABSPPCi : AArch64::RETAASPPCi))
323 .addSym(Sym: PACSym)
324 .copyImplicitOps(OtherMI: *MBBI)
325 .setMIFlag(MachineInstr::FrameDestroy);
326 } else {
327 if (MFnI->branchProtectionPAuthLR()) {
328 emitEpiloguePACSymOffsetIntoReg(TII: *TII, MBB, I: MBBI, DL, PACSym,
329 Reg: AArch64::X16);
330 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PACM))
331 .setMIFlag(MachineInstr::FrameDestroy);
332 }
333 BuildMI(BB&: MBB, I: TI, MIMD: DL, MCID: TII->get(Opcode: UseBKey ? AArch64::RETAB : AArch64::RETAA))
334 .copyImplicitOps(OtherMI: *MBBI)
335 .setMIFlag(MachineInstr::FrameDestroy);
336 }
337 MBB.erase(I: TI);
338 return;
339 }
340
341 // If PAUTH_EPILOGUE is at insertion point with a net zero offset on SP, we
342 // can use an AUT form with a hardcoded SP discriminator.
343 if (ArgumentStackToRestore == 0) {
344 if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
345 assert(PACSym && "No PAC instruction to refer to");
346 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
347 MCID: TII->get(Opcode: UseBKey ? AArch64::AUTIBSPPCi : AArch64::AUTIASPPCi))
348 .addSym(Sym: PACSym)
349 .setMIFlag(MachineInstr::FrameDestroy);
350 emitAUTCFI(MBB, MBBI, EmitCFI: EmitAsyncCFI);
351 } else {
352 if (MFnI->branchProtectionPAuthLR()) {
353 emitEpiloguePACSymOffsetIntoReg(TII: *TII, MBB, I: MBBI, DL, PACSym,
354 Reg: AArch64::X16);
355
356 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PACM))
357 .setMIFlag(MachineInstr::FrameDestroy);
358 }
359 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
360 MCID: TII->get(Opcode: UseBKey ? AArch64::AUTIBSP : AArch64::AUTIASP))
361 .setMIFlag(MachineInstr::FrameDestroy);
362 emitAUTCFI(MBB, MBBI, EmitCFI: EmitAsyncCFI);
363 }
364
365 if (NeedsWinCFI) {
366 assert(UseBKey &&
367 "Windows SEH PAC unwind info only supports B-key signing");
368 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_PACSignLR))
369 .setMIFlag(MachineInstr::FrameDestroy);
370 }
371
372 return;
373 }
374
375 // When ArgumentStackToRestore > 0, this function received more argument
376 // space than the tail callee pops. The epilogue contains an SP adjustment
377 // (e.g. "add sp, sp, #N") to discard the leftover argument space.
378 //
379 // When ArgumentStackToRestore < 0, the tail callee pops more argument space
380 // than this function received, so after the frame teardown, SP is below the
381 // entry SP used as the signing modifier.
382 //
383 // We cannot simply bump SP first and then use AUTI[AB]SP with the bumped
384 // value, because the live arguments would fall below SP and potentially
385 // outside the red-zone.
386 //
387 // At this point there is an offset to the incoming SP, and we can't use the
388 // aut variants that hard-code SP. Reconstruct entry SP in x16 and
389 // authenticate using AUTI[AB]1716 (x17=LR, x16=entry_SP).
390 emitFrameOffset(MBB, MBBI, DL, DestReg: AArch64::X16, SrcReg: AArch64::SP,
391 Offset: StackOffset::getFixed(Fixed: -ArgumentStackToRestore), TII,
392 MachineInstr::FrameDestroy);
393
394 if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
395 emitMOVWithFrameDestroy(MBB, MBBI, DL, TII, Dst: AArch64::X17, Src: AArch64::LR);
396
397 assert(PACSym && "No PAC instruction to refer to");
398 emitEpiloguePACSymOffsetIntoReg(TII: *TII, MBB, I: MBBI, DL, PACSym, Reg: AArch64::X15);
399
400 unsigned AutOpc = UseBKey ? AArch64::AUTIB171615 : AArch64::AUTIA171615;
401 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AutOpc))
402 .setMIFlag(MachineInstr::FrameDestroy);
403 emitAUTCFI(MBB, MBBI, EmitCFI: EmitAsyncCFI);
404
405 emitMOVWithFrameDestroy(MBB, MBBI, DL, TII, Dst: AArch64::LR, Src: AArch64::X17);
406 } else if (MFnI->branchProtectionPAuthLR()) {
407 emitMOVWithFrameDestroy(MBB, MBBI, DL, TII, Dst: AArch64::X17, Src: AArch64::LR);
408
409 assert(PACSym && "No PAC instruction to refer to");
410 emitEpiloguePACSymOffsetIntoReg(TII: *TII, MBB, I: MBBI, DL, PACSym, Reg: AArch64::X15);
411
412 // The PACM hint-space instruction modifies the following AUTI[AB]1716
413 // to optionally take x15 as an extra operand depending on the
414 // presence of +pauth-lr at runtime. On machines without +pauth-lr, it
415 // behaves as a nop, and the address of the PACI[AB]SP in x15 is
416 // ignored.
417 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PACM))
418 .setMIFlag(MachineInstr::FrameDestroy);
419
420 unsigned AutOpc = UseBKey ? AArch64::AUTIB1716 : AArch64::AUTIA1716;
421 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AutOpc))
422 .setMIFlag(MachineInstr::FrameDestroy);
423 emitAUTCFI(MBB, MBBI, EmitCFI: EmitAsyncCFI);
424
425 emitMOVWithFrameDestroy(MBB, MBBI, DL, TII, Dst: AArch64::LR, Src: AArch64::X17);
426 } else if (Subtarget->hasPAuth()) {
427 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: UseBKey ? AArch64::AUTIB : AArch64::AUTIA),
428 DestReg: AArch64::LR)
429 .addUse(RegNo: AArch64::LR)
430 .addUse(RegNo: AArch64::X16)
431 .setMIFlag(MachineInstr::FrameDestroy);
432 emitAUTCFI(MBB, MBBI, EmitCFI: EmitAsyncCFI);
433 } else {
434 emitMOVWithFrameDestroy(MBB, MBBI, DL, TII, Dst: AArch64::X17, Src: AArch64::LR);
435
436 unsigned AutOpc = UseBKey ? AArch64::AUTIB1716 : AArch64::AUTIA1716;
437 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AutOpc))
438 .setMIFlag(MachineInstr::FrameDestroy);
439 emitAUTCFI(MBB, MBBI, EmitCFI: EmitAsyncCFI);
440
441 emitMOVWithFrameDestroy(MBB, MBBI, DL, TII, Dst: AArch64::LR, Src: AArch64::X17);
442 }
443
444 if (NeedsWinCFI) {
445 assert(UseBKey &&
446 "Windows SEH PAC unwind info only supports B-key signing");
447 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_PACSignLR))
448 .setMIFlag(MachineInstr::FrameDestroy);
449 }
450}
451
452unsigned llvm::AArch64PAuth::getCheckerSizeInBytes(AuthCheckMethod Method) {
453 switch (Method) {
454 case AuthCheckMethod::None:
455 return 0;
456 case AuthCheckMethod::DummyLoad:
457 return 4;
458 case AuthCheckMethod::HighBitsNoTBI:
459 return 12;
460 case AuthCheckMethod::XPACHint:
461 case AuthCheckMethod::XPAC:
462 return 20;
463 }
464 llvm_unreachable("Unknown AuthCheckMethod enum");
465}
466
467bool AArch64PointerAuthImpl::run(MachineFunction &MF) {
468 Subtarget = &MF.getSubtarget<AArch64Subtarget>();
469 TII = Subtarget->getInstrInfo();
470
471 SmallVector<MachineBasicBlock::instr_iterator> PAuthPseudoInstrs;
472
473 bool Modified = false;
474
475 for (auto &MBB : MF) {
476 for (auto &MI : MBB) {
477 switch (MI.getOpcode()) {
478 default:
479 break;
480 case AArch64::PAUTH_PROLOGUE:
481 case AArch64::PAUTH_EPILOGUE:
482 PAuthPseudoInstrs.push_back(Elt: MI.getIterator());
483 break;
484 }
485 }
486 }
487
488 for (auto It : PAuthPseudoInstrs) {
489 switch (It->getOpcode()) {
490 case AArch64::PAUTH_PROLOGUE:
491 signLR(MF, MBBI: It);
492 break;
493 case AArch64::PAUTH_EPILOGUE:
494 authenticateLR(MF, MBBI: It);
495 break;
496 default:
497 llvm_unreachable("Unhandled opcode");
498 }
499 It->eraseFromParent();
500 Modified = true;
501 }
502
503 Modified |= emitSignReturnAddressHardening(MF);
504
505 return Modified;
506}
507
508bool AArch64PointerAuthImpl::emitSignReturnAddressHardening(
509 MachineFunction &MF) {
510 const auto *FI = MF.getInfo<AArch64FunctionInfo>();
511 assert(FI && "FI can't be null");
512 if (!FI->shouldSignReturnAddress(MF) || !FI->shouldHardenSignReturnAddress())
513 return false;
514 assert(Subtarget && "Subtarget must be initialized");
515
516 RegScavenger RS;
517 bool Modified = false;
518 for (MachineBasicBlock &MBB : MF) {
519 MachineBasicBlock::iterator RetInstIter = MBB.getFirstTerminator();
520
521 if (RetInstIter == MBB.end() || RetInstIter->getOpcode() != AArch64::RET)
522 continue;
523
524 assert(RetInstIter->getOperand(0).getReg() == AArch64::LR &&
525 "Return instruction must be returning via LR");
526
527 MachineBasicBlock::iterator InsertionPoint = RetInstIter;
528 // In the case of Windows SEH, the hardening sequence does not immediately
529 // precede the return instruction. Instead, it precedes the SEH_EpilogEnd
530 // pseudo-instruction, which itself is expected to be the predecessor of
531 // the return. Plus, each instruction in the sequence needs one SEH_Nop.
532 const bool NeedsWinCFI = MF.hasWinCFI();
533 if (NeedsWinCFI) {
534 --InsertionPoint;
535 assert(InsertionPoint->getOpcode() == AArch64::SEH_EpilogEnd);
536 }
537 DebugLoc DL = InsertionPoint->getDebugLoc();
538 const auto EmitSEHNopIfRequired = [&]() {
539 if (NeedsWinCFI)
540 BuildMI(BB&: MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_Nop))
541 .setMIFlag(MachineInstr::FrameDestroy);
542 };
543
544 RS.enterBasicBlockEnd(MBB);
545 Register XReg = RS.scavengeRegisterBackwards(
546 RC: AArch64::GPR64RegClass, To: InsertionPoint,
547 /*RestoreAfter=*/false, /*SPAdj=*/0, /*AllowSpill=*/false);
548 if (XReg == AArch64::NoRegister) {
549 // Couldn't find a free register to use for the hardening. Skip.
550 MF.getContext().reportWarning(
551 L: SMLoc(), Msg: "harden-pac-ret failed for function " + MF.getName());
552 continue;
553 }
554
555 // Register copies are done using ORRXrs directly instead of using the
556 // pseudo-instruction COPY because this function can be called after
557 // pseudo-instruction expansion takes place, for example via the machine
558 // outliner pass.
559 emitMOVWithFrameDestroy(MBB, MBBI&: InsertionPoint, DL, TII, Dst: XReg, Src: AArch64::LR);
560 EmitSEHNopIfRequired();
561
562 // The XPACI instruction is only available with FEAT_PAUTH. So if the
563 // subtarget does not have it, the alternative XPACLRI instruction must be
564 // used instead. The latter is in hint space, therefore can be used even
565 // if FEAT_PAUTH is absent.
566 if (Subtarget->hasPAuth()) {
567 BuildMI(BB&: MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: AArch64::XPACI), DestReg: XReg)
568 .addUse(RegNo: XReg)
569 .setMIFlag(MachineInstr::FrameDestroy);
570 EmitSEHNopIfRequired();
571 Register WReg =
572 Subtarget->getRegisterInfo()->getSubReg(Reg: XReg, Idx: AArch64::sub_32);
573 BuildMI(BB&: MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: AArch64::LDRWui), DestReg: WReg)
574 .addUse(RegNo: XReg)
575 .addImm(Val: 0)
576 .addMemOperand(MMO: MF.getMachineMemOperand(
577 PtrInfo: MachinePointerInfo(),
578 F: MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile, Size: 4,
579 BaseAlignment: Align(4)))
580 .setMIFlag(MachineInstr::FrameDestroy);
581 EmitSEHNopIfRequired();
582 } else {
583 // Emit a CFI directive to tell unwinders that the return address is now
584 // saved in XReg.
585 CFIInstBuilder(MBB, InsertionPoint, MachineInstr::FrameDestroy)
586 .buildRegister(Reg1: AArch64::LR, Reg2: XReg);
587 BuildMI(BB&: MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: AArch64::XPACLRI))
588 .setMIFlag(MachineInstr::FrameDestroy);
589 EmitSEHNopIfRequired();
590 BuildMI(BB&: MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: AArch64::LDRWui), DestReg: AArch64::W30)
591 .addUse(RegNo: AArch64::LR)
592 .addImm(Val: 0)
593 .addMemOperand(MMO: MF.getMachineMemOperand(
594 PtrInfo: MachinePointerInfo(),
595 F: MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile, Size: 4,
596 BaseAlignment: Align(4)))
597 .setMIFlag(MachineInstr::FrameDestroy);
598 EmitSEHNopIfRequired();
599 BuildMI(BB&: MBB, I: RetInstIter, MIMD: DL, MCID: TII->get(Opcode: AArch64::RET))
600 .addUse(RegNo: XReg)
601 .copyImplicitOps(OtherMI: *RetInstIter);
602 MBB.erase(I: RetInstIter);
603 }
604 Modified = true;
605 }
606
607 return Modified;
608}
609
610bool AArch64PointerAuthLegacy::runOnMachineFunction(MachineFunction &MF) {
611 return AArch64PointerAuthImpl().run(MF);
612}
613
614PreservedAnalyses
615AArch64PointerAuthPass::run(MachineFunction &MF,
616 MachineFunctionAnalysisManager &MFAM) {
617 const bool Changed = AArch64PointerAuthImpl().run(MF);
618 if (!Changed)
619 return PreservedAnalyses::all();
620 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
621 PA.preserveSet<CFGAnalyses>();
622 return PA;
623}
624