1//===-- AArch64PointerAuth.cpp -- Harden code using PAuth ------------------==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AArch64PointerAuth.h"
10
11#include "AArch64.h"
12#include "AArch64FrameLowering.h"
13#include "AArch64InstrInfo.h"
14#include "AArch64MachineFunctionInfo.h"
15#include "AArch64RegisterInfo.h"
16#include "AArch64Subtarget.h"
17#include "llvm/CodeGen/CFIInstBuilder.h"
18#include "llvm/CodeGen/MachineBasicBlock.h"
19#include "llvm/CodeGen/MachineInstrBuilder.h"
20#include "llvm/CodeGen/MachineModuleInfo.h"
21#include "llvm/CodeGen/RegisterScavenging.h"
22#include "llvm/IR/Module.h"
23
24using namespace llvm;
25using namespace llvm::AArch64PAuth;
26
27#define AARCH64_POINTER_AUTH_NAME "AArch64 Pointer Authentication"
28
29namespace {
30
31class AArch64PointerAuthImpl {
32public:
33 bool run(MachineFunction &MF);
34
35private:
36 const AArch64Subtarget *Subtarget = nullptr;
37 const AArch64InstrInfo *TII = nullptr;
38
39 void signLR(MachineFunction &MF, MachineBasicBlock::iterator MBBI) const;
40
41 void authenticateLR(MachineFunction &MF,
42 MachineBasicBlock::iterator MBBI) const;
43
44 bool emitSignReturnAddressHardening(MachineFunction &MF);
45};
46
47class AArch64PointerAuthLegacy : public MachineFunctionPass {
48public:
49 static char ID;
50
51 AArch64PointerAuthLegacy() : MachineFunctionPass(ID) {}
52
53 bool runOnMachineFunction(MachineFunction &MF) override;
54
55 StringRef getPassName() const override { return AARCH64_POINTER_AUTH_NAME; }
56};
57
58} // end anonymous namespace
59
60INITIALIZE_PASS(AArch64PointerAuthLegacy, "aarch64-ptrauth",
61 AARCH64_POINTER_AUTH_NAME, false, false)
62
63FunctionPass *llvm::createAArch64PointerAuthPass() {
64 return new AArch64PointerAuthLegacy();
65}
66
67char AArch64PointerAuthLegacy::ID = 0;
68
69static void emitEpiloguePACSymOffsetIntoReg(const TargetInstrInfo &TII,
70 MachineBasicBlock &MBB,
71 MachineBasicBlock::iterator I,
72 DebugLoc DL, MCSymbol *PACSym,
73 Register Reg) {
74 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AArch64::ADRP), DestReg: Reg)
75 .addSym(Sym: PACSym, TargetFlags: AArch64II::MO_PAGE)
76 .setMIFlag(MachineInstr::FrameDestroy);
77 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AArch64::ADDXri), DestReg: Reg)
78 .addReg(RegNo: Reg)
79 .addSym(Sym: PACSym, TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC)
80 .addImm(Val: 0)
81 .setMIFlag(MachineInstr::FrameDestroy);
82}
83
84// Wrap a given PAC instruction in CFI that describes it.
85//
86// Depending on the type of CFI required, we may need to emit the directive
87// either before or after the instruction, so that unwinders can correctly
88// interpret the location of the signing instruction.
89//
90// As a general rule, CFI opcodes describe the actions needed to recover the
91// register state leading up to a not-yet-retired instruction, with one
92// exception: .cfi_negate_ra_state_with_pc always comes before the paci[ab]sppc,
93// since the unwinder uses the location of the CFI itself to derive the address
94// of the signing instruction [1].
95// 1: https://github.com/llvm/llvm-project/pull/137795#issuecomment-2838779129
96template <typename BuildPACMIFn>
97static void decoratePACWithCFI(MachineBasicBlock &MBB,
98 MachineBasicBlock::iterator MBBI, bool EmitCFI,
99 BuildPACMIFn BuildPACMI) {
100 if (!EmitCFI) {
101 BuildPACMI();
102 return;
103 }
104
105 auto &MF = *MBB.getParent();
106 auto &MFnI = *MF.getInfo<AArch64FunctionInfo>();
107 CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
108 const Triple &TT = MF.getFunction().getParent()->getTargetTriple();
109 SetRAStateMode Mode = MF.getSubtarget<AArch64Subtarget>()
110 .getCLOpts()
111 .cfi_llvm_set_ra_sign_state;
112
113 if (MFnI.branchProtectionPAuthLR()) {
114 switch (Mode) {
115 case SetRAStateMode::Never:
116 CFIBuilder.buildNegateRAStateWithPC();
117 BuildPACMI();
118 break;
119 case SetRAStateMode::PAuthLR:
120 case SetRAStateMode::Always: {
121 BuildPACMI();
122 MCSymbol *PACSym = MFnI.getSigningInstrLabel();
123 assert(PACSym && "No PAC instruction to refer to");
124 CFIBuilder.buildSetRAState(State: 2, PACSym);
125 break;
126 }
127 }
128 } else {
129 switch (Mode) {
130 case SetRAStateMode::Never:
131 case SetRAStateMode::PAuthLR:
132 BuildPACMI();
133 if (!TT.isOSBinFormatMachO()) {
134 CFIBuilder.buildNegateRAState();
135 }
136 break;
137 case SetRAStateMode::Always:
138 BuildPACMI();
139 CFIBuilder.buildSetRAState(State: 1, PACSym: nullptr);
140 break;
141 }
142 }
143}
144
145static void emitAUTCFI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
146 bool EmitCFI) {
147 if (!EmitCFI)
148 return;
149
150 auto &MF = *MBB.getParent();
151 auto &MFnI = *MF.getInfo<AArch64FunctionInfo>();
152 CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameDestroy);
153 const Triple &TT = MF.getFunction().getParent()->getTargetTriple();
154 SetRAStateMode Mode = MF.getSubtarget<AArch64Subtarget>()
155 .getCLOpts()
156 .cfi_llvm_set_ra_sign_state;
157
158 if (MFnI.branchProtectionPAuthLR()) {
159 switch (Mode) {
160 case SetRAStateMode::Never:
161 // DW_CFA_AARCH64_negate_ra_state_with_pc is semantically broken for
162 // functions where shrinkwrapping places signing/authenticating pairs on
163 // distinct CFG paths.
164 //
165 // DWARF CFI is evaluated linearly over the byte stream, not along control
166 // flow edges. The toggle semantics of this directive therefore cannot
167 // faithfully represent the signed/unsigned RA state for all possible CFG
168 // paths. The added complexity versus DW_CFA_AARCH64_negate_ra_state is
169 // that an unwinder must also reconstruct the PC of the PACI[AB]SPPC in
170 // order to verify the signed LR, and that address is derived from the
171 // location of this directive in the linear CFI stream.
172 //
173 // The correct fix is to use DW_CFA_AARCH64_set_ra_state_with_pc, which
174 // sets the RA state and signing address absolutely rather than toggling
175 // them. An unwinder that supports this directive can reconstruct the
176 // correct state on any CFG path, regardless of how many
177 // signing/authenticating pairs exist in the function. However, not all
178 // unwinders support this directive, so we cannot rely on it exclusively.
179 //
180 // For unwinders that only support DW_CFA_AARCH64_negate_ra_state_with_pc,
181 // libunwind exploits a loophole: it records the address at the
182 // DW_CFA_AARCH64_negate_ra_state_with_pc site to authenticate the LR, but
183 // does not care that the CFI state remains "signed with pc" after
184 // authentication has occurred. This means we can safely omit the
185 // FrameDestroy emission of this directive, treating it solely as a marker
186 // for the signing site, as long as each function has at most one such
187 // signing location. That invariant holds today because shrinkwrapping
188 // does not yet hoist or sink PAuth_LR frame code across CFG join/split
189 // points; once it does, we must avoid those transformations on platforms
190 // that have this limitation.
191 //
192 // https://github.com/ARM-software/abi-aa/issues/327
193 // https://github.com/ARM-software/abi-aa/pull/346
194 break;
195 case SetRAStateMode::PAuthLR:
196 case SetRAStateMode::Always:
197 CFIBuilder.buildSetRAState(State: 0, PACSym: nullptr);
198 break;
199 }
200 } else if (!TT.isOSBinFormatMachO()) {
201 switch (Mode) {
202 case SetRAStateMode::Never:
203 case SetRAStateMode::PAuthLR:
204 CFIBuilder.buildNegateRAState();
205 break;
206 case SetRAStateMode::Always:
207 CFIBuilder.buildSetRAState(State: 0, PACSym: nullptr);
208 break;
209 }
210 }
211}
212
213static inline void emitMOVWithFrameDestroy(MachineBasicBlock &MBB,
214 MachineBasicBlock::iterator &MBBI,
215 DebugLoc DL,
216 const AArch64InstrInfo *TII,
217 Register Dst, Register Src) {
218 assert(&MBB == MBBI->getParent());
219 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ORRXrs), DestReg: Dst)
220 .addReg(RegNo: AArch64::XZR)
221 .addReg(RegNo: Src)
222 .addImm(Val: 0)
223 .setMIFlag(MachineInstr::FrameDestroy);
224}
225
226void AArch64PointerAuthImpl::signLR(MachineFunction &MF,
227 MachineBasicBlock::iterator MBBI) const {
228 auto &MFnI = *MF.getInfo<AArch64FunctionInfo>();
229 bool UseBKey = MFnI.shouldSignWithBKey();
230 bool EmitCFI = MFnI.needsDwarfUnwindInfo(MF);
231 bool NeedsWinCFI = MF.hasWinCFI();
232
233 MachineBasicBlock &MBB = *MBBI->getParent();
234
235 // Debug location must be unknown, see AArch64FrameLowering::emitPrologue.
236 DebugLoc DL;
237
238 if (UseBKey && !MF.getTarget().getTargetTriple().isOSBinFormatMachO()) {
239 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::EMITBKEY))
240 .setMIFlag(MachineInstr::FrameSetup);
241 }
242
243 // PAuthLR authentication instructions need to know the value of PC at the
244 // point of signing (PACI*).
245 if (MFnI.branchProtectionPAuthLR()) {
246 MCSymbol *PACSym = MF.getContext().createTempSymbol();
247 MFnI.setSigningInstrLabel(PACSym);
248 }
249
250 // No SEH opcode for this one; it doesn't materialize into an
251 // instruction on Windows.
252 if (MFnI.branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
253 if (!MF.getTarget().getTargetTriple().isOSBinFormatMachO())
254 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::EMITNONHINTED))
255 .setMIFlag(MachineInstr::FrameSetup);
256
257 decoratePACWithCFI(MBB, MBBI, EmitCFI, BuildPACMI: [&]() {
258 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
259 MCID: TII->get(Opcode: UseBKey ? AArch64::PACIBSPPC : AArch64::PACIASPPC))
260 .setMIFlag(MachineInstr::FrameSetup)
261 ->setPreInstrSymbol(MF, Symbol: MFnI.getSigningInstrLabel());
262 });
263 } else {
264 if (MFnI.branchProtectionPAuthLR()) {
265 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PACM))
266 .setMIFlag(MachineInstr::FrameSetup);
267 }
268 decoratePACWithCFI(MBB, MBBI, EmitCFI, BuildPACMI: [&]() {
269 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
270 MCID: TII->get(Opcode: UseBKey ? AArch64::PACIBSP : AArch64::PACIASP))
271 .setMIFlag(MachineInstr::FrameSetup)
272 ->setPreInstrSymbol(MF, Symbol: MFnI.getSigningInstrLabel());
273 });
274 }
275
276 if (!EmitCFI && NeedsWinCFI) {
277 assert(UseBKey &&
278 "Windows SEH PAC unwind info only supports B-key signing");
279 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_PACSignLR))
280 .setMIFlag(MachineInstr::FrameSetup);
281 }
282}
283
284void AArch64PointerAuthImpl::authenticateLR(
285 MachineFunction &MF, MachineBasicBlock::iterator MBBI) const {
286 const AArch64FunctionInfo *MFnI = MF.getInfo<AArch64FunctionInfo>();
287 bool UseBKey = MFnI->shouldSignWithBKey();
288 bool EmitAsyncCFI = MFnI->needsAsyncDwarfUnwindInfo(MF);
289 bool NeedsWinCFI = MF.hasWinCFI();
290
291 MachineBasicBlock &MBB = *MBBI->getParent();
292 DebugLoc DL = MBBI->getDebugLoc();
293 // MBBI points to a PAUTH_EPILOGUE instruction to be replaced and
294 // TI points to a terminator instruction that may or may not be combined.
295 // Note that inserting new instructions "before MBBI" and "before TI" is
296 // not the same because if ShadowCallStack is enabled, its instructions
297 // are placed between MBBI and TI.
298 MachineBasicBlock::iterator TI = MBB.getFirstInstrTerminator();
299
300 MCSymbol *PACSym = MFnI->getSigningInstrLabel();
301 auto &AFL = *static_cast<const AArch64FrameLowering *>(
302 MF.getSubtarget().getFrameLowering());
303 int64_t ArgumentStackToRestore = AFL.getArgumentStackToRestore(MF, MBB);
304
305 // The signing SP is already in X16 if we are before the epilogue proper.
306 bool EntrySPInX16 = MBBI->getOpcode() == AArch64::PAUTH_EPILOGUE_ENTRY_SP;
307 assert((!EntrySPInX16 || ArgumentStackToRestore != 0) &&
308 "Early authentication is only for epilogues that change SP");
309
310 // The AUTIASP instruction assembles to a hint instruction before v8.3a so
311 // this instruction can safely be used for any v8a architecture.
312 // From v8.3a onwards there are optimised authenticate LR and return
313 // instructions, namely RETA{A,B}, that can be used instead. In this case
314 // the DW_CFA_AARCH64_negate_ra_state can't be emitted. Additionally,
315 // RET{A,B} requires the SP to match its incoming value on entry to the
316 // function.
317 //
318 // If the PAC-RET hardening based on load from the return address is
319 // enabled, fallback to the use of AUTIASP/AUTIBSP and RET.
320 bool TerminatorIsCombinable = std::next(x: MBBI) == TI && TI != MBB.end() &&
321 TI->getOpcode() == AArch64::RET &&
322 ArgumentStackToRestore == 0 &&
323 !MFnI->shouldHardenSignReturnAddress();
324
325 if (Subtarget->hasPAuth() && TerminatorIsCombinable && !NeedsWinCFI &&
326 !MF.getFunction().hasFnAttribute(Kind: Attribute::ShadowCallStack)) {
327 if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
328 assert(PACSym && "No PAC instruction to refer to");
329 BuildMI(BB&: MBB, I: TI, MIMD: DL,
330 MCID: TII->get(Opcode: UseBKey ? AArch64::RETABSPPCi : AArch64::RETAASPPCi))
331 .addSym(Sym: PACSym)
332 .copyImplicitOps(OtherMI: *MBBI)
333 .setMIFlag(MachineInstr::FrameDestroy);
334 } else {
335 if (MFnI->branchProtectionPAuthLR()) {
336 emitEpiloguePACSymOffsetIntoReg(TII: *TII, MBB, I: MBBI, DL, PACSym,
337 Reg: AArch64::X16);
338 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PACM))
339 .setMIFlag(MachineInstr::FrameDestroy);
340 }
341 BuildMI(BB&: MBB, I: TI, MIMD: DL, MCID: TII->get(Opcode: UseBKey ? AArch64::RETAB : AArch64::RETAA))
342 .copyImplicitOps(OtherMI: *MBBI)
343 .setMIFlag(MachineInstr::FrameDestroy);
344 }
345 MBB.erase(I: TI);
346 return;
347 }
348
349 // If PAUTH_EPILOGUE is at insertion point with a net zero offset on SP, we
350 // can use an AUT form with a hardcoded SP discriminator.
351 if (ArgumentStackToRestore == 0) {
352 if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
353 assert(PACSym && "No PAC instruction to refer to");
354 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
355 MCID: TII->get(Opcode: UseBKey ? AArch64::AUTIBSPPCi : AArch64::AUTIASPPCi))
356 .addSym(Sym: PACSym)
357 .setMIFlag(MachineInstr::FrameDestroy);
358 emitAUTCFI(MBB, MBBI, EmitCFI: EmitAsyncCFI);
359 } else {
360 if (MFnI->branchProtectionPAuthLR()) {
361 emitEpiloguePACSymOffsetIntoReg(TII: *TII, MBB, I: MBBI, DL, PACSym,
362 Reg: AArch64::X16);
363
364 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PACM))
365 .setMIFlag(MachineInstr::FrameDestroy);
366 }
367 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
368 MCID: TII->get(Opcode: UseBKey ? AArch64::AUTIBSP : AArch64::AUTIASP))
369 .setMIFlag(MachineInstr::FrameDestroy);
370 emitAUTCFI(MBB, MBBI, EmitCFI: EmitAsyncCFI);
371 }
372
373 if (NeedsWinCFI) {
374 assert(UseBKey &&
375 "Windows SEH PAC unwind info only supports B-key signing");
376 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_PACSignLR))
377 .setMIFlag(MachineInstr::FrameDestroy);
378 }
379
380 return;
381 }
382
383 // Windows unwind info describes the authentication as a single pac_sign_lr
384 // unwind code, which the unwinder applies with the SP it has at that point.
385 // That is the signing SP only if the epilogue leaves SP at its entry value,
386 // and a single code can't stand in for the multi-instruction sequence below.
387 // So there, the epilogue emitter authenticates before the epilogue starts,
388 // where the prologue's unwind codes still apply, and the epilogue's unwind
389 // codes don't mention pac_sign_lr.
390 if (NeedsWinCFI && !EntrySPInX16)
391 report_fatal_error(reason: "Can't handle a tail call that changes the stack "
392 "argument size in a function that signs its return "
393 "address on Windows");
394
395 // When ArgumentStackToRestore > 0, this function received more argument
396 // space than the tail callee pops. The epilogue contains an SP adjustment
397 // (e.g. "add sp, sp, #N") to discard the leftover argument space.
398 //
399 // When ArgumentStackToRestore < 0, the tail callee pops more argument space
400 // than this function received, so after the frame teardown, SP is below the
401 // entry SP used as the signing modifier.
402 //
403 // We cannot simply bump SP first and then use AUTI[AB]SP with the bumped
404 // value, because the live arguments would fall below SP and potentially
405 // outside the red-zone.
406 //
407 // At this point there is an offset to the incoming SP, and we can't use the
408 // aut variants that hard-code SP. Reconstruct entry SP in x16 and
409 // authenticate using AUTI[AB]1716 (x17=LR, x16=entry_SP).
410 if (!EntrySPInX16)
411 emitFrameOffset(MBB, MBBI, DL, DestReg: AArch64::X16, SrcReg: AArch64::SP,
412 Offset: StackOffset::getFixed(Fixed: -ArgumentStackToRestore), TII,
413 MachineInstr::FrameDestroy);
414
415 // At this point, if EntrySPInX16 is true, it means we're using
416 // PAUTH_EPILOGUE_ENTRY_SP, so the signed return address will already be in
417 // x17 if the sequence below works on that, as the epilogue emitter works out
418 // using AArch64PAuth::authenticatesLRViaX17, and in LR if it doesn't. Each
419 // of the sequences below asserts that it agrees.
420 [[maybe_unused]] const bool ViaX17 = AArch64PAuth::authenticatesLRViaX17(
421 BranchProtectionPAuthLR: MFnI->branchProtectionPAuthLR(), HasPAuth: Subtarget->hasPAuth());
422
423 if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
424 assert(ViaX17 && "Expected to authenticate via X17");
425 if (!EntrySPInX16)
426 emitMOVWithFrameDestroy(MBB, MBBI, DL, TII, Dst: AArch64::X17, Src: AArch64::LR);
427
428 assert(PACSym && "No PAC instruction to refer to");
429 emitEpiloguePACSymOffsetIntoReg(TII: *TII, MBB, I: MBBI, DL, PACSym, Reg: AArch64::X15);
430
431 unsigned AutOpc = UseBKey ? AArch64::AUTIB171615 : AArch64::AUTIA171615;
432 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AutOpc))
433 .setMIFlag(MachineInstr::FrameDestroy);
434 emitAUTCFI(MBB, MBBI, EmitCFI: EmitAsyncCFI);
435
436 emitMOVWithFrameDestroy(MBB, MBBI, DL, TII, Dst: AArch64::LR, Src: AArch64::X17);
437 } else if (MFnI->branchProtectionPAuthLR()) {
438 assert(ViaX17 && "Expected to authenticate via X17");
439 if (!EntrySPInX16)
440 emitMOVWithFrameDestroy(MBB, MBBI, DL, TII, Dst: AArch64::X17, Src: AArch64::LR);
441
442 assert(PACSym && "No PAC instruction to refer to");
443 emitEpiloguePACSymOffsetIntoReg(TII: *TII, MBB, I: MBBI, DL, PACSym, Reg: AArch64::X15);
444
445 // The PACM hint-space instruction modifies the following AUTI[AB]1716
446 // to optionally take x15 as an extra operand depending on the
447 // presence of +pauth-lr at runtime. On machines without +pauth-lr, it
448 // behaves as a nop, and the address of the PACI[AB]SP in x15 is
449 // ignored.
450 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PACM))
451 .setMIFlag(MachineInstr::FrameDestroy);
452
453 unsigned AutOpc = UseBKey ? AArch64::AUTIB1716 : AArch64::AUTIA1716;
454 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AutOpc))
455 .setMIFlag(MachineInstr::FrameDestroy);
456 emitAUTCFI(MBB, MBBI, EmitCFI: EmitAsyncCFI);
457
458 emitMOVWithFrameDestroy(MBB, MBBI, DL, TII, Dst: AArch64::LR, Src: AArch64::X17);
459 } else if (Subtarget->hasPAuth()) {
460 assert(!ViaX17 && "Expected to authenticate using LR directly");
461 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: UseBKey ? AArch64::AUTIB : AArch64::AUTIA),
462 DestReg: AArch64::LR)
463 .addUse(RegNo: AArch64::LR)
464 .addUse(RegNo: AArch64::X16)
465 .setMIFlag(MachineInstr::FrameDestroy);
466 emitAUTCFI(MBB, MBBI, EmitCFI: EmitAsyncCFI);
467 } else {
468 assert(ViaX17 && "Expected to authenticate via X17");
469 if (!EntrySPInX16)
470 emitMOVWithFrameDestroy(MBB, MBBI, DL, TII, Dst: AArch64::X17, Src: AArch64::LR);
471
472 unsigned AutOpc = UseBKey ? AArch64::AUTIB1716 : AArch64::AUTIA1716;
473 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AutOpc))
474 .setMIFlag(MachineInstr::FrameDestroy);
475 emitAUTCFI(MBB, MBBI, EmitCFI: EmitAsyncCFI);
476
477 emitMOVWithFrameDestroy(MBB, MBBI, DL, TII, Dst: AArch64::LR, Src: AArch64::X17);
478 }
479
480 if (NeedsWinCFI && !EntrySPInX16) {
481 assert(UseBKey &&
482 "Windows SEH PAC unwind info only supports B-key signing");
483 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_PACSignLR))
484 .setMIFlag(MachineInstr::FrameDestroy);
485 }
486}
487
488unsigned llvm::AArch64PAuth::getCheckerSizeInBytes(AuthCheckMethod Method) {
489 switch (Method) {
490 case AuthCheckMethod::None:
491 return 0;
492 case AuthCheckMethod::DummyLoad:
493 return 4;
494 case AuthCheckMethod::HighBitsNoTBI:
495 return 12;
496 case AuthCheckMethod::XPACHint:
497 case AuthCheckMethod::XPAC:
498 return 20;
499 }
500 llvm_unreachable("Unknown AuthCheckMethod enum");
501}
502
503bool AArch64PointerAuthImpl::run(MachineFunction &MF) {
504 Subtarget = &MF.getSubtarget<AArch64Subtarget>();
505 TII = Subtarget->getInstrInfo();
506
507 SmallVector<MachineBasicBlock::instr_iterator> PAuthPseudoInstrs;
508
509 bool Modified = false;
510
511 for (auto &MBB : MF) {
512 for (auto &MI : MBB) {
513 switch (MI.getOpcode()) {
514 default:
515 break;
516 case AArch64::PAUTH_PROLOGUE:
517 case AArch64::PAUTH_EPILOGUE:
518 case AArch64::PAUTH_EPILOGUE_ENTRY_SP:
519 PAuthPseudoInstrs.push_back(Elt: MI.getIterator());
520 break;
521 }
522 }
523 }
524
525 for (auto It : PAuthPseudoInstrs) {
526 switch (It->getOpcode()) {
527 case AArch64::PAUTH_PROLOGUE:
528 signLR(MF, MBBI: It);
529 break;
530 case AArch64::PAUTH_EPILOGUE:
531 case AArch64::PAUTH_EPILOGUE_ENTRY_SP:
532 authenticateLR(MF, MBBI: It);
533 break;
534 default:
535 llvm_unreachable("Unhandled opcode");
536 }
537 It->eraseFromParent();
538 Modified = true;
539 }
540
541 Modified |= emitSignReturnAddressHardening(MF);
542
543 return Modified;
544}
545
546bool AArch64PointerAuthImpl::emitSignReturnAddressHardening(
547 MachineFunction &MF) {
548 const auto *FI = MF.getInfo<AArch64FunctionInfo>();
549 assert(FI && "FI can't be null");
550 if (!FI->shouldSignReturnAddress(MF) || !FI->shouldHardenSignReturnAddress())
551 return false;
552 assert(Subtarget && "Subtarget must be initialized");
553
554 RegScavenger RS;
555 bool Modified = false;
556 for (MachineBasicBlock &MBB : MF) {
557 MachineBasicBlock::iterator RetInstIter = MBB.getFirstTerminator();
558
559 if (RetInstIter == MBB.end() || RetInstIter->getOpcode() != AArch64::RET)
560 continue;
561
562 assert(RetInstIter->getOperand(0).getReg() == AArch64::LR &&
563 "Return instruction must be returning via LR");
564
565 MachineBasicBlock::iterator InsertionPoint = RetInstIter;
566 // In the case of Windows SEH, the hardening sequence does not immediately
567 // precede the return instruction. Instead, it precedes the SEH_EpilogEnd
568 // pseudo-instruction, which itself is expected to be the predecessor of
569 // the return. Plus, each instruction in the sequence needs one SEH_Nop.
570 const bool NeedsWinCFI = MF.hasWinCFI();
571 if (NeedsWinCFI) {
572 --InsertionPoint;
573 assert(InsertionPoint->getOpcode() == AArch64::SEH_EpilogEnd);
574 }
575 DebugLoc DL = InsertionPoint->getDebugLoc();
576 const auto EmitSEHNopIfRequired = [&]() {
577 if (NeedsWinCFI)
578 BuildMI(BB&: MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_Nop))
579 .setMIFlag(MachineInstr::FrameDestroy);
580 };
581
582 RS.enterBasicBlockEnd(MBB);
583 Register XReg = RS.scavengeRegisterBackwards(
584 RC: AArch64::GPR64RegClass, To: InsertionPoint,
585 /*RestoreAfter=*/false, /*SPAdj=*/0, /*AllowSpill=*/false);
586 if (XReg == AArch64::NoRegister) {
587 // Couldn't find a free register to use for the hardening. Skip.
588 MF.getContext().reportWarning(
589 L: SMLoc(), Msg: "harden-pac-ret failed for function " + MF.getName());
590 continue;
591 }
592
593 // Register copies are done using ORRXrs directly instead of using the
594 // pseudo-instruction COPY because this function can be called after
595 // pseudo-instruction expansion takes place, for example via the machine
596 // outliner pass.
597 emitMOVWithFrameDestroy(MBB, MBBI&: InsertionPoint, DL, TII, Dst: XReg, Src: AArch64::LR);
598 EmitSEHNopIfRequired();
599
600 // The XPACI instruction is only available with FEAT_PAUTH. So if the
601 // subtarget does not have it, the alternative XPACLRI instruction must be
602 // used instead. The latter is in hint space, therefore can be used even
603 // if FEAT_PAUTH is absent.
604 if (Subtarget->hasPAuth()) {
605 BuildMI(BB&: MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: AArch64::XPACI), DestReg: XReg)
606 .addUse(RegNo: XReg)
607 .setMIFlag(MachineInstr::FrameDestroy);
608 EmitSEHNopIfRequired();
609 Register WReg =
610 Subtarget->getRegisterInfo()->getSubReg(Reg: XReg, Idx: AArch64::sub_32);
611 BuildMI(BB&: MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: AArch64::LDRWui), DestReg: WReg)
612 .addUse(RegNo: XReg)
613 .addImm(Val: 0)
614 .addMemOperand(MMO: MF.getMachineMemOperand(
615 PtrInfo: MachinePointerInfo(),
616 F: MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile, Size: 4,
617 BaseAlignment: Align(4)))
618 .setMIFlag(MachineInstr::FrameDestroy);
619 EmitSEHNopIfRequired();
620 } else {
621 // Emit a CFI directive to tell unwinders that the return address is now
622 // saved in XReg.
623 CFIInstBuilder(MBB, InsertionPoint, MachineInstr::FrameDestroy)
624 .buildRegister(Reg1: AArch64::LR, Reg2: XReg);
625 BuildMI(BB&: MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: AArch64::XPACLRI))
626 .setMIFlag(MachineInstr::FrameDestroy);
627 EmitSEHNopIfRequired();
628 BuildMI(BB&: MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: AArch64::LDRWui), DestReg: AArch64::W30)
629 .addUse(RegNo: AArch64::LR)
630 .addImm(Val: 0)
631 .addMemOperand(MMO: MF.getMachineMemOperand(
632 PtrInfo: MachinePointerInfo(),
633 F: MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile, Size: 4,
634 BaseAlignment: Align(4)))
635 .setMIFlag(MachineInstr::FrameDestroy);
636 EmitSEHNopIfRequired();
637 BuildMI(BB&: MBB, I: RetInstIter, MIMD: DL, MCID: TII->get(Opcode: AArch64::RET))
638 .addUse(RegNo: XReg)
639 .copyImplicitOps(OtherMI: *RetInstIter);
640 MBB.erase(I: RetInstIter);
641 }
642 Modified = true;
643 }
644
645 return Modified;
646}
647
648bool AArch64PointerAuthLegacy::runOnMachineFunction(MachineFunction &MF) {
649 return AArch64PointerAuthImpl().run(MF);
650}
651
652PreservedAnalyses
653AArch64PointerAuthPass::run(MachineFunction &MF,
654 MachineFunctionAnalysisManager &MFAM) {
655 const bool Changed = AArch64PointerAuthImpl().run(MF);
656 if (!Changed)
657 return PreservedAnalyses::all();
658 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
659 PA.preserveSet<CFGAnalyses>();
660 return PA;
661}
662