1//===-- AArch64PointerAuth.cpp -- Harden code using PAuth ------------------==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AArch64PointerAuth.h"
10
11#include "AArch64.h"
12#include "AArch64FrameLowering.h"
13#include "AArch64InstrInfo.h"
14#include "AArch64MachineFunctionInfo.h"
15#include "AArch64Subtarget.h"
16#include "llvm/CodeGen/CFIInstBuilder.h"
17#include "llvm/CodeGen/MachineBasicBlock.h"
18#include "llvm/CodeGen/MachineInstrBuilder.h"
19#include "llvm/CodeGen/MachineModuleInfo.h"
20
21using namespace llvm;
22using namespace llvm::AArch64PAuth;
23
24#define AARCH64_POINTER_AUTH_NAME "AArch64 Pointer Authentication"
25
26namespace {
27
28class AArch64PointerAuthImpl {
29public:
30 bool run(MachineFunction &MF);
31
32private:
33 const AArch64Subtarget *Subtarget = nullptr;
34 const AArch64InstrInfo *TII = nullptr;
35
36 void signLR(MachineFunction &MF, MachineBasicBlock::iterator MBBI) const;
37
38 void authenticateLR(MachineFunction &MF,
39 MachineBasicBlock::iterator MBBI) const;
40};
41
42class AArch64PointerAuthLegacy : public MachineFunctionPass {
43public:
44 static char ID;
45
46 AArch64PointerAuthLegacy() : MachineFunctionPass(ID) {}
47
48 bool runOnMachineFunction(MachineFunction &MF) override;
49
50 StringRef getPassName() const override { return AARCH64_POINTER_AUTH_NAME; }
51};
52
53} // end anonymous namespace
54
55INITIALIZE_PASS(AArch64PointerAuthLegacy, "aarch64-ptrauth",
56 AARCH64_POINTER_AUTH_NAME, false, false)
57
58FunctionPass *llvm::createAArch64PointerAuthPass() {
59 return new AArch64PointerAuthLegacy();
60}
61
62char AArch64PointerAuthLegacy::ID = 0;
63
64static void emitEpiloguePACSymOffsetIntoReg(const TargetInstrInfo &TII,
65 MachineBasicBlock &MBB,
66 MachineBasicBlock::iterator I,
67 DebugLoc DL, MCSymbol *PACSym,
68 Register Reg) {
69 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AArch64::ADRP), DestReg: Reg)
70 .addSym(Sym: PACSym, TargetFlags: AArch64II::MO_PAGE)
71 .setMIFlag(MachineInstr::FrameDestroy);
72 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AArch64::ADDXri), DestReg: Reg)
73 .addReg(RegNo: Reg)
74 .addSym(Sym: PACSym, TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC)
75 .addImm(Val: 0)
76 .setMIFlag(MachineInstr::FrameDestroy);
77}
78
79static void emitPACCFI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
80 MachineInstr::MIFlag Flags, bool EmitCFI) {
81 if (!EmitCFI)
82 return;
83
84 auto &MF = *MBB.getParent();
85 auto &MFnI = *MF.getInfo<AArch64FunctionInfo>();
86
87 // DW_CFA_AARCH64_negate_ra_state_with_pc is semantically broken for
88 // functions where shrinkwrapping places signing/authenticating pairs on
89 // distinct CFG paths.
90 //
91 // DWARF CFI is evaluated linearly over the byte stream, not along control
92 // flow edges. The toggle semantics of this directive therefore cannot
93 // faithfully represent the signed/unsigned RA state for all possible CFG
94 // paths. The added complexity versus DW_CFA_AARCH64_negate_ra_state is that
95 // an unwinder must also reconstruct the PC of the PACI[AB]SPPC in order to
96 // verify the signed LR, and that address is derived from the location of this
97 // directive in the linear CFI stream.
98 //
99 // The correct fix is to use DW_CFA_AARCH64_set_ra_state_with_pc, which sets
100 // the RA state and signing address absolutely rather than toggling them. An
101 // unwinder that supports this directive can reconstruct the correct state on
102 // any CFG path, regardless of how many signing/authenticating pairs exist in
103 // the function. However, not all unwinders support this directive, so we
104 // cannot rely on it exclusively.
105 //
106 // For unwinders that only support DW_CFA_AARCH64_negate_ra_state_with_pc,
107 // libunwind exploits a loophole: it records the address at the
108 // DW_CFA_AARCH64_negate_ra_state_with_pc site to authenticate the LR, but
109 // does not care that the CFI state remains "signed with pc" after
110 // authentication has occurred. This means we can safely omit the
111 // FrameDestroy emission of this directive, treating it solely as a marker
112 // for the signing site, as long as each function has at most one such
113 // signing location. That invariant holds today because shrinkwrapping
114 // does not yet hoist or sink PAuth_LR frame code across CFG join/split
115 // points; once it does, we must avoid those transformations on platforms that
116 // have this limitation.
117 //
118 // https://github.com/ARM-software/abi-aa/issues/327
119 // https://github.com/ARM-software/abi-aa/pull/346
120 if (Flags == MachineInstr::FrameDestroy && MFnI.branchProtectionPAuthLR())
121 return;
122
123 CFIInstBuilder CFIBuilder(MBB, MBBI, Flags);
124 if (MFnI.branchProtectionPAuthLR()) {
125 CFIBuilder.buildNegateRAStateWithPC();
126 } else if (!MF.getTarget().getTargetTriple().isOSBinFormatMachO()) {
127 CFIBuilder.buildNegateRAState();
128 }
129}
130
131void AArch64PointerAuthImpl::signLR(MachineFunction &MF,
132 MachineBasicBlock::iterator MBBI) const {
133 auto &MFnI = *MF.getInfo<AArch64FunctionInfo>();
134 bool UseBKey = MFnI.shouldSignWithBKey();
135 bool EmitCFI = MFnI.needsDwarfUnwindInfo(MF);
136 bool NeedsWinCFI = MF.hasWinCFI();
137
138 MachineBasicBlock &MBB = *MBBI->getParent();
139
140 // Debug location must be unknown, see AArch64FrameLowering::emitPrologue.
141 DebugLoc DL;
142
143 if (UseBKey && !MF.getTarget().getTargetTriple().isOSBinFormatMachO()) {
144 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::EMITBKEY))
145 .setMIFlag(MachineInstr::FrameSetup);
146 }
147
148 // PAuthLR authentication instructions need to know the value of PC at the
149 // point of signing (PACI*).
150 if (MFnI.branchProtectionPAuthLR()) {
151 MCSymbol *PACSym = MF.getContext().createTempSymbol();
152 MFnI.setSigningInstrLabel(PACSym);
153 }
154
155 // No SEH opcode for this one; it doesn't materialize into an
156 // instruction on Windows.
157 if (MFnI.branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
158 emitPACCFI(MBB, MBBI, Flags: MachineInstr::FrameSetup, EmitCFI);
159 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
160 MCID: TII->get(Opcode: UseBKey ? AArch64::PACIBSPPC : AArch64::PACIASPPC))
161 .setMIFlag(MachineInstr::FrameSetup)
162 ->setPreInstrSymbol(MF, Symbol: MFnI.getSigningInstrLabel());
163 } else {
164 if (MFnI.branchProtectionPAuthLR()) {
165 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PACM))
166 .setMIFlag(MachineInstr::FrameSetup);
167 emitPACCFI(MBB, MBBI, Flags: MachineInstr::FrameSetup, EmitCFI);
168 }
169 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
170 MCID: TII->get(Opcode: UseBKey ? AArch64::PACIBSP : AArch64::PACIASP))
171 .setMIFlag(MachineInstr::FrameSetup)
172 ->setPreInstrSymbol(MF, Symbol: MFnI.getSigningInstrLabel());
173 if (!MFnI.branchProtectionPAuthLR())
174 emitPACCFI(MBB, MBBI, Flags: MachineInstr::FrameSetup, EmitCFI);
175 }
176
177 if (!EmitCFI && NeedsWinCFI) {
178 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_PACSignLR))
179 .setMIFlag(MachineInstr::FrameSetup);
180 }
181}
182
183void AArch64PointerAuthImpl::authenticateLR(
184 MachineFunction &MF, MachineBasicBlock::iterator MBBI) const {
185 const AArch64FunctionInfo *MFnI = MF.getInfo<AArch64FunctionInfo>();
186 bool UseBKey = MFnI->shouldSignWithBKey();
187 bool EmitAsyncCFI = MFnI->needsAsyncDwarfUnwindInfo(MF);
188 bool NeedsWinCFI = MF.hasWinCFI();
189
190 MachineBasicBlock &MBB = *MBBI->getParent();
191 DebugLoc DL = MBBI->getDebugLoc();
192 // MBBI points to a PAUTH_EPILOGUE instruction to be replaced and
193 // TI points to a terminator instruction that may or may not be combined.
194 // Note that inserting new instructions "before MBBI" and "before TI" is
195 // not the same because if ShadowCallStack is enabled, its instructions
196 // are placed between MBBI and TI.
197 MachineBasicBlock::iterator TI = MBB.getFirstInstrTerminator();
198
199 // The AUTIASP instruction assembles to a hint instruction before v8.3a so
200 // this instruction can safely used for any v8a architecture.
201 // From v8.3a onwards there are optimised authenticate LR and return
202 // instructions, namely RETA{A,B}, that can be used instead. In this case the
203 // DW_CFA_AARCH64_negate_ra_state can't be emitted.
204 bool TerminatorIsCombinable =
205 TI != MBB.end() && TI->getOpcode() == AArch64::RET;
206 MCSymbol *PACSym = MFnI->getSigningInstrLabel();
207
208 if (Subtarget->hasPAuth() && TerminatorIsCombinable && !NeedsWinCFI &&
209 !MF.getFunction().hasFnAttribute(Kind: Attribute::ShadowCallStack)) {
210 if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
211 assert(PACSym && "No PAC instruction to refer to");
212 BuildMI(BB&: MBB, I: TI, MIMD: DL,
213 MCID: TII->get(Opcode: UseBKey ? AArch64::RETABSPPCi : AArch64::RETAASPPCi))
214 .addSym(Sym: PACSym)
215 .copyImplicitOps(OtherMI: *MBBI)
216 .setMIFlag(MachineInstr::FrameDestroy);
217 } else {
218 if (MFnI->branchProtectionPAuthLR()) {
219 emitEpiloguePACSymOffsetIntoReg(TII: *TII, MBB, I: MBBI, DL, PACSym,
220 Reg: AArch64::X16);
221 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PACM))
222 .setMIFlag(MachineInstr::FrameDestroy);
223 }
224 BuildMI(BB&: MBB, I: TI, MIMD: DL, MCID: TII->get(Opcode: UseBKey ? AArch64::RETAB : AArch64::RETAA))
225 .copyImplicitOps(OtherMI: *MBBI)
226 .setMIFlag(MachineInstr::FrameDestroy);
227 }
228 MBB.erase(I: TI);
229 return;
230 }
231
232 auto &AFL = *static_cast<const AArch64FrameLowering *>(
233 MF.getSubtarget().getFrameLowering());
234 int64_t ArgumentStackToRestore = AFL.getArgumentStackToRestore(MF, MBB);
235
236 // When ArgumentStackToRestore < 0, the tail callee pops more argument space
237 // than this function received, so after the frame teardown SP is below the
238 // entry SP used as the signing modifier. Reconstruct entry SP in x16 and
239 // authenticate using AUTI[AB]1716 (x17=LR, x16=entry_SP).
240 if (ArgumentStackToRestore < 0) {
241 emitFrameOffset(MBB, MBBI, DL, DestReg: AArch64::X16, SrcReg: AArch64::SP,
242 Offset: StackOffset::getFixed(Fixed: -ArgumentStackToRestore), TII,
243 MachineInstr::FrameDestroy);
244
245 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ORRXrs), DestReg: AArch64::X17)
246 .addReg(RegNo: AArch64::XZR)
247 .addReg(RegNo: AArch64::LR)
248 .addImm(Val: 0)
249 .setMIFlag(MachineInstr::FrameDestroy);
250
251 if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
252 assert(PACSym && "No PAC instruction to refer to");
253 emitEpiloguePACSymOffsetIntoReg(TII: *TII, MBB, I: MBBI, DL, PACSym,
254 Reg: AArch64::X15);
255
256 emitPACCFI(MBB, MBBI, Flags: MachineInstr::FrameDestroy, EmitCFI: EmitAsyncCFI);
257 unsigned AutOpc = UseBKey ? AArch64::AUTIB171615 : AArch64::AUTIA171615;
258 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AutOpc))
259 .setMIFlag(MachineInstr::FrameDestroy);
260 } else if (MFnI->branchProtectionPAuthLR()) {
261 assert(PACSym && "No PAC instruction to refer to");
262 emitEpiloguePACSymOffsetIntoReg(TII: *TII, MBB, I: MBBI, DL, PACSym,
263 Reg: AArch64::X15);
264
265 // The PACM hint-space instruction modifies the following AUTI[AB]1716
266 // to optionally take x15 as an extra operand depending on the
267 // presence of +pauth-lr at runtime. On machines without +pauth-lr, it
268 // behaves as a nop, and the address of the PACI[AB]SP in x15 is
269 // ignored.
270 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PACM))
271 .setMIFlag(MachineInstr::FrameDestroy);
272
273 emitPACCFI(MBB, MBBI, Flags: MachineInstr::FrameDestroy, EmitCFI: EmitAsyncCFI);
274 unsigned AutOpc = UseBKey ? AArch64::AUTIB1716 : AArch64::AUTIA1716;
275 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AutOpc))
276 .setMIFlag(MachineInstr::FrameDestroy);
277 } else {
278 unsigned AutOpc = UseBKey ? AArch64::AUTIB1716 : AArch64::AUTIA1716;
279 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AutOpc))
280 .setMIFlag(MachineInstr::FrameDestroy);
281 emitPACCFI(MBB, MBBI, Flags: MachineInstr::FrameDestroy, EmitCFI: EmitAsyncCFI);
282 }
283
284 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ORRXrs), DestReg: AArch64::LR)
285 .addReg(RegNo: AArch64::XZR)
286 .addReg(RegNo: AArch64::X17)
287 .addImm(Val: 0)
288 .setMIFlag(MachineInstr::FrameDestroy);
289 return;
290 }
291
292 // When ArgumentStackToRestore > 0, this function received more argument
293 // space than the tail callee pops. The epilogue contains an SP adjustment
294 // (e.g. "add sp, sp, #N") to discard the leftover argument space. We must
295 // authenticate *before* that adjustment so that AUTI[AB]SP sees the entry
296 // SP discriminator. Move any such SP-adjusting instructions to after the
297 // authentication instruction.
298 //
299 // We cannot simply bump SP first and then use AUTI[AB]SP with the bumped
300 // value, because the live arguments would fall below SP and potentially
301 // outside the red-zone.
302 SmallVector<MachineInstr *, 2> SPMods;
303 if (ArgumentStackToRestore > 0) {
304 for (auto I = MBBI; I->getFlag(Flag: MachineInstr::FrameDestroy); --I) {
305 if ((I->getOpcode() == AArch64::ADDXri ||
306 I->getOpcode() == AArch64::SUBXri) &&
307 I->getOperand(i: 0).getReg() == AArch64::SP &&
308 I->getOperand(i: 1).getReg() == AArch64::SP)
309 SPMods.push_back(Elt: &*I);
310 }
311 }
312 for (auto *MI : SPMods)
313 MI->removeFromParent();
314
315 if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
316 assert(PACSym && "No PAC instruction to refer to");
317 emitPACCFI(MBB, MBBI, Flags: MachineInstr::FrameDestroy, EmitCFI: EmitAsyncCFI);
318 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
319 MCID: TII->get(Opcode: UseBKey ? AArch64::AUTIBSPPCi : AArch64::AUTIASPPCi))
320 .addSym(Sym: PACSym)
321 .setMIFlag(MachineInstr::FrameDestroy);
322 } else {
323 if (MFnI->branchProtectionPAuthLR()) {
324 emitEpiloguePACSymOffsetIntoReg(TII: *TII, MBB, I: MBBI, DL, PACSym,
325 Reg: AArch64::X16);
326
327 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PACM))
328 .setMIFlag(MachineInstr::FrameDestroy);
329 emitPACCFI(MBB, MBBI, Flags: MachineInstr::FrameDestroy, EmitCFI: EmitAsyncCFI);
330 }
331 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
332 MCID: TII->get(Opcode: UseBKey ? AArch64::AUTIBSP : AArch64::AUTIASP))
333 .setMIFlag(MachineInstr::FrameDestroy);
334 if (!MFnI->branchProtectionPAuthLR())
335 emitPACCFI(MBB, MBBI, Flags: MachineInstr::FrameDestroy, EmitCFI: EmitAsyncCFI);
336 }
337
338 if (NeedsWinCFI) {
339 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_PACSignLR))
340 .setMIFlag(MachineInstr::FrameDestroy);
341 }
342
343 for (auto *MI : SPMods)
344 MBB.insert(I: MBBI, MI);
345}
346
347unsigned llvm::AArch64PAuth::getCheckerSizeInBytes(AuthCheckMethod Method) {
348 switch (Method) {
349 case AuthCheckMethod::None:
350 return 0;
351 case AuthCheckMethod::DummyLoad:
352 return 4;
353 case AuthCheckMethod::HighBitsNoTBI:
354 return 12;
355 case AuthCheckMethod::XPACHint:
356 case AuthCheckMethod::XPAC:
357 return 20;
358 }
359 llvm_unreachable("Unknown AuthCheckMethod enum");
360}
361
362bool AArch64PointerAuthImpl::run(MachineFunction &MF) {
363 Subtarget = &MF.getSubtarget<AArch64Subtarget>();
364 TII = Subtarget->getInstrInfo();
365
366 SmallVector<MachineBasicBlock::instr_iterator> PAuthPseudoInstrs;
367
368 bool Modified = false;
369
370 for (auto &MBB : MF) {
371 for (auto &MI : MBB) {
372 switch (MI.getOpcode()) {
373 default:
374 break;
375 case AArch64::PAUTH_PROLOGUE:
376 case AArch64::PAUTH_EPILOGUE:
377 PAuthPseudoInstrs.push_back(Elt: MI.getIterator());
378 break;
379 }
380 }
381 }
382
383 for (auto It : PAuthPseudoInstrs) {
384 switch (It->getOpcode()) {
385 case AArch64::PAUTH_PROLOGUE:
386 signLR(MF, MBBI: It);
387 break;
388 case AArch64::PAUTH_EPILOGUE:
389 authenticateLR(MF, MBBI: It);
390 break;
391 default:
392 llvm_unreachable("Unhandled opcode");
393 }
394 It->eraseFromParent();
395 Modified = true;
396 }
397
398 return Modified;
399}
400
401bool AArch64PointerAuthLegacy::runOnMachineFunction(MachineFunction &MF) {
402 return AArch64PointerAuthImpl().run(MF);
403}
404
405PreservedAnalyses
406AArch64PointerAuthPass::run(MachineFunction &MF,
407 MachineFunctionAnalysisManager &MFAM) {
408 const bool Changed = AArch64PointerAuthImpl().run(MF);
409 if (!Changed)
410 return PreservedAnalyses::all();
411 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
412 PA.preserveSet<CFGAnalyses>();
413 return PA;
414}
415