1//===-- AArch64PointerAuth.cpp -- Harden code using PAuth ------------------==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AArch64PointerAuth.h"
10
11#include "AArch64.h"
12#include "AArch64FrameLowering.h"
13#include "AArch64InstrInfo.h"
14#include "AArch64MachineFunctionInfo.h"
15#include "AArch64Subtarget.h"
16#include "llvm/CodeGen/CFIInstBuilder.h"
17#include "llvm/CodeGen/MachineBasicBlock.h"
18#include "llvm/CodeGen/MachineInstrBuilder.h"
19#include "llvm/CodeGen/MachineModuleInfo.h"
20
21using namespace llvm;
22using namespace llvm::AArch64PAuth;
23
24#define AARCH64_POINTER_AUTH_NAME "AArch64 Pointer Authentication"
25
26namespace {
27
28class AArch64PointerAuthImpl {
29public:
30 bool run(MachineFunction &MF);
31
32private:
33 const AArch64Subtarget *Subtarget = nullptr;
34 const AArch64InstrInfo *TII = nullptr;
35
36 void signLR(MachineFunction &MF, MachineBasicBlock::iterator MBBI) const;
37
38 void authenticateLR(MachineFunction &MF,
39 MachineBasicBlock::iterator MBBI) const;
40};
41
42class AArch64PointerAuthLegacy : public MachineFunctionPass {
43public:
44 static char ID;
45
46 AArch64PointerAuthLegacy() : MachineFunctionPass(ID) {}
47
48 bool runOnMachineFunction(MachineFunction &MF) override;
49
50 StringRef getPassName() const override { return AARCH64_POINTER_AUTH_NAME; }
51};
52
53} // end anonymous namespace
54
55INITIALIZE_PASS(AArch64PointerAuthLegacy, "aarch64-ptrauth",
56 AARCH64_POINTER_AUTH_NAME, false, false)
57
58FunctionPass *llvm::createAArch64PointerAuthPass() {
59 return new AArch64PointerAuthLegacy();
60}
61
62char AArch64PointerAuthLegacy::ID = 0;
63
64static void emitEpiloguePACSymOffsetIntoReg(const TargetInstrInfo &TII,
65 MachineBasicBlock &MBB,
66 MachineBasicBlock::iterator I,
67 DebugLoc DL, MCSymbol *PACSym,
68 Register Reg) {
69 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AArch64::ADRP), DestReg: Reg)
70 .addSym(Sym: PACSym, TargetFlags: AArch64II::MO_PAGE)
71 .setMIFlag(MachineInstr::FrameDestroy);
72 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AArch64::ADDXri), DestReg: Reg)
73 .addReg(RegNo: Reg)
74 .addSym(Sym: PACSym, TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC)
75 .addImm(Val: 0)
76 .setMIFlag(MachineInstr::FrameDestroy);
77}
78
79// Wrap a given PAC instruction in CFI that describes it.
80// Depending on the type of CFI required, we may need to emit the directive
81// either before or after the instruction, so that unwinders can correctly
82// interpret the location of the signing instruction.
83template <typename BuildPACMIFn>
84static void decoratePACWithCFI(MachineBasicBlock &MBB,
85 MachineBasicBlock::iterator MBBI, bool EmitCFI,
86 BuildPACMIFn BuildPACMI) {
87 if (!EmitCFI) {
88 BuildPACMI();
89 return;
90 }
91
92 auto &MF = *MBB.getParent();
93 auto &MFnI = *MF.getInfo<AArch64FunctionInfo>();
94
95 CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
96 if (MFnI.branchProtectionPAuthLR()) {
97 CFIBuilder.buildNegateRAStateWithPC();
98 BuildPACMI();
99 } else {
100 BuildPACMI();
101 if (!MF.getTarget().getTargetTriple().isOSBinFormatMachO()) {
102 CFIBuilder.buildNegateRAState();
103 }
104 }
105}
106
107static void emitAUTCFI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
108 bool EmitCFI) {
109 if (!EmitCFI)
110 return;
111
112 auto &MF = *MBB.getParent();
113 auto &MFnI = *MF.getInfo<AArch64FunctionInfo>();
114 CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameDestroy);
115 const Triple &TT = MF.getTarget().getTargetTriple();
116
117 if (MFnI.branchProtectionPAuthLR()) {
118 // DW_CFA_AARCH64_negate_ra_state_with_pc is semantically broken for
119 // functions where shrinkwrapping places signing/authenticating pairs on
120 // distinct CFG paths.
121 //
122 // DWARF CFI is evaluated linearly over the byte stream, not along control
123 // flow edges. The toggle semantics of this directive therefore cannot
124 // faithfully represent the signed/unsigned RA state for all possible CFG
125 // paths. The added complexity versus DW_CFA_AARCH64_negate_ra_state is
126 // that an unwinder must also reconstruct the PC of the PACI[AB]SPPC in
127 // order to verify the signed LR, and that address is derived from the
128 // location of this directive in the linear CFI stream.
129 //
130 // The correct fix is to use DW_CFA_AARCH64_set_ra_state_with_pc, which
131 // sets the RA state and signing address absolutely rather than toggling
132 // them. An unwinder that supports this directive can reconstruct the
133 // correct state on any CFG path, regardless of how many
134 // signing/authenticating pairs exist in the function. However, not all
135 // unwinders support this directive, so we cannot rely on it exclusively.
136 //
137 // For unwinders that only support DW_CFA_AARCH64_negate_ra_state_with_pc,
138 // libunwind exploits a loophole: it records the address at the
139 // DW_CFA_AARCH64_negate_ra_state_with_pc site to authenticate the LR, but
140 // does not care that the CFI state remains "signed with pc" after
141 // authentication has occurred. This means we can safely omit the
142 // FrameDestroy emission of this directive, treating it solely as a marker
143 // for the signing site, as long as each function has at most one such
144 // signing location. That invariant holds today because shrinkwrapping
145 // does not yet hoist or sink PAuth_LR frame code across CFG join/split
146 // points; once it does, we must avoid those transformations on platforms
147 // that have this limitation.
148 //
149 // https://github.com/ARM-software/abi-aa/issues/327
150 // https://github.com/ARM-software/abi-aa/pull/346
151 } else if (!TT.isOSBinFormatMachO()) {
152 CFIBuilder.buildNegateRAState();
153 }
154}
155
156void AArch64PointerAuthImpl::signLR(MachineFunction &MF,
157 MachineBasicBlock::iterator MBBI) const {
158 auto &MFnI = *MF.getInfo<AArch64FunctionInfo>();
159 bool UseBKey = MFnI.shouldSignWithBKey();
160 bool EmitCFI = MFnI.needsDwarfUnwindInfo(MF);
161 bool NeedsWinCFI = MF.hasWinCFI();
162
163 MachineBasicBlock &MBB = *MBBI->getParent();
164
165 // Debug location must be unknown, see AArch64FrameLowering::emitPrologue.
166 DebugLoc DL;
167
168 if (UseBKey && !MF.getTarget().getTargetTriple().isOSBinFormatMachO()) {
169 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::EMITBKEY))
170 .setMIFlag(MachineInstr::FrameSetup);
171 }
172
173 // PAuthLR authentication instructions need to know the value of PC at the
174 // point of signing (PACI*).
175 if (MFnI.branchProtectionPAuthLR()) {
176 MCSymbol *PACSym = MF.getContext().createTempSymbol();
177 MFnI.setSigningInstrLabel(PACSym);
178 }
179
180 // No SEH opcode for this one; it doesn't materialize into an
181 // instruction on Windows.
182 if (MFnI.branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
183 decoratePACWithCFI(MBB, MBBI, EmitCFI, BuildPACMI: [&]() {
184 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
185 MCID: TII->get(Opcode: UseBKey ? AArch64::PACIBSPPC : AArch64::PACIASPPC))
186 .setMIFlag(MachineInstr::FrameSetup)
187 ->setPreInstrSymbol(MF, Symbol: MFnI.getSigningInstrLabel());
188 });
189 } else {
190 if (MFnI.branchProtectionPAuthLR()) {
191 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PACM))
192 .setMIFlag(MachineInstr::FrameSetup);
193 }
194 decoratePACWithCFI(MBB, MBBI, EmitCFI, BuildPACMI: [&]() {
195 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
196 MCID: TII->get(Opcode: UseBKey ? AArch64::PACIBSP : AArch64::PACIASP))
197 .setMIFlag(MachineInstr::FrameSetup)
198 ->setPreInstrSymbol(MF, Symbol: MFnI.getSigningInstrLabel());
199 });
200 }
201
202 if (!EmitCFI && NeedsWinCFI) {
203 assert(UseBKey &&
204 "Windows SEH PAC unwind info only supports B-key signing");
205 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_PACSignLR))
206 .setMIFlag(MachineInstr::FrameSetup);
207 }
208}
209
210void AArch64PointerAuthImpl::authenticateLR(
211 MachineFunction &MF, MachineBasicBlock::iterator MBBI) const {
212 const AArch64FunctionInfo *MFnI = MF.getInfo<AArch64FunctionInfo>();
213 bool UseBKey = MFnI->shouldSignWithBKey();
214 bool EmitAsyncCFI = MFnI->needsAsyncDwarfUnwindInfo(MF);
215 bool NeedsWinCFI = MF.hasWinCFI();
216
217 MachineBasicBlock &MBB = *MBBI->getParent();
218 DebugLoc DL = MBBI->getDebugLoc();
219 // MBBI points to a PAUTH_EPILOGUE instruction to be replaced and
220 // TI points to a terminator instruction that may or may not be combined.
221 // Note that inserting new instructions "before MBBI" and "before TI" is
222 // not the same because if ShadowCallStack is enabled, its instructions
223 // are placed between MBBI and TI.
224 MachineBasicBlock::iterator TI = MBB.getFirstInstrTerminator();
225
226 // The AUTIASP instruction assembles to a hint instruction before v8.3a so
227 // this instruction can safely used for any v8a architecture.
228 // From v8.3a onwards there are optimised authenticate LR and return
229 // instructions, namely RETA{A,B}, that can be used instead. In this case the
230 // DW_CFA_AARCH64_negate_ra_state can't be emitted.
231 bool TerminatorIsCombinable =
232 TI != MBB.end() && TI->getOpcode() == AArch64::RET;
233 MCSymbol *PACSym = MFnI->getSigningInstrLabel();
234
235 if (Subtarget->hasPAuth() && TerminatorIsCombinable && !NeedsWinCFI &&
236 !MF.getFunction().hasFnAttribute(Kind: Attribute::ShadowCallStack)) {
237 if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
238 assert(PACSym && "No PAC instruction to refer to");
239 BuildMI(BB&: MBB, I: TI, MIMD: DL,
240 MCID: TII->get(Opcode: UseBKey ? AArch64::RETABSPPCi : AArch64::RETAASPPCi))
241 .addSym(Sym: PACSym)
242 .copyImplicitOps(OtherMI: *MBBI)
243 .setMIFlag(MachineInstr::FrameDestroy);
244 } else {
245 if (MFnI->branchProtectionPAuthLR()) {
246 emitEpiloguePACSymOffsetIntoReg(TII: *TII, MBB, I: MBBI, DL, PACSym,
247 Reg: AArch64::X16);
248 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PACM))
249 .setMIFlag(MachineInstr::FrameDestroy);
250 }
251 BuildMI(BB&: MBB, I: TI, MIMD: DL, MCID: TII->get(Opcode: UseBKey ? AArch64::RETAB : AArch64::RETAA))
252 .copyImplicitOps(OtherMI: *MBBI)
253 .setMIFlag(MachineInstr::FrameDestroy);
254 }
255 MBB.erase(I: TI);
256 return;
257 }
258
259 auto &AFL = *static_cast<const AArch64FrameLowering *>(
260 MF.getSubtarget().getFrameLowering());
261 int64_t ArgumentStackToRestore = AFL.getArgumentStackToRestore(MF, MBB);
262
263 // When ArgumentStackToRestore < 0, the tail callee pops more argument space
264 // than this function received, so after the frame teardown SP is below the
265 // entry SP used as the signing modifier. Reconstruct entry SP in x16 and
266 // authenticate using AUTI[AB]1716 (x17=LR, x16=entry_SP).
267 if (ArgumentStackToRestore < 0) {
268 emitFrameOffset(MBB, MBBI, DL, DestReg: AArch64::X16, SrcReg: AArch64::SP,
269 Offset: StackOffset::getFixed(Fixed: -ArgumentStackToRestore), TII,
270 MachineInstr::FrameDestroy);
271
272 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ORRXrs), DestReg: AArch64::X17)
273 .addReg(RegNo: AArch64::XZR)
274 .addReg(RegNo: AArch64::LR)
275 .addImm(Val: 0)
276 .setMIFlag(MachineInstr::FrameDestroy);
277
278 if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
279 assert(PACSym && "No PAC instruction to refer to");
280 emitEpiloguePACSymOffsetIntoReg(TII: *TII, MBB, I: MBBI, DL, PACSym,
281 Reg: AArch64::X15);
282
283 emitAUTCFI(MBB, MBBI, EmitCFI: EmitAsyncCFI);
284 unsigned AutOpc = UseBKey ? AArch64::AUTIB171615 : AArch64::AUTIA171615;
285 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AutOpc))
286 .setMIFlag(MachineInstr::FrameDestroy);
287 } else if (MFnI->branchProtectionPAuthLR()) {
288 assert(PACSym && "No PAC instruction to refer to");
289 emitEpiloguePACSymOffsetIntoReg(TII: *TII, MBB, I: MBBI, DL, PACSym,
290 Reg: AArch64::X15);
291
292 // The PACM hint-space instruction modifies the following AUTI[AB]1716
293 // to optionally take x15 as an extra operand depending on the
294 // presence of +pauth-lr at runtime. On machines without +pauth-lr, it
295 // behaves as a nop, and the address of the PACI[AB]SP in x15 is
296 // ignored.
297 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PACM))
298 .setMIFlag(MachineInstr::FrameDestroy);
299
300 emitAUTCFI(MBB, MBBI, EmitCFI: EmitAsyncCFI);
301 unsigned AutOpc = UseBKey ? AArch64::AUTIB1716 : AArch64::AUTIA1716;
302 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AutOpc))
303 .setMIFlag(MachineInstr::FrameDestroy);
304 } else {
305 unsigned AutOpc = UseBKey ? AArch64::AUTIB1716 : AArch64::AUTIA1716;
306 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AutOpc))
307 .setMIFlag(MachineInstr::FrameDestroy);
308 emitAUTCFI(MBB, MBBI, EmitCFI: EmitAsyncCFI);
309 }
310
311 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ORRXrs), DestReg: AArch64::LR)
312 .addReg(RegNo: AArch64::XZR)
313 .addReg(RegNo: AArch64::X17)
314 .addImm(Val: 0)
315 .setMIFlag(MachineInstr::FrameDestroy);
316 return;
317 }
318
319 // When ArgumentStackToRestore > 0, this function received more argument
320 // space than the tail callee pops. The epilogue contains an SP adjustment
321 // (e.g. "add sp, sp, #N") to discard the leftover argument space. We must
322 // authenticate *before* that adjustment so that AUTI[AB]SP sees the entry
323 // SP discriminator. Move any such SP-adjusting instructions to after the
324 // authentication instruction.
325 //
326 // We cannot simply bump SP first and then use AUTI[AB]SP with the bumped
327 // value, because the live arguments would fall below SP and potentially
328 // outside the red-zone.
329 SmallVector<MachineInstr *, 2> SPMods;
330 if (ArgumentStackToRestore > 0) {
331 for (auto I = MBBI; I->getFlag(Flag: MachineInstr::FrameDestroy); --I) {
332 if ((I->getOpcode() == AArch64::ADDXri ||
333 I->getOpcode() == AArch64::SUBXri) &&
334 I->getOperand(i: 0).getReg() == AArch64::SP &&
335 I->getOperand(i: 1).getReg() == AArch64::SP)
336 SPMods.push_back(Elt: &*I);
337 }
338 }
339 for (auto *MI : SPMods)
340 MI->removeFromParent();
341
342 if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
343 assert(PACSym && "No PAC instruction to refer to");
344 emitAUTCFI(MBB, MBBI, EmitCFI: EmitAsyncCFI);
345 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
346 MCID: TII->get(Opcode: UseBKey ? AArch64::AUTIBSPPCi : AArch64::AUTIASPPCi))
347 .addSym(Sym: PACSym)
348 .setMIFlag(MachineInstr::FrameDestroy);
349 } else {
350 if (MFnI->branchProtectionPAuthLR()) {
351 emitEpiloguePACSymOffsetIntoReg(TII: *TII, MBB, I: MBBI, DL, PACSym,
352 Reg: AArch64::X16);
353
354 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PACM))
355 .setMIFlag(MachineInstr::FrameDestroy);
356 emitAUTCFI(MBB, MBBI, EmitCFI: EmitAsyncCFI);
357 }
358 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
359 MCID: TII->get(Opcode: UseBKey ? AArch64::AUTIBSP : AArch64::AUTIASP))
360 .setMIFlag(MachineInstr::FrameDestroy);
361 if (!MFnI->branchProtectionPAuthLR())
362 emitAUTCFI(MBB, MBBI, EmitCFI: EmitAsyncCFI);
363 }
364
365 if (NeedsWinCFI) {
366 assert(UseBKey &&
367 "Windows SEH PAC unwind info only supports B-key signing");
368 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_PACSignLR))
369 .setMIFlag(MachineInstr::FrameDestroy);
370 }
371
372 for (auto *MI : SPMods)
373 MBB.insert(I: MBBI, MI);
374}
375
376unsigned llvm::AArch64PAuth::getCheckerSizeInBytes(AuthCheckMethod Method) {
377 switch (Method) {
378 case AuthCheckMethod::None:
379 return 0;
380 case AuthCheckMethod::DummyLoad:
381 return 4;
382 case AuthCheckMethod::HighBitsNoTBI:
383 return 12;
384 case AuthCheckMethod::XPACHint:
385 case AuthCheckMethod::XPAC:
386 return 20;
387 }
388 llvm_unreachable("Unknown AuthCheckMethod enum");
389}
390
391bool AArch64PointerAuthImpl::run(MachineFunction &MF) {
392 Subtarget = &MF.getSubtarget<AArch64Subtarget>();
393 TII = Subtarget->getInstrInfo();
394
395 SmallVector<MachineBasicBlock::instr_iterator> PAuthPseudoInstrs;
396
397 bool Modified = false;
398
399 for (auto &MBB : MF) {
400 for (auto &MI : MBB) {
401 switch (MI.getOpcode()) {
402 default:
403 break;
404 case AArch64::PAUTH_PROLOGUE:
405 case AArch64::PAUTH_EPILOGUE:
406 PAuthPseudoInstrs.push_back(Elt: MI.getIterator());
407 break;
408 }
409 }
410 }
411
412 for (auto It : PAuthPseudoInstrs) {
413 switch (It->getOpcode()) {
414 case AArch64::PAUTH_PROLOGUE:
415 signLR(MF, MBBI: It);
416 break;
417 case AArch64::PAUTH_EPILOGUE:
418 authenticateLR(MF, MBBI: It);
419 break;
420 default:
421 llvm_unreachable("Unhandled opcode");
422 }
423 It->eraseFromParent();
424 Modified = true;
425 }
426
427 return Modified;
428}
429
430bool AArch64PointerAuthLegacy::runOnMachineFunction(MachineFunction &MF) {
431 return AArch64PointerAuthImpl().run(MF);
432}
433
434PreservedAnalyses
435AArch64PointerAuthPass::run(MachineFunction &MF,
436 MachineFunctionAnalysisManager &MFAM) {
437 const bool Changed = AArch64PointerAuthImpl().run(MF);
438 if (!Changed)
439 return PreservedAnalyses::all();
440 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
441 PA.preserveSet<CFGAnalyses>();
442 return PA;
443}
444