1//===-- AArch64PointerAuth.cpp -- Harden code using PAuth ------------------==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AArch64PointerAuth.h"
10
11#include "AArch64.h"
12#include "AArch64InstrInfo.h"
13#include "AArch64MachineFunctionInfo.h"
14#include "AArch64Subtarget.h"
15#include "Utils/AArch64BaseInfo.h"
16#include "llvm/CodeGen/MachineBasicBlock.h"
17#include "llvm/CodeGen/MachineInstrBuilder.h"
18#include "llvm/CodeGen/MachineModuleInfo.h"
19
20using namespace llvm;
21using namespace llvm::AArch64PAuth;
22
23#define AARCH64_POINTER_AUTH_NAME "AArch64 Pointer Authentication"
24
25namespace {
26
27class AArch64PointerAuth : public MachineFunctionPass {
28public:
29 static char ID;
30
31 AArch64PointerAuth() : MachineFunctionPass(ID) {}
32
33 bool runOnMachineFunction(MachineFunction &MF) override;
34
35 StringRef getPassName() const override { return AARCH64_POINTER_AUTH_NAME; }
36
37private:
38 /// An immediate operand passed to BRK instruction, if it is ever emitted.
39 static unsigned BrkOperandForKey(AArch64PACKey::ID KeyId) {
40 const unsigned BrkOperandBase = 0xc470;
41 return BrkOperandBase + KeyId;
42 }
43
44 const AArch64Subtarget *Subtarget = nullptr;
45 const AArch64InstrInfo *TII = nullptr;
46 const AArch64RegisterInfo *TRI = nullptr;
47
48 void signLR(MachineFunction &MF, MachineBasicBlock::iterator MBBI) const;
49
50 void authenticateLR(MachineFunction &MF,
51 MachineBasicBlock::iterator MBBI) const;
52
53 /// Stores blend(AddrDisc, IntDisc) to the Result register.
54 void emitBlend(MachineBasicBlock::iterator MBBI, Register Result,
55 Register AddrDisc, unsigned IntDisc) const;
56
57 /// Expands PAUTH_BLEND pseudo instruction.
58 void expandPAuthBlend(MachineBasicBlock::iterator MBBI) const;
59
60 bool checkAuthenticatedLR(MachineBasicBlock::iterator TI) const;
61};
62
63} // end anonymous namespace
64
65INITIALIZE_PASS(AArch64PointerAuth, "aarch64-ptrauth",
66 AARCH64_POINTER_AUTH_NAME, false, false)
67
68FunctionPass *llvm::createAArch64PointerAuthPass() {
69 return new AArch64PointerAuth();
70}
71
72char AArch64PointerAuth::ID = 0;
73
74// Where PAuthLR support is not known at compile time, it is supported using
75// PACM. PACM is in the hint space so has no effect when PAuthLR is not
76// supported by the hardware, but will alter the behaviour of PACI*SP, AUTI*SP
77// and RETAA/RETAB if the hardware supports PAuthLR.
78static void BuildPACM(const AArch64Subtarget &Subtarget, MachineBasicBlock &MBB,
79 MachineBasicBlock::iterator MBBI, DebugLoc DL,
80 MachineInstr::MIFlag Flags, MCSymbol *PACSym = nullptr) {
81 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
82 auto &MFnI = *MBB.getParent()->getInfo<AArch64FunctionInfo>();
83
84 // ADR X16,<address_of_PACIASP>
85 if (PACSym) {
86 assert(Flags == MachineInstr::FrameDestroy);
87 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADR))
88 .addReg(RegNo: AArch64::X16, flags: RegState::Define)
89 .addSym(Sym: PACSym);
90 }
91
92 // Only emit PACM if -mbranch-protection has +pc and the target does not
93 // have feature +pauth-lr.
94 if (MFnI.branchProtectionPAuthLR() && !Subtarget.hasPAuthLR())
95 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PACM)).setMIFlag(Flags);
96}
97
98void AArch64PointerAuth::signLR(MachineFunction &MF,
99 MachineBasicBlock::iterator MBBI) const {
100 auto &MFnI = *MF.getInfo<AArch64FunctionInfo>();
101 bool UseBKey = MFnI.shouldSignWithBKey();
102 bool EmitCFI = MFnI.needsDwarfUnwindInfo(MF);
103 bool EmitAsyncCFI = MFnI.needsAsyncDwarfUnwindInfo(MF);
104 bool NeedsWinCFI = MF.hasWinCFI();
105
106 MachineBasicBlock &MBB = *MBBI->getParent();
107
108 // Debug location must be unknown, see AArch64FrameLowering::emitPrologue.
109 DebugLoc DL;
110
111 if (UseBKey) {
112 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::EMITBKEY))
113 .setMIFlag(MachineInstr::FrameSetup);
114 }
115
116 // PAuthLR authentication instructions need to know the value of PC at the
117 // point of signing (PACI*).
118 if (MFnI.branchProtectionPAuthLR()) {
119 MCSymbol *PACSym = MF.getContext().createTempSymbol();
120 MFnI.setSigningInstrLabel(PACSym);
121 }
122
123 // No SEH opcode for this one; it doesn't materialize into an
124 // instruction on Windows.
125 if (MFnI.branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
126 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
127 MCID: TII->get(Opcode: MFnI.shouldSignWithBKey() ? AArch64::PACIBSPPC
128 : AArch64::PACIASPPC))
129 .setMIFlag(MachineInstr::FrameSetup)
130 ->setPreInstrSymbol(MF, Symbol: MFnI.getSigningInstrLabel());
131 } else {
132 BuildPACM(Subtarget: *Subtarget, MBB, MBBI, DL, Flags: MachineInstr::FrameSetup);
133 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
134 MCID: TII->get(Opcode: MFnI.shouldSignWithBKey() ? AArch64::PACIBSP
135 : AArch64::PACIASP))
136 .setMIFlag(MachineInstr::FrameSetup)
137 ->setPreInstrSymbol(MF, Symbol: MFnI.getSigningInstrLabel());
138 }
139
140 if (EmitCFI) {
141 if (!EmitAsyncCFI) {
142 // Reduce the size of the generated call frame information for synchronous
143 // CFI by bundling the new CFI instruction with others in the prolog, so
144 // that no additional DW_CFA_advance_loc is needed.
145 for (auto I = MBBI; I != MBB.end(); ++I) {
146 if (I->getOpcode() == TargetOpcode::CFI_INSTRUCTION &&
147 I->getFlag(Flag: MachineInstr::FrameSetup)) {
148 MBBI = I;
149 break;
150 }
151 }
152 }
153 unsigned CFIIndex =
154 MF.addFrameInst(Inst: MCCFIInstruction::createNegateRAState(L: nullptr));
155 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
156 .addCFIIndex(CFIIndex)
157 .setMIFlags(MachineInstr::FrameSetup);
158 } else if (NeedsWinCFI) {
159 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_PACSignLR))
160 .setMIFlag(MachineInstr::FrameSetup);
161 }
162}
163
164void AArch64PointerAuth::authenticateLR(
165 MachineFunction &MF, MachineBasicBlock::iterator MBBI) const {
166 const AArch64FunctionInfo *MFnI = MF.getInfo<AArch64FunctionInfo>();
167 bool UseBKey = MFnI->shouldSignWithBKey();
168 bool EmitAsyncCFI = MFnI->needsAsyncDwarfUnwindInfo(MF);
169 bool NeedsWinCFI = MF.hasWinCFI();
170
171 MachineBasicBlock &MBB = *MBBI->getParent();
172 DebugLoc DL = MBBI->getDebugLoc();
173 // MBBI points to a PAUTH_EPILOGUE instruction to be replaced and
174 // TI points to a terminator instruction that may or may not be combined.
175 // Note that inserting new instructions "before MBBI" and "before TI" is
176 // not the same because if ShadowCallStack is enabled, its instructions
177 // are placed between MBBI and TI.
178 MachineBasicBlock::iterator TI = MBB.getFirstInstrTerminator();
179
180 // The AUTIASP instruction assembles to a hint instruction before v8.3a so
181 // this instruction can safely used for any v8a architecture.
182 // From v8.3a onwards there are optimised authenticate LR and return
183 // instructions, namely RETA{A,B}, that can be used instead. In this case the
184 // DW_CFA_AARCH64_negate_ra_state can't be emitted.
185 bool TerminatorIsCombinable =
186 TI != MBB.end() && TI->getOpcode() == AArch64::RET;
187 MCSymbol *PACSym = MFnI->getSigningInstrLabel();
188
189 if (Subtarget->hasPAuth() && TerminatorIsCombinable && !NeedsWinCFI &&
190 !MF.getFunction().hasFnAttribute(Kind: Attribute::ShadowCallStack)) {
191 if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
192 assert(PACSym && "No PAC instruction to refer to");
193 BuildMI(BB&: MBB, I: TI, MIMD: DL,
194 MCID: TII->get(Opcode: UseBKey ? AArch64::RETABSPPCi : AArch64::RETAASPPCi))
195 .addSym(Sym: PACSym)
196 .copyImplicitOps(OtherMI: *MBBI)
197 .setMIFlag(MachineInstr::FrameDestroy);
198 } else {
199 BuildPACM(Subtarget: *Subtarget, MBB, MBBI: TI, DL, Flags: MachineInstr::FrameDestroy, PACSym);
200 BuildMI(BB&: MBB, I: TI, MIMD: DL, MCID: TII->get(Opcode: UseBKey ? AArch64::RETAB : AArch64::RETAA))
201 .copyImplicitOps(OtherMI: *MBBI)
202 .setMIFlag(MachineInstr::FrameDestroy);
203 }
204 MBB.erase(I: TI);
205 } else {
206 if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
207 assert(PACSym && "No PAC instruction to refer to");
208 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
209 MCID: TII->get(Opcode: UseBKey ? AArch64::AUTIBSPPCi : AArch64::AUTIASPPCi))
210 .addSym(Sym: PACSym)
211 .setMIFlag(MachineInstr::FrameDestroy);
212 } else {
213 BuildPACM(Subtarget: *Subtarget, MBB, MBBI, DL, Flags: MachineInstr::FrameDestroy, PACSym);
214 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
215 MCID: TII->get(Opcode: UseBKey ? AArch64::AUTIBSP : AArch64::AUTIASP))
216 .setMIFlag(MachineInstr::FrameDestroy);
217 }
218
219 if (EmitAsyncCFI) {
220 unsigned CFIIndex =
221 MF.addFrameInst(Inst: MCCFIInstruction::createNegateRAState(L: nullptr));
222 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
223 .addCFIIndex(CFIIndex)
224 .setMIFlags(MachineInstr::FrameDestroy);
225 }
226 if (NeedsWinCFI) {
227 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_PACSignLR))
228 .setMIFlag(MachineInstr::FrameDestroy);
229 }
230 }
231}
232
233namespace {
234
235// Mark dummy LDR instruction as volatile to prevent removing it as dead code.
236MachineMemOperand *createCheckMemOperand(MachineFunction &MF,
237 const AArch64Subtarget &Subtarget) {
238 MachinePointerInfo PointerInfo(Subtarget.getAddressCheckPSV());
239 auto MOVolatileLoad =
240 MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
241
242 return MF.getMachineMemOperand(PtrInfo: PointerInfo, F: MOVolatileLoad, Size: 4, BaseAlignment: Align(4));
243}
244
245} // namespace
246
247void llvm::AArch64PAuth::checkAuthenticatedRegister(
248 MachineBasicBlock::iterator MBBI, AuthCheckMethod Method,
249 Register AuthenticatedReg, Register TmpReg, bool UseIKey, unsigned BrkImm) {
250
251 MachineBasicBlock &MBB = *MBBI->getParent();
252 MachineFunction &MF = *MBB.getParent();
253 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
254 const AArch64InstrInfo *TII = Subtarget.getInstrInfo();
255 DebugLoc DL = MBBI->getDebugLoc();
256
257 // All terminator instructions should be grouped at the end of the machine
258 // basic block, with no non-terminator instructions between them. Depending on
259 // the method requested, we will insert some regular instructions, maybe
260 // followed by a conditional branch instruction, which is a terminator, before
261 // MBBI. Thus, MBBI is expected to be the first terminator of its MBB.
262 assert(MBBI->isTerminator() && MBBI == MBB.getFirstTerminator() &&
263 "MBBI should be the first terminator in MBB");
264
265 // First, handle the methods not requiring creating extra MBBs.
266 switch (Method) {
267 default:
268 break;
269 case AuthCheckMethod::None:
270 return;
271 case AuthCheckMethod::DummyLoad:
272 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::LDRWui), DestReg: getWRegFromXReg(Reg: TmpReg))
273 .addReg(RegNo: AuthenticatedReg)
274 .addImm(Val: 0)
275 .addMemOperand(MMO: createCheckMemOperand(MF, Subtarget));
276 return;
277 }
278
279 // Control flow has to be changed, so arrange new MBBs.
280
281 // The block that explicitly generates a break-point exception on failure.
282 MachineBasicBlock *BreakBlock =
283 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
284 MF.push_back(MBB: BreakBlock);
285 MBB.addSuccessor(Succ: BreakBlock);
286
287 BuildMI(BB: BreakBlock, MIMD: DL, MCID: TII->get(Opcode: AArch64::BRK)).addImm(Val: BrkImm);
288
289 switch (Method) {
290 case AuthCheckMethod::None:
291 case AuthCheckMethod::DummyLoad:
292 llvm_unreachable("Should be handled above");
293 case AuthCheckMethod::HighBitsNoTBI:
294 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::EORXrs), DestReg: TmpReg)
295 .addReg(RegNo: AuthenticatedReg)
296 .addReg(RegNo: AuthenticatedReg)
297 .addImm(Val: 1);
298 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::TBNZX))
299 .addReg(RegNo: TmpReg)
300 .addImm(Val: 62)
301 .addMBB(MBB: BreakBlock);
302 return;
303 case AuthCheckMethod::XPACHint:
304 assert(AuthenticatedReg == AArch64::LR &&
305 "XPACHint mode is only compatible with checking the LR register");
306 assert(UseIKey && "XPACHint mode is only compatible with I-keys");
307 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ORRXrs), DestReg: TmpReg)
308 .addReg(RegNo: AArch64::XZR)
309 .addReg(RegNo: AArch64::LR)
310 .addImm(Val: 0);
311 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::XPACLRI));
312 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SUBSXrs), DestReg: AArch64::XZR)
313 .addReg(RegNo: TmpReg)
314 .addReg(RegNo: AArch64::LR)
315 .addImm(Val: 0);
316 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::Bcc))
317 .addImm(Val: AArch64CC::NE)
318 .addMBB(MBB: BreakBlock);
319 return;
320 }
321 llvm_unreachable("Unknown AuthCheckMethod enum");
322}
323
324unsigned llvm::AArch64PAuth::getCheckerSizeInBytes(AuthCheckMethod Method) {
325 switch (Method) {
326 case AuthCheckMethod::None:
327 return 0;
328 case AuthCheckMethod::DummyLoad:
329 return 4;
330 case AuthCheckMethod::HighBitsNoTBI:
331 return 12;
332 case AuthCheckMethod::XPACHint:
333 return 20;
334 }
335 llvm_unreachable("Unknown AuthCheckMethod enum");
336}
337
338bool AArch64PointerAuth::checkAuthenticatedLR(
339 MachineBasicBlock::iterator TI) const {
340 const AArch64FunctionInfo *MFnI = TI->getMF()->getInfo<AArch64FunctionInfo>();
341 AArch64PACKey::ID KeyId =
342 MFnI->shouldSignWithBKey() ? AArch64PACKey::IB : AArch64PACKey::IA;
343
344 AuthCheckMethod Method =
345 Subtarget->getAuthenticatedLRCheckMethod(MF: *TI->getMF());
346
347 if (Method == AuthCheckMethod::None)
348 return false;
349
350 // FIXME If FEAT_FPAC is implemented by the CPU, this check can be skipped.
351
352 assert(!TI->getMF()->hasWinCFI() && "WinCFI is not yet supported");
353
354 // The following code may create a signing oracle:
355 //
356 // <authenticate LR>
357 // TCRETURN ; the callee may sign and spill the LR in its prologue
358 //
359 // To avoid generating a signing oracle, check the authenticated value
360 // before possibly re-signing it in the callee, as follows:
361 //
362 // <authenticate LR>
363 // <check if LR contains a valid address>
364 // b.<cond> break_block
365 // ret_block:
366 // TCRETURN
367 // break_block:
368 // brk <BrkOperand>
369 //
370 // or just
371 //
372 // <authenticate LR>
373 // ldr tmp, [lr]
374 // TCRETURN
375
376 // TmpReg is chosen assuming X16 and X17 are dead after TI.
377 assert(AArch64InstrInfo::isTailCallReturnInst(*TI) &&
378 "Tail call is expected");
379 Register TmpReg =
380 TI->readsRegister(Reg: AArch64::X16, TRI) ? AArch64::X17 : AArch64::X16;
381 assert(!TI->readsRegister(TmpReg, TRI) &&
382 "More than a single register is used by TCRETURN");
383
384 checkAuthenticatedRegister(MBBI: TI, Method, AuthenticatedReg: AArch64::LR, TmpReg, /*UseIKey=*/true,
385 BrkImm: BrkOperandForKey(KeyId));
386
387 return true;
388}
389
390void AArch64PointerAuth::emitBlend(MachineBasicBlock::iterator MBBI,
391 Register Result, Register AddrDisc,
392 unsigned IntDisc) const {
393 MachineBasicBlock &MBB = *MBBI->getParent();
394 DebugLoc DL = MBBI->getDebugLoc();
395
396 if (Result != AddrDisc)
397 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ORRXrs), DestReg: Result)
398 .addReg(RegNo: AArch64::XZR)
399 .addReg(RegNo: AddrDisc)
400 .addImm(Val: 0);
401
402 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::MOVKXi), DestReg: Result)
403 .addReg(RegNo: Result)
404 .addImm(Val: IntDisc)
405 .addImm(Val: 48);
406}
407
408void AArch64PointerAuth::expandPAuthBlend(
409 MachineBasicBlock::iterator MBBI) const {
410 Register ResultReg = MBBI->getOperand(i: 0).getReg();
411 Register AddrDisc = MBBI->getOperand(i: 1).getReg();
412 unsigned IntDisc = MBBI->getOperand(i: 2).getImm();
413 emitBlend(MBBI, Result: ResultReg, AddrDisc, IntDisc);
414}
415
416bool AArch64PointerAuth::runOnMachineFunction(MachineFunction &MF) {
417 const auto *MFnI = MF.getInfo<AArch64FunctionInfo>();
418
419 Subtarget = &MF.getSubtarget<AArch64Subtarget>();
420 TII = Subtarget->getInstrInfo();
421 TRI = Subtarget->getRegisterInfo();
422
423 SmallVector<MachineBasicBlock::instr_iterator> PAuthPseudoInstrs;
424 SmallVector<MachineBasicBlock::instr_iterator> TailCallInstrs;
425
426 bool Modified = false;
427 bool HasAuthenticationInstrs = false;
428
429 for (auto &MBB : MF) {
430 // Using instr_iterator to catch unsupported bundled TCRETURN* instructions
431 // instead of just skipping them.
432 for (auto &MI : MBB.instrs()) {
433 switch (MI.getOpcode()) {
434 default:
435 // Bundled TCRETURN* instructions (such as created by KCFI)
436 // are not supported yet, but no support is required if no
437 // PAUTH_EPILOGUE instructions exist in the same function.
438 // Skip the BUNDLE instruction itself (actual bundled instructions
439 // follow it in the instruction list).
440 if (MI.isBundle())
441 continue;
442 if (AArch64InstrInfo::isTailCallReturnInst(MI))
443 TailCallInstrs.push_back(Elt: MI.getIterator());
444 break;
445 case AArch64::PAUTH_PROLOGUE:
446 case AArch64::PAUTH_EPILOGUE:
447 case AArch64::PAUTH_BLEND:
448 assert(!MI.isBundled());
449 PAuthPseudoInstrs.push_back(Elt: MI.getIterator());
450 break;
451 }
452 }
453 }
454
455 for (auto It : PAuthPseudoInstrs) {
456 switch (It->getOpcode()) {
457 case AArch64::PAUTH_PROLOGUE:
458 signLR(MF, MBBI: It);
459 break;
460 case AArch64::PAUTH_EPILOGUE:
461 authenticateLR(MF, MBBI: It);
462 HasAuthenticationInstrs = true;
463 break;
464 case AArch64::PAUTH_BLEND:
465 expandPAuthBlend(MBBI: It);
466 break;
467 default:
468 llvm_unreachable("Unhandled opcode");
469 }
470 It->eraseFromParent();
471 Modified = true;
472 }
473
474 // FIXME Do we need to emit any PAuth-related epilogue code at all
475 // when SCS is enabled?
476 if (HasAuthenticationInstrs &&
477 !MFnI->needsShadowCallStackPrologueEpilogue(MF)) {
478 for (auto TailCall : TailCallInstrs) {
479 assert(!TailCall->isBundled() && "Not yet supported");
480 Modified |= checkAuthenticatedLR(TI: TailCall);
481 }
482 }
483
484 return Modified;
485}
486