| 1 | //===-- AArch64PointerAuth.cpp -- Harden code using PAuth ------------------==// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "AArch64PointerAuth.h" |
| 10 | |
| 11 | #include "AArch64.h" |
| 12 | #include "AArch64FrameLowering.h" |
| 13 | #include "AArch64InstrInfo.h" |
| 14 | #include "AArch64MachineFunctionInfo.h" |
| 15 | #include "AArch64RegisterInfo.h" |
| 16 | #include "AArch64Subtarget.h" |
| 17 | #include "llvm/CodeGen/CFIInstBuilder.h" |
| 18 | #include "llvm/CodeGen/MachineBasicBlock.h" |
| 19 | #include "llvm/CodeGen/MachineInstrBuilder.h" |
| 20 | #include "llvm/CodeGen/MachineModuleInfo.h" |
| 21 | #include "llvm/CodeGen/RegisterScavenging.h" |
| 22 | #include "llvm/IR/Module.h" |
| 23 | |
| 24 | using namespace llvm; |
| 25 | using namespace llvm::AArch64PAuth; |
| 26 | |
| 27 | #define AARCH64_POINTER_AUTH_NAME "AArch64 Pointer Authentication" |
| 28 | |
| 29 | namespace { |
| 30 | |
| 31 | class AArch64PointerAuthImpl { |
| 32 | public: |
| 33 | bool run(MachineFunction &MF); |
| 34 | |
| 35 | private: |
| 36 | const AArch64Subtarget *Subtarget = nullptr; |
| 37 | const AArch64InstrInfo *TII = nullptr; |
| 38 | |
| 39 | void signLR(MachineFunction &MF, MachineBasicBlock::iterator MBBI) const; |
| 40 | |
| 41 | void authenticateLR(MachineFunction &MF, |
| 42 | MachineBasicBlock::iterator MBBI) const; |
| 43 | |
| 44 | bool emitSignReturnAddressHardening(MachineFunction &MF); |
| 45 | }; |
| 46 | |
| 47 | class AArch64PointerAuthLegacy : public MachineFunctionPass { |
| 48 | public: |
| 49 | static char ID; |
| 50 | |
| 51 | AArch64PointerAuthLegacy() : MachineFunctionPass(ID) {} |
| 52 | |
| 53 | bool runOnMachineFunction(MachineFunction &MF) override; |
| 54 | |
| 55 | StringRef getPassName() const override { return AARCH64_POINTER_AUTH_NAME; } |
| 56 | }; |
| 57 | |
| 58 | } // end anonymous namespace |
| 59 | |
| 60 | INITIALIZE_PASS(AArch64PointerAuthLegacy, "aarch64-ptrauth" , |
| 61 | AARCH64_POINTER_AUTH_NAME, false, false) |
| 62 | |
| 63 | FunctionPass *llvm::createAArch64PointerAuthPass() { |
| 64 | return new AArch64PointerAuthLegacy(); |
| 65 | } |
| 66 | |
| 67 | char AArch64PointerAuthLegacy::ID = 0; |
| 68 | |
| 69 | static void emitEpiloguePACSymOffsetIntoReg(const TargetInstrInfo &TII, |
| 70 | MachineBasicBlock &MBB, |
| 71 | MachineBasicBlock::iterator I, |
| 72 | DebugLoc DL, MCSymbol *PACSym, |
| 73 | Register Reg) { |
| 74 | BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AArch64::ADRP), DestReg: Reg) |
| 75 | .addSym(Sym: PACSym, TargetFlags: AArch64II::MO_PAGE) |
| 76 | .setMIFlag(MachineInstr::FrameDestroy); |
| 77 | BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AArch64::ADDXri), DestReg: Reg) |
| 78 | .addReg(RegNo: Reg) |
| 79 | .addSym(Sym: PACSym, TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC) |
| 80 | .addImm(Val: 0) |
| 81 | .setMIFlag(MachineInstr::FrameDestroy); |
| 82 | } |
| 83 | |
| 84 | // Wrap a given PAC instruction in CFI that describes it. |
| 85 | // |
| 86 | // Depending on the type of CFI required, we may need to emit the directive |
| 87 | // either before or after the instruction, so that unwinders can correctly |
| 88 | // interpret the location of the signing instruction. |
| 89 | // |
| 90 | // As a general rule, CFI opcodes describe the actions needed to recover the |
| 91 | // register state leading up to a not-yet-retired instruction, with one |
| 92 | // exception: .cfi_negate_ra_state_with_pc always comes before the paci[ab]sppc, |
| 93 | // since the unwinder uses the location of the CFI itself to derive the address |
| 94 | // of the signing instruction [1]. |
| 95 | // 1: https://github.com/llvm/llvm-project/pull/137795#issuecomment-2838779129 |
| 96 | template <typename BuildPACMIFn> |
| 97 | static void decoratePACWithCFI(MachineBasicBlock &MBB, |
| 98 | MachineBasicBlock::iterator MBBI, bool EmitCFI, |
| 99 | BuildPACMIFn BuildPACMI) { |
| 100 | if (!EmitCFI) { |
| 101 | BuildPACMI(); |
| 102 | return; |
| 103 | } |
| 104 | |
| 105 | auto &MF = *MBB.getParent(); |
| 106 | auto &MFnI = *MF.getInfo<AArch64FunctionInfo>(); |
| 107 | CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup); |
| 108 | const Triple &TT = MF.getFunction().getParent()->getTargetTriple(); |
| 109 | SetRAStateMode Mode = MF.getSubtarget<AArch64Subtarget>() |
| 110 | .getCLOpts() |
| 111 | .cfi_llvm_set_ra_sign_state; |
| 112 | |
| 113 | if (MFnI.branchProtectionPAuthLR()) { |
| 114 | switch (Mode) { |
| 115 | case SetRAStateMode::Never: |
| 116 | CFIBuilder.buildNegateRAStateWithPC(); |
| 117 | BuildPACMI(); |
| 118 | break; |
| 119 | case SetRAStateMode::PAuthLR: |
| 120 | case SetRAStateMode::Always: { |
| 121 | BuildPACMI(); |
| 122 | MCSymbol *PACSym = MFnI.getSigningInstrLabel(); |
| 123 | assert(PACSym && "No PAC instruction to refer to" ); |
| 124 | CFIBuilder.buildSetRAState(State: 2, PACSym); |
| 125 | break; |
| 126 | } |
| 127 | } |
| 128 | } else { |
| 129 | switch (Mode) { |
| 130 | case SetRAStateMode::Never: |
| 131 | case SetRAStateMode::PAuthLR: |
| 132 | BuildPACMI(); |
| 133 | if (!TT.isOSBinFormatMachO()) { |
| 134 | CFIBuilder.buildNegateRAState(); |
| 135 | } |
| 136 | break; |
| 137 | case SetRAStateMode::Always: |
| 138 | BuildPACMI(); |
| 139 | CFIBuilder.buildSetRAState(State: 1, PACSym: nullptr); |
| 140 | break; |
| 141 | } |
| 142 | } |
| 143 | } |
| 144 | |
| 145 | static void emitAUTCFI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, |
| 146 | bool EmitCFI) { |
| 147 | if (!EmitCFI) |
| 148 | return; |
| 149 | |
| 150 | auto &MF = *MBB.getParent(); |
| 151 | auto &MFnI = *MF.getInfo<AArch64FunctionInfo>(); |
| 152 | CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameDestroy); |
| 153 | const Triple &TT = MF.getFunction().getParent()->getTargetTriple(); |
| 154 | SetRAStateMode Mode = MF.getSubtarget<AArch64Subtarget>() |
| 155 | .getCLOpts() |
| 156 | .cfi_llvm_set_ra_sign_state; |
| 157 | |
| 158 | if (MFnI.branchProtectionPAuthLR()) { |
| 159 | switch (Mode) { |
| 160 | case SetRAStateMode::Never: |
| 161 | // DW_CFA_AARCH64_negate_ra_state_with_pc is semantically broken for |
| 162 | // functions where shrinkwrapping places signing/authenticating pairs on |
| 163 | // distinct CFG paths. |
| 164 | // |
| 165 | // DWARF CFI is evaluated linearly over the byte stream, not along control |
| 166 | // flow edges. The toggle semantics of this directive therefore cannot |
| 167 | // faithfully represent the signed/unsigned RA state for all possible CFG |
| 168 | // paths. The added complexity versus DW_CFA_AARCH64_negate_ra_state is |
| 169 | // that an unwinder must also reconstruct the PC of the PACI[AB]SPPC in |
| 170 | // order to verify the signed LR, and that address is derived from the |
| 171 | // location of this directive in the linear CFI stream. |
| 172 | // |
| 173 | // The correct fix is to use DW_CFA_AARCH64_set_ra_state_with_pc, which |
| 174 | // sets the RA state and signing address absolutely rather than toggling |
| 175 | // them. An unwinder that supports this directive can reconstruct the |
| 176 | // correct state on any CFG path, regardless of how many |
| 177 | // signing/authenticating pairs exist in the function. However, not all |
| 178 | // unwinders support this directive, so we cannot rely on it exclusively. |
| 179 | // |
| 180 | // For unwinders that only support DW_CFA_AARCH64_negate_ra_state_with_pc, |
| 181 | // libunwind exploits a loophole: it records the address at the |
| 182 | // DW_CFA_AARCH64_negate_ra_state_with_pc site to authenticate the LR, but |
| 183 | // does not care that the CFI state remains "signed with pc" after |
| 184 | // authentication has occurred. This means we can safely omit the |
| 185 | // FrameDestroy emission of this directive, treating it solely as a marker |
| 186 | // for the signing site, as long as each function has at most one such |
| 187 | // signing location. That invariant holds today because shrinkwrapping |
| 188 | // does not yet hoist or sink PAuth_LR frame code across CFG join/split |
| 189 | // points; once it does, we must avoid those transformations on platforms |
| 190 | // that have this limitation. |
| 191 | // |
| 192 | // https://github.com/ARM-software/abi-aa/issues/327 |
| 193 | // https://github.com/ARM-software/abi-aa/pull/346 |
| 194 | break; |
| 195 | case SetRAStateMode::PAuthLR: |
| 196 | case SetRAStateMode::Always: |
| 197 | CFIBuilder.buildSetRAState(State: 0, PACSym: nullptr); |
| 198 | break; |
| 199 | } |
| 200 | } else if (!TT.isOSBinFormatMachO()) { |
| 201 | switch (Mode) { |
| 202 | case SetRAStateMode::Never: |
| 203 | case SetRAStateMode::PAuthLR: |
| 204 | CFIBuilder.buildNegateRAState(); |
| 205 | break; |
| 206 | case SetRAStateMode::Always: |
| 207 | CFIBuilder.buildSetRAState(State: 0, PACSym: nullptr); |
| 208 | break; |
| 209 | } |
| 210 | } |
| 211 | } |
| 212 | |
| 213 | static inline void emitMOVWithFrameDestroy(MachineBasicBlock &MBB, |
| 214 | MachineBasicBlock::iterator &MBBI, |
| 215 | DebugLoc DL, |
| 216 | const AArch64InstrInfo *TII, |
| 217 | Register Dst, Register Src) { |
| 218 | assert(&MBB == MBBI->getParent()); |
| 219 | BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ORRXrs), DestReg: Dst) |
| 220 | .addReg(RegNo: AArch64::XZR) |
| 221 | .addReg(RegNo: Src) |
| 222 | .addImm(Val: 0) |
| 223 | .setMIFlag(MachineInstr::FrameDestroy); |
| 224 | } |
| 225 | |
| 226 | void AArch64PointerAuthImpl::signLR(MachineFunction &MF, |
| 227 | MachineBasicBlock::iterator MBBI) const { |
| 228 | auto &MFnI = *MF.getInfo<AArch64FunctionInfo>(); |
| 229 | bool UseBKey = MFnI.shouldSignWithBKey(); |
| 230 | bool EmitCFI = MFnI.needsDwarfUnwindInfo(MF); |
| 231 | bool NeedsWinCFI = MF.hasWinCFI(); |
| 232 | |
| 233 | MachineBasicBlock &MBB = *MBBI->getParent(); |
| 234 | |
| 235 | // Debug location must be unknown, see AArch64FrameLowering::emitPrologue. |
| 236 | DebugLoc DL; |
| 237 | |
| 238 | if (UseBKey && !MF.getTarget().getTargetTriple().isOSBinFormatMachO()) { |
| 239 | BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::EMITBKEY)) |
| 240 | .setMIFlag(MachineInstr::FrameSetup); |
| 241 | } |
| 242 | |
| 243 | // PAuthLR authentication instructions need to know the value of PC at the |
| 244 | // point of signing (PACI*). |
| 245 | if (MFnI.branchProtectionPAuthLR()) { |
| 246 | MCSymbol *PACSym = MF.getContext().createTempSymbol(); |
| 247 | MFnI.setSigningInstrLabel(PACSym); |
| 248 | } |
| 249 | |
| 250 | // No SEH opcode for this one; it doesn't materialize into an |
| 251 | // instruction on Windows. |
| 252 | if (MFnI.branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) { |
| 253 | if (!MF.getTarget().getTargetTriple().isOSBinFormatMachO()) |
| 254 | BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::EMITNONHINTED)) |
| 255 | .setMIFlag(MachineInstr::FrameSetup); |
| 256 | |
| 257 | decoratePACWithCFI(MBB, MBBI, EmitCFI, BuildPACMI: [&]() { |
| 258 | BuildMI(BB&: MBB, I: MBBI, MIMD: DL, |
| 259 | MCID: TII->get(Opcode: UseBKey ? AArch64::PACIBSPPC : AArch64::PACIASPPC)) |
| 260 | .setMIFlag(MachineInstr::FrameSetup) |
| 261 | ->setPreInstrSymbol(MF, Symbol: MFnI.getSigningInstrLabel()); |
| 262 | }); |
| 263 | } else { |
| 264 | if (MFnI.branchProtectionPAuthLR()) { |
| 265 | BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PACM)) |
| 266 | .setMIFlag(MachineInstr::FrameSetup); |
| 267 | } |
| 268 | decoratePACWithCFI(MBB, MBBI, EmitCFI, BuildPACMI: [&]() { |
| 269 | BuildMI(BB&: MBB, I: MBBI, MIMD: DL, |
| 270 | MCID: TII->get(Opcode: UseBKey ? AArch64::PACIBSP : AArch64::PACIASP)) |
| 271 | .setMIFlag(MachineInstr::FrameSetup) |
| 272 | ->setPreInstrSymbol(MF, Symbol: MFnI.getSigningInstrLabel()); |
| 273 | }); |
| 274 | } |
| 275 | |
| 276 | if (!EmitCFI && NeedsWinCFI) { |
| 277 | assert(UseBKey && |
| 278 | "Windows SEH PAC unwind info only supports B-key signing" ); |
| 279 | BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_PACSignLR)) |
| 280 | .setMIFlag(MachineInstr::FrameSetup); |
| 281 | } |
| 282 | } |
| 283 | |
| 284 | void AArch64PointerAuthImpl::authenticateLR( |
| 285 | MachineFunction &MF, MachineBasicBlock::iterator MBBI) const { |
| 286 | const AArch64FunctionInfo *MFnI = MF.getInfo<AArch64FunctionInfo>(); |
| 287 | bool UseBKey = MFnI->shouldSignWithBKey(); |
| 288 | bool EmitAsyncCFI = MFnI->needsAsyncDwarfUnwindInfo(MF); |
| 289 | bool NeedsWinCFI = MF.hasWinCFI(); |
| 290 | |
| 291 | MachineBasicBlock &MBB = *MBBI->getParent(); |
| 292 | DebugLoc DL = MBBI->getDebugLoc(); |
| 293 | // MBBI points to a PAUTH_EPILOGUE instruction to be replaced and |
| 294 | // TI points to a terminator instruction that may or may not be combined. |
| 295 | // Note that inserting new instructions "before MBBI" and "before TI" is |
| 296 | // not the same because if ShadowCallStack is enabled, its instructions |
| 297 | // are placed between MBBI and TI. |
| 298 | MachineBasicBlock::iterator TI = MBB.getFirstInstrTerminator(); |
| 299 | |
| 300 | MCSymbol *PACSym = MFnI->getSigningInstrLabel(); |
| 301 | auto &AFL = *static_cast<const AArch64FrameLowering *>( |
| 302 | MF.getSubtarget().getFrameLowering()); |
| 303 | int64_t ArgumentStackToRestore = AFL.getArgumentStackToRestore(MF, MBB); |
| 304 | |
| 305 | // The signing SP is already in X16 if we are before the epilogue proper. |
| 306 | bool EntrySPInX16 = MBBI->getOpcode() == AArch64::PAUTH_EPILOGUE_ENTRY_SP; |
| 307 | assert((!EntrySPInX16 || ArgumentStackToRestore != 0) && |
| 308 | "Early authentication is only for epilogues that change SP" ); |
| 309 | |
| 310 | // The AUTIASP instruction assembles to a hint instruction before v8.3a so |
| 311 | // this instruction can safely be used for any v8a architecture. |
| 312 | // From v8.3a onwards there are optimised authenticate LR and return |
| 313 | // instructions, namely RETA{A,B}, that can be used instead. In this case |
| 314 | // the DW_CFA_AARCH64_negate_ra_state can't be emitted. Additionally, |
| 315 | // RET{A,B} requires the SP to match its incoming value on entry to the |
| 316 | // function. |
| 317 | // |
| 318 | // If the PAC-RET hardening based on load from the return address is |
| 319 | // enabled, fallback to the use of AUTIASP/AUTIBSP and RET. |
| 320 | bool TerminatorIsCombinable = std::next(x: MBBI) == TI && TI != MBB.end() && |
| 321 | TI->getOpcode() == AArch64::RET && |
| 322 | ArgumentStackToRestore == 0 && |
| 323 | !MFnI->shouldHardenSignReturnAddress(); |
| 324 | |
| 325 | if (Subtarget->hasPAuth() && TerminatorIsCombinable && !NeedsWinCFI && |
| 326 | !MF.getFunction().hasFnAttribute(Kind: Attribute::ShadowCallStack)) { |
| 327 | if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) { |
| 328 | assert(PACSym && "No PAC instruction to refer to" ); |
| 329 | BuildMI(BB&: MBB, I: TI, MIMD: DL, |
| 330 | MCID: TII->get(Opcode: UseBKey ? AArch64::RETABSPPCi : AArch64::RETAASPPCi)) |
| 331 | .addSym(Sym: PACSym) |
| 332 | .copyImplicitOps(OtherMI: *MBBI) |
| 333 | .setMIFlag(MachineInstr::FrameDestroy); |
| 334 | } else { |
| 335 | if (MFnI->branchProtectionPAuthLR()) { |
| 336 | emitEpiloguePACSymOffsetIntoReg(TII: *TII, MBB, I: MBBI, DL, PACSym, |
| 337 | Reg: AArch64::X16); |
| 338 | BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PACM)) |
| 339 | .setMIFlag(MachineInstr::FrameDestroy); |
| 340 | } |
| 341 | BuildMI(BB&: MBB, I: TI, MIMD: DL, MCID: TII->get(Opcode: UseBKey ? AArch64::RETAB : AArch64::RETAA)) |
| 342 | .copyImplicitOps(OtherMI: *MBBI) |
| 343 | .setMIFlag(MachineInstr::FrameDestroy); |
| 344 | } |
| 345 | MBB.erase(I: TI); |
| 346 | return; |
| 347 | } |
| 348 | |
| 349 | // If PAUTH_EPILOGUE is at insertion point with a net zero offset on SP, we |
| 350 | // can use an AUT form with a hardcoded SP discriminator. |
| 351 | if (ArgumentStackToRestore == 0) { |
| 352 | if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) { |
| 353 | assert(PACSym && "No PAC instruction to refer to" ); |
| 354 | BuildMI(BB&: MBB, I: MBBI, MIMD: DL, |
| 355 | MCID: TII->get(Opcode: UseBKey ? AArch64::AUTIBSPPCi : AArch64::AUTIASPPCi)) |
| 356 | .addSym(Sym: PACSym) |
| 357 | .setMIFlag(MachineInstr::FrameDestroy); |
| 358 | emitAUTCFI(MBB, MBBI, EmitCFI: EmitAsyncCFI); |
| 359 | } else { |
| 360 | if (MFnI->branchProtectionPAuthLR()) { |
| 361 | emitEpiloguePACSymOffsetIntoReg(TII: *TII, MBB, I: MBBI, DL, PACSym, |
| 362 | Reg: AArch64::X16); |
| 363 | |
| 364 | BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PACM)) |
| 365 | .setMIFlag(MachineInstr::FrameDestroy); |
| 366 | } |
| 367 | BuildMI(BB&: MBB, I: MBBI, MIMD: DL, |
| 368 | MCID: TII->get(Opcode: UseBKey ? AArch64::AUTIBSP : AArch64::AUTIASP)) |
| 369 | .setMIFlag(MachineInstr::FrameDestroy); |
| 370 | emitAUTCFI(MBB, MBBI, EmitCFI: EmitAsyncCFI); |
| 371 | } |
| 372 | |
| 373 | if (NeedsWinCFI) { |
| 374 | assert(UseBKey && |
| 375 | "Windows SEH PAC unwind info only supports B-key signing" ); |
| 376 | BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_PACSignLR)) |
| 377 | .setMIFlag(MachineInstr::FrameDestroy); |
| 378 | } |
| 379 | |
| 380 | return; |
| 381 | } |
| 382 | |
| 383 | // Windows unwind info describes the authentication as a single pac_sign_lr |
| 384 | // unwind code, which the unwinder applies with the SP it has at that point. |
| 385 | // That is the signing SP only if the epilogue leaves SP at its entry value, |
| 386 | // and a single code can't stand in for the multi-instruction sequence below. |
| 387 | // So there, the epilogue emitter authenticates before the epilogue starts, |
| 388 | // where the prologue's unwind codes still apply, and the epilogue's unwind |
| 389 | // codes don't mention pac_sign_lr. |
| 390 | if (NeedsWinCFI && !EntrySPInX16) |
| 391 | report_fatal_error(reason: "Can't handle a tail call that changes the stack " |
| 392 | "argument size in a function that signs its return " |
| 393 | "address on Windows" ); |
| 394 | |
| 395 | // When ArgumentStackToRestore > 0, this function received more argument |
| 396 | // space than the tail callee pops. The epilogue contains an SP adjustment |
| 397 | // (e.g. "add sp, sp, #N") to discard the leftover argument space. |
| 398 | // |
| 399 | // When ArgumentStackToRestore < 0, the tail callee pops more argument space |
| 400 | // than this function received, so after the frame teardown, SP is below the |
| 401 | // entry SP used as the signing modifier. |
| 402 | // |
| 403 | // We cannot simply bump SP first and then use AUTI[AB]SP with the bumped |
| 404 | // value, because the live arguments would fall below SP and potentially |
| 405 | // outside the red-zone. |
| 406 | // |
| 407 | // At this point there is an offset to the incoming SP, and we can't use the |
| 408 | // aut variants that hard-code SP. Reconstruct entry SP in x16 and |
| 409 | // authenticate using AUTI[AB]1716 (x17=LR, x16=entry_SP). |
| 410 | if (!EntrySPInX16) |
| 411 | emitFrameOffset(MBB, MBBI, DL, DestReg: AArch64::X16, SrcReg: AArch64::SP, |
| 412 | Offset: StackOffset::getFixed(Fixed: -ArgumentStackToRestore), TII, |
| 413 | MachineInstr::FrameDestroy); |
| 414 | |
| 415 | // At this point, if EntrySPInX16 is true, it means we're using |
| 416 | // PAUTH_EPILOGUE_ENTRY_SP, so the signed return address will already be in |
| 417 | // x17 if the sequence below works on that, as the epilogue emitter works out |
| 418 | // using AArch64PAuth::authenticatesLRViaX17, and in LR if it doesn't. Each |
| 419 | // of the sequences below asserts that it agrees. |
| 420 | [[maybe_unused]] const bool ViaX17 = AArch64PAuth::authenticatesLRViaX17( |
| 421 | BranchProtectionPAuthLR: MFnI->branchProtectionPAuthLR(), HasPAuth: Subtarget->hasPAuth()); |
| 422 | |
| 423 | if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) { |
| 424 | assert(ViaX17 && "Expected to authenticate via X17" ); |
| 425 | if (!EntrySPInX16) |
| 426 | emitMOVWithFrameDestroy(MBB, MBBI, DL, TII, Dst: AArch64::X17, Src: AArch64::LR); |
| 427 | |
| 428 | assert(PACSym && "No PAC instruction to refer to" ); |
| 429 | emitEpiloguePACSymOffsetIntoReg(TII: *TII, MBB, I: MBBI, DL, PACSym, Reg: AArch64::X15); |
| 430 | |
| 431 | unsigned AutOpc = UseBKey ? AArch64::AUTIB171615 : AArch64::AUTIA171615; |
| 432 | BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AutOpc)) |
| 433 | .setMIFlag(MachineInstr::FrameDestroy); |
| 434 | emitAUTCFI(MBB, MBBI, EmitCFI: EmitAsyncCFI); |
| 435 | |
| 436 | emitMOVWithFrameDestroy(MBB, MBBI, DL, TII, Dst: AArch64::LR, Src: AArch64::X17); |
| 437 | } else if (MFnI->branchProtectionPAuthLR()) { |
| 438 | assert(ViaX17 && "Expected to authenticate via X17" ); |
| 439 | if (!EntrySPInX16) |
| 440 | emitMOVWithFrameDestroy(MBB, MBBI, DL, TII, Dst: AArch64::X17, Src: AArch64::LR); |
| 441 | |
| 442 | assert(PACSym && "No PAC instruction to refer to" ); |
| 443 | emitEpiloguePACSymOffsetIntoReg(TII: *TII, MBB, I: MBBI, DL, PACSym, Reg: AArch64::X15); |
| 444 | |
| 445 | // The PACM hint-space instruction modifies the following AUTI[AB]1716 |
| 446 | // to optionally take x15 as an extra operand depending on the |
| 447 | // presence of +pauth-lr at runtime. On machines without +pauth-lr, it |
| 448 | // behaves as a nop, and the address of the PACI[AB]SP in x15 is |
| 449 | // ignored. |
| 450 | BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PACM)) |
| 451 | .setMIFlag(MachineInstr::FrameDestroy); |
| 452 | |
| 453 | unsigned AutOpc = UseBKey ? AArch64::AUTIB1716 : AArch64::AUTIA1716; |
| 454 | BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AutOpc)) |
| 455 | .setMIFlag(MachineInstr::FrameDestroy); |
| 456 | emitAUTCFI(MBB, MBBI, EmitCFI: EmitAsyncCFI); |
| 457 | |
| 458 | emitMOVWithFrameDestroy(MBB, MBBI, DL, TII, Dst: AArch64::LR, Src: AArch64::X17); |
| 459 | } else if (Subtarget->hasPAuth()) { |
| 460 | assert(!ViaX17 && "Expected to authenticate using LR directly" ); |
| 461 | BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: UseBKey ? AArch64::AUTIB : AArch64::AUTIA), |
| 462 | DestReg: AArch64::LR) |
| 463 | .addUse(RegNo: AArch64::LR) |
| 464 | .addUse(RegNo: AArch64::X16) |
| 465 | .setMIFlag(MachineInstr::FrameDestroy); |
| 466 | emitAUTCFI(MBB, MBBI, EmitCFI: EmitAsyncCFI); |
| 467 | } else { |
| 468 | assert(ViaX17 && "Expected to authenticate via X17" ); |
| 469 | if (!EntrySPInX16) |
| 470 | emitMOVWithFrameDestroy(MBB, MBBI, DL, TII, Dst: AArch64::X17, Src: AArch64::LR); |
| 471 | |
| 472 | unsigned AutOpc = UseBKey ? AArch64::AUTIB1716 : AArch64::AUTIA1716; |
| 473 | BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AutOpc)) |
| 474 | .setMIFlag(MachineInstr::FrameDestroy); |
| 475 | emitAUTCFI(MBB, MBBI, EmitCFI: EmitAsyncCFI); |
| 476 | |
| 477 | emitMOVWithFrameDestroy(MBB, MBBI, DL, TII, Dst: AArch64::LR, Src: AArch64::X17); |
| 478 | } |
| 479 | |
| 480 | if (NeedsWinCFI && !EntrySPInX16) { |
| 481 | assert(UseBKey && |
| 482 | "Windows SEH PAC unwind info only supports B-key signing" ); |
| 483 | BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_PACSignLR)) |
| 484 | .setMIFlag(MachineInstr::FrameDestroy); |
| 485 | } |
| 486 | } |
| 487 | |
| 488 | unsigned llvm::AArch64PAuth::getCheckerSizeInBytes(AuthCheckMethod Method) { |
| 489 | switch (Method) { |
| 490 | case AuthCheckMethod::None: |
| 491 | return 0; |
| 492 | case AuthCheckMethod::DummyLoad: |
| 493 | return 4; |
| 494 | case AuthCheckMethod::HighBitsNoTBI: |
| 495 | return 12; |
| 496 | case AuthCheckMethod::XPACHint: |
| 497 | case AuthCheckMethod::XPAC: |
| 498 | return 20; |
| 499 | } |
| 500 | llvm_unreachable("Unknown AuthCheckMethod enum" ); |
| 501 | } |
| 502 | |
| 503 | bool AArch64PointerAuthImpl::run(MachineFunction &MF) { |
| 504 | Subtarget = &MF.getSubtarget<AArch64Subtarget>(); |
| 505 | TII = Subtarget->getInstrInfo(); |
| 506 | |
| 507 | SmallVector<MachineBasicBlock::instr_iterator> PAuthPseudoInstrs; |
| 508 | |
| 509 | bool Modified = false; |
| 510 | |
| 511 | for (auto &MBB : MF) { |
| 512 | for (auto &MI : MBB) { |
| 513 | switch (MI.getOpcode()) { |
| 514 | default: |
| 515 | break; |
| 516 | case AArch64::PAUTH_PROLOGUE: |
| 517 | case AArch64::PAUTH_EPILOGUE: |
| 518 | case AArch64::PAUTH_EPILOGUE_ENTRY_SP: |
| 519 | PAuthPseudoInstrs.push_back(Elt: MI.getIterator()); |
| 520 | break; |
| 521 | } |
| 522 | } |
| 523 | } |
| 524 | |
| 525 | for (auto It : PAuthPseudoInstrs) { |
| 526 | switch (It->getOpcode()) { |
| 527 | case AArch64::PAUTH_PROLOGUE: |
| 528 | signLR(MF, MBBI: It); |
| 529 | break; |
| 530 | case AArch64::PAUTH_EPILOGUE: |
| 531 | case AArch64::PAUTH_EPILOGUE_ENTRY_SP: |
| 532 | authenticateLR(MF, MBBI: It); |
| 533 | break; |
| 534 | default: |
| 535 | llvm_unreachable("Unhandled opcode" ); |
| 536 | } |
| 537 | It->eraseFromParent(); |
| 538 | Modified = true; |
| 539 | } |
| 540 | |
| 541 | Modified |= emitSignReturnAddressHardening(MF); |
| 542 | |
| 543 | return Modified; |
| 544 | } |
| 545 | |
| 546 | bool AArch64PointerAuthImpl::emitSignReturnAddressHardening( |
| 547 | MachineFunction &MF) { |
| 548 | const auto *FI = MF.getInfo<AArch64FunctionInfo>(); |
| 549 | assert(FI && "FI can't be null" ); |
| 550 | if (!FI->shouldSignReturnAddress(MF) || !FI->shouldHardenSignReturnAddress()) |
| 551 | return false; |
| 552 | assert(Subtarget && "Subtarget must be initialized" ); |
| 553 | |
| 554 | RegScavenger RS; |
| 555 | bool Modified = false; |
| 556 | for (MachineBasicBlock &MBB : MF) { |
| 557 | MachineBasicBlock::iterator RetInstIter = MBB.getFirstTerminator(); |
| 558 | |
| 559 | if (RetInstIter == MBB.end() || RetInstIter->getOpcode() != AArch64::RET) |
| 560 | continue; |
| 561 | |
| 562 | assert(RetInstIter->getOperand(0).getReg() == AArch64::LR && |
| 563 | "Return instruction must be returning via LR" ); |
| 564 | |
| 565 | MachineBasicBlock::iterator InsertionPoint = RetInstIter; |
| 566 | // In the case of Windows SEH, the hardening sequence does not immediately |
| 567 | // precede the return instruction. Instead, it precedes the SEH_EpilogEnd |
| 568 | // pseudo-instruction, which itself is expected to be the predecessor of |
| 569 | // the return. Plus, each instruction in the sequence needs one SEH_Nop. |
| 570 | const bool NeedsWinCFI = MF.hasWinCFI(); |
| 571 | if (NeedsWinCFI) { |
| 572 | --InsertionPoint; |
| 573 | assert(InsertionPoint->getOpcode() == AArch64::SEH_EpilogEnd); |
| 574 | } |
| 575 | DebugLoc DL = InsertionPoint->getDebugLoc(); |
| 576 | const auto EmitSEHNopIfRequired = [&]() { |
| 577 | if (NeedsWinCFI) |
| 578 | BuildMI(BB&: MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_Nop)) |
| 579 | .setMIFlag(MachineInstr::FrameDestroy); |
| 580 | }; |
| 581 | |
| 582 | RS.enterBasicBlockEnd(MBB); |
| 583 | Register XReg = RS.scavengeRegisterBackwards( |
| 584 | RC: AArch64::GPR64RegClass, To: InsertionPoint, |
| 585 | /*RestoreAfter=*/false, /*SPAdj=*/0, /*AllowSpill=*/false); |
| 586 | if (XReg == AArch64::NoRegister) { |
| 587 | // Couldn't find a free register to use for the hardening. Skip. |
| 588 | MF.getContext().reportWarning( |
| 589 | L: SMLoc(), Msg: "harden-pac-ret failed for function " + MF.getName()); |
| 590 | continue; |
| 591 | } |
| 592 | |
| 593 | // Register copies are done using ORRXrs directly instead of using the |
| 594 | // pseudo-instruction COPY because this function can be called after |
| 595 | // pseudo-instruction expansion takes place, for example via the machine |
| 596 | // outliner pass. |
| 597 | emitMOVWithFrameDestroy(MBB, MBBI&: InsertionPoint, DL, TII, Dst: XReg, Src: AArch64::LR); |
| 598 | EmitSEHNopIfRequired(); |
| 599 | |
| 600 | // The XPACI instruction is only available with FEAT_PAUTH. So if the |
| 601 | // subtarget does not have it, the alternative XPACLRI instruction must be |
| 602 | // used instead. The latter is in hint space, therefore can be used even |
| 603 | // if FEAT_PAUTH is absent. |
| 604 | if (Subtarget->hasPAuth()) { |
| 605 | BuildMI(BB&: MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: AArch64::XPACI), DestReg: XReg) |
| 606 | .addUse(RegNo: XReg) |
| 607 | .setMIFlag(MachineInstr::FrameDestroy); |
| 608 | EmitSEHNopIfRequired(); |
| 609 | Register WReg = |
| 610 | Subtarget->getRegisterInfo()->getSubReg(Reg: XReg, Idx: AArch64::sub_32); |
| 611 | BuildMI(BB&: MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: AArch64::LDRWui), DestReg: WReg) |
| 612 | .addUse(RegNo: XReg) |
| 613 | .addImm(Val: 0) |
| 614 | .addMemOperand(MMO: MF.getMachineMemOperand( |
| 615 | PtrInfo: MachinePointerInfo(), |
| 616 | F: MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile, Size: 4, |
| 617 | BaseAlignment: Align(4))) |
| 618 | .setMIFlag(MachineInstr::FrameDestroy); |
| 619 | EmitSEHNopIfRequired(); |
| 620 | } else { |
| 621 | // Emit a CFI directive to tell unwinders that the return address is now |
| 622 | // saved in XReg. |
| 623 | CFIInstBuilder(MBB, InsertionPoint, MachineInstr::FrameDestroy) |
| 624 | .buildRegister(Reg1: AArch64::LR, Reg2: XReg); |
| 625 | BuildMI(BB&: MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: AArch64::XPACLRI)) |
| 626 | .setMIFlag(MachineInstr::FrameDestroy); |
| 627 | EmitSEHNopIfRequired(); |
| 628 | BuildMI(BB&: MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: AArch64::LDRWui), DestReg: AArch64::W30) |
| 629 | .addUse(RegNo: AArch64::LR) |
| 630 | .addImm(Val: 0) |
| 631 | .addMemOperand(MMO: MF.getMachineMemOperand( |
| 632 | PtrInfo: MachinePointerInfo(), |
| 633 | F: MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile, Size: 4, |
| 634 | BaseAlignment: Align(4))) |
| 635 | .setMIFlag(MachineInstr::FrameDestroy); |
| 636 | EmitSEHNopIfRequired(); |
| 637 | BuildMI(BB&: MBB, I: RetInstIter, MIMD: DL, MCID: TII->get(Opcode: AArch64::RET)) |
| 638 | .addUse(RegNo: XReg) |
| 639 | .copyImplicitOps(OtherMI: *RetInstIter); |
| 640 | MBB.erase(I: RetInstIter); |
| 641 | } |
| 642 | Modified = true; |
| 643 | } |
| 644 | |
| 645 | return Modified; |
| 646 | } |
| 647 | |
| 648 | bool AArch64PointerAuthLegacy::runOnMachineFunction(MachineFunction &MF) { |
| 649 | return AArch64PointerAuthImpl().run(MF); |
| 650 | } |
| 651 | |
| 652 | PreservedAnalyses |
| 653 | AArch64PointerAuthPass::run(MachineFunction &MF, |
| 654 | MachineFunctionAnalysisManager &MFAM) { |
| 655 | const bool Changed = AArch64PointerAuthImpl().run(MF); |
| 656 | if (!Changed) |
| 657 | return PreservedAnalyses::all(); |
| 658 | PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses(); |
| 659 | PA.preserveSet<CFGAnalyses>(); |
| 660 | return PA; |
| 661 | } |
| 662 | |