1//===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a printer that converts from our internal representation
10// of machine-dependent LLVM code to the AArch64 assembly language.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AArch64AsmPrinter.h"
15#include "AArch64.h"
16#include "AArch64MCInstLower.h"
17#include "AArch64MachineFunctionInfo.h"
18#include "AArch64RegisterInfo.h"
19#include "AArch64Subtarget.h"
20#include "AArch64TargetObjectFile.h"
21#include "MCTargetDesc/AArch64AddressingModes.h"
22#include "MCTargetDesc/AArch64InstPrinter.h"
23#include "MCTargetDesc/AArch64MCAsmInfo.h"
24#include "MCTargetDesc/AArch64MCTargetDesc.h"
25#include "MCTargetDesc/AArch64TargetStreamer.h"
26#include "TargetInfo/AArch64TargetInfo.h"
27#include "Utils/AArch64BaseInfo.h"
28#include "llvm/ADT/ScopeExit.h"
29#include "llvm/ADT/SmallString.h"
30#include "llvm/ADT/SmallVector.h"
31#include "llvm/ADT/Statistic.h"
32#include "llvm/ADT/StringRef.h"
33#include "llvm/ADT/Twine.h"
34#include "llvm/BinaryFormat/COFF.h"
35#include "llvm/BinaryFormat/ELF.h"
36#include "llvm/BinaryFormat/MachO.h"
37#include "llvm/CodeGen/AsmPrinter.h"
38#include "llvm/CodeGen/AsmPrinterAnalysis.h"
39#include "llvm/CodeGen/FaultMaps.h"
40#include "llvm/CodeGen/MachineBasicBlock.h"
41#include "llvm/CodeGen/MachineFunction.h"
42#include "llvm/CodeGen/MachineInstr.h"
43#include "llvm/CodeGen/MachineJumpTableInfo.h"
44#include "llvm/CodeGen/MachineModuleInfoImpls.h"
45#include "llvm/CodeGen/MachineOperand.h"
46#include "llvm/CodeGen/StackMaps.h"
47#include "llvm/CodeGen/TargetRegisterInfo.h"
48#include "llvm/IR/Analysis.h"
49#include "llvm/IR/DataLayout.h"
50#include "llvm/IR/DebugInfoMetadata.h"
51#include "llvm/IR/Mangler.h"
52#include "llvm/IR/Module.h"
53#include "llvm/IR/PassManager.h"
54#include "llvm/MC/MCAsmInfo.h"
55#include "llvm/MC/MCContext.h"
56#include "llvm/MC/MCExpr.h"
57#include "llvm/MC/MCInst.h"
58#include "llvm/MC/MCInstBuilder.h"
59#include "llvm/MC/MCSectionELF.h"
60#include "llvm/MC/MCSectionMachO.h"
61#include "llvm/MC/MCStreamer.h"
62#include "llvm/MC/MCSymbol.h"
63#include "llvm/MC/MCValue.h"
64#include "llvm/MC/TargetRegistry.h"
65#include "llvm/Support/Casting.h"
66#include "llvm/Support/Compiler.h"
67#include "llvm/Support/ErrorHandling.h"
68#include "llvm/Support/SaveAndRestore.h"
69#include "llvm/Support/raw_ostream.h"
70#include "llvm/Target/TargetMachine.h"
71#include "llvm/TargetParser/Triple.h"
72#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
73#include <cassert>
74#include <cstdint>
75#include <map>
76#include <memory>
77
78using namespace llvm;
79using AArch64PAuth::PtrauthCheckMode;
80
81#define DEBUG_TYPE "AArch64AsmPrinter"
82
83// Doesn't count FPR128 ZCZ instructions which are handled
84// by TableGen pattern matching
85STATISTIC(NumZCZeroingInstrsFPR,
86 "Number of zero-cycle FPR zeroing instructions expanded from "
87 "canonical pseudo instructions");
88
89namespace {
90
91class AArch64AsmPrinter : public AsmPrinter {
92 AArch64MCInstLower MCInstLowering;
93 FaultMaps FM;
94 const AArch64Subtarget *STI;
95 bool ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = false;
96 bool PtrauthInitFini = false;
97 bool PtrauthInitFiniAddressDisc = false;
98#ifndef NDEBUG
99 unsigned InstsEmitted;
100#endif
101 bool EnableImportCallOptimization = false;
102 MapVector<MCSection *, std::vector<std::pair<MCSymbol *, MCSymbol *>>>
103 SectionToImportedFunctionCalls;
104 unsigned PAuthIFuncNextUniqueID = 1;
105
106public:
107 static char ID;
108
109 AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
110 : AsmPrinter(TM, std::move(Streamer), ID),
111 MCInstLowering(OutContext, *this), FM(*this) {}
112
113 StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
114
115 /// Wrapper for MCInstLowering.lowerOperand() for the
116 /// tblgen'erated pseudo lowering.
117 bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
118 return MCInstLowering.lowerOperand(MO, MCOp);
119 }
120
121 const MCExpr *lowerConstantPtrAuth(const ConstantPtrAuth &CPA) override;
122
123 const MCExpr *lowerBlockAddressConstant(const BlockAddress &BA) override;
124
125 void emitStartOfAsmFile(Module &M) override;
126 void emitJumpTableImpl(const MachineJumpTableInfo &MJTI,
127 ArrayRef<unsigned> JumpTableIndices) override;
128 std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
129 codeview::JumpTableEntrySize>
130 getCodeViewJumpTableInfo(int JTI, const MachineInstr *BranchInstr,
131 const MCSymbol *BranchLabel) const override;
132
133 void emitFunctionEntryLabel() override;
134
135 void emitXXStructor(const DataLayout &DL, const Constant *CV) override;
136
137 void LowerJumpTableDest(MCStreamer &OutStreamer, const MachineInstr &MI);
138
139 void LowerHardenedBRJumpTable(const MachineInstr &MI);
140
141 void LowerMOPS(MCStreamer &OutStreamer, const MachineInstr &MI);
142
143 void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
144 const MachineInstr &MI);
145 void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
146 const MachineInstr &MI);
147 void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
148 const MachineInstr &MI);
149 void LowerFAULTING_OP(const MachineInstr &MI);
150
151 void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
152 void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
153 void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
154 void LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI, bool Typed);
155
156 typedef std::tuple<unsigned, bool, uint32_t, bool, uint64_t>
157 HwasanMemaccessTuple;
158 std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
159 void LowerKCFI_CHECK(const MachineInstr &MI);
160 void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
161 void emitHwasanMemaccessSymbols(Module &M);
162
163 void emitSled(const MachineInstr &MI, SledKind Kind);
164
165 // Returns whether Reg may be used to store sensitive temporary values when
166 // expanding PtrAuth pseudos. Some OSes may take extra care to protect a
167 // small subset of GPRs on context switches - use these registers then.
168 //
169 // If there are no preferred registers, returns true for any Reg.
170 bool isPtrauthRegSafe(Register Reg) const {
171 if (STI->isX16X17Safer())
172 return Reg == AArch64::X16 || Reg == AArch64::X17;
173
174 return true;
175 }
176
177 // Emit the sequence for BRA/BLRA (authenticate + branch/call).
178 void emitPtrauthBranch(const MachineInstr *MI);
179
180 void emitPtrauthCheckAuthenticatedValue(Register TestedReg,
181 Register ScratchReg,
182 AArch64PACKey::ID Key,
183 AArch64PAuth::AuthCheckMethod Method,
184 const MCSymbol *OnFailure = nullptr);
185
186 // Check authenticated LR before tail calling.
187 void emitPtrauthTailCallHardening(const MachineInstr *TC);
188
189 struct PtrAuthSchema {
190 static PtrAuthSchema CreateImmReg(AArch64PACKey::ID Key, uint64_t IntDisc,
191 const MachineOperand &AddrDiscOp);
192 static PtrAuthSchema CreateRegReg(AArch64PACKey::ID Key, Register AddrDisc,
193 Register PCDisc);
194
195 AArch64PACKey::ID Key;
196 uint64_t IntDisc;
197 Register AddrDisc;
198 bool AddrDiscIsKilled;
199 Register PCDisc;
200
201 bool addrDiscIsKilledAndNoneOf(std::initializer_list<Register> Regs) {
202 return AddrDiscIsKilled && !llvm::is_contained(Set: Regs, Element: AddrDisc);
203 }
204 };
205
206 // Helper for emitting AUTRELLOADPAC: increment Pointer by Addend and then by
207 // a 32-bit signed value loaded from memory. The instructions emitted are
208 //
209 // ldrsw Scratch, [Pointer, #Addend]!
210 // add Pointer, Pointer, Scratch
211 //
212 // for small Addend value, with longer sequences required for wider Addend.
213 void emitPtrauthApplyIndirectAddend(Register Pointer, Register Scratch,
214 int64_t Addend);
215
216 // Emit the sequence for AUT or AUTPAC (or their PC-blending variants).
217 // Addend is only used for AUTRELLOADPAC.
218 void emitPtrauthAuthResign(Register Pointer, Register Scratch,
219 PtrAuthSchema AuthSchema,
220 std::optional<PtrAuthSchema> SignSchema,
221 std::optional<int64_t> Addend, Value *DS);
222
223 // Emit R_AARCH64_PATCHINST, the deactivation symbol relocation. Returns true
224 // if no instruction should be emitted because the deactivation symbol is
225 // defined in the current module so this function emitted a NOP instead.
226 bool emitDeactivationSymbolRelocation(Value *DS);
227
228 // Emit the sequence for PAC.
229 void emitPtrauthSign(const MachineInstr *MI);
230
231 // Emit the sequence to compute the discriminator.
232 //
233 // The Scratch register passed to this function must be safe, as returned by
234 // isPtrauthRegSafe(ScratchReg).
235 //
236 // The returned register is either ScratchReg, AddrDisc, or XZR. Furthermore,
237 // it is guaranteed to be safe (or XZR), with the only exception of
238 // passing-through an *unmodified* unsafe AddrDisc register.
239 //
240 // If the expanded pseudo is allowed to clobber AddrDisc register, setting
241 // MayClobberAddrDisc may save one MOV instruction, provided
242 // isPtrauthRegSafe(AddrDisc) is true:
243 //
244 // mov x17, x16
245 // movk x17, #1234, lsl #48
246 // ; x16 is not used anymore
247 //
248 // can be replaced by
249 //
250 // movk x16, #1234, lsl #48
251 Register emitPtrauthDiscriminator(uint64_t Disc, Register AddrDisc,
252 Register ScratchReg,
253 bool MayClobberAddrDisc = false);
254
255 // Emit the sequence for LOADauthptrstatic
256 void LowerLOADauthptrstatic(const MachineInstr &MI);
257
258 // Emit the sequence for LOADgotPAC/MOVaddrPAC (either GOT adrp-ldr or
259 // adrp-add followed by PAC sign)
260 void LowerMOVaddrPAC(const MachineInstr &MI);
261
262 // Emit the sequence for LOADgotAUTH (load signed pointer from signed ELF GOT
263 // and authenticate it with, if FPAC bit is not set, check+trap sequence after
264 // authenticating)
265 void LowerLOADgotAUTH(const MachineInstr &MI);
266
267 void emitAddImm(MCRegister Val, int64_t Addend, MCRegister Tmp);
268 void emitAddress(MCRegister Reg, const MCExpr *Expr, MCRegister Tmp,
269 bool DSOLocal, const MCSubtargetInfo &STI);
270
271 const MCExpr *emitPAuthRelocationAsIRelative(
272 const MCExpr *Target, uint64_t Disc, AArch64PACKey::ID KeyID,
273 bool HasAddressDiversity, bool IsDSOLocal, const MCExpr *DSExpr);
274
275 /// tblgen'erated driver function for lowering simple MI->MC
276 /// pseudo instructions.
277 bool lowerPseudoInstExpansion(const MachineInstr *MI, MCInst &Inst);
278
279 // Emit Build Attributes
280 void emitAttributes(unsigned Flags, uint64_t PAuthABIPlatform,
281 uint64_t PAuthABIVersion, AArch64TargetStreamer *TS);
282
283 // Emit expansion of Compare-and-branch pseudo instructions
284 void emitCBPseudoExpansion(const MachineInstr *MI);
285
286 // Emit expansion of atomic store with hint pseudo instructions
287 void emitAtomicHintPseudoExpansion(const MachineInstr *MI);
288 void emitAtomicHintPseudoExpansionRO(const MachineInstr *MI);
289 void emitAtomicHintPseudoExpansionImm(const MachineInstr *MI);
290
291 // Emit expansion of atomic fetch with hint pseudo instructions
292 void emitAtomicFetchHintPseudoExpansion(const MachineInstr *MI);
293
294 void EmitToStreamer(MCStreamer &S, const MCInst &Inst);
295 void EmitToStreamer(const MCInst &Inst) {
296 EmitToStreamer(S&: *OutStreamer, Inst);
297 }
298
299 void emitInstruction(const MachineInstr *MI) override;
300
301 void emitFunctionHeaderComment() override;
302
303 void getAnalysisUsage(AnalysisUsage &AU) const override {
304 AsmPrinter::getAnalysisUsage(AU);
305 AU.setPreservesAll();
306 }
307
308 bool runOnMachineFunction(MachineFunction &MF) override {
309 if (auto *PSIW = getAnalysisIfAvailable<ProfileSummaryInfoWrapperPass>())
310 PSI = &PSIW->getPSI();
311 if (auto *SDPIW =
312 getAnalysisIfAvailable<StaticDataProfileInfoWrapperPass>())
313 SDPI = &SDPIW->getStaticDataProfileInfo();
314
315 AArch64FI = MF.getInfo<AArch64FunctionInfo>();
316 STI = &MF.getSubtarget<AArch64Subtarget>();
317
318 SetupMachineFunction(MF);
319
320 if (STI->isTargetCOFF()) {
321 bool Local = MF.getFunction().hasLocalLinkage();
322 COFF::SymbolStorageClass Scl =
323 Local ? COFF::IMAGE_SYM_CLASS_STATIC : COFF::IMAGE_SYM_CLASS_EXTERNAL;
324 int Type =
325 COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
326
327 OutStreamer->beginCOFFSymbolDef(Symbol: CurrentFnSym);
328 OutStreamer->emitCOFFSymbolStorageClass(StorageClass: Scl);
329 OutStreamer->emitCOFFSymbolType(Type);
330 OutStreamer->endCOFFSymbolDef();
331 }
332
333 // Emit the rest of the function body.
334 emitFunctionBody();
335
336 // Emit the XRay table for this function.
337 emitXRayTable();
338
339 // We didn't modify anything.
340 return false;
341 }
342
343 const MCExpr *lowerConstant(const Constant *CV,
344 const Constant *BaseCV = nullptr,
345 uint64_t Offset = 0) override;
346
347private:
348 void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
349 bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
350 bool printAsmRegInClass(const MachineOperand &MO,
351 const TargetRegisterClass *RC, unsigned AltName,
352 raw_ostream &O);
353
354 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
355 const char *ExtraCode, raw_ostream &O) override;
356 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
357 const char *ExtraCode, raw_ostream &O) override;
358
359 void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
360
361 void emitFunctionBodyEnd() override;
362 void emitBasicBlockEnd(const MachineBasicBlock &MBB) override;
363 void emitGlobalAlias(const Module &M, const GlobalAlias &GA) override;
364
365 MCSymbol *GetCPISymbol(unsigned CPID) const override;
366 void emitEndOfAsmFile(Module &M) override;
367
368 AArch64FunctionInfo *AArch64FI = nullptr;
369
370 /// Emit the LOHs contained in AArch64FI.
371 void emitLOHs();
372
373 void emitMovXReg(Register Dest, Register Src);
374 void emitMOVZ(Register Dest, uint64_t Imm, unsigned Shift);
375 void emitMOVK(Register Dest, uint64_t Imm, unsigned Shift);
376
377 void emitAUT(AArch64PACKey::ID Key, Register Pointer, Register Disc);
378 void emitPAC(AArch64PACKey::ID Key, Register Pointer, Register Disc);
379 void emitBLRA(bool IsCall, AArch64PACKey::ID Key, Register Target,
380 Register Disc);
381
382 /// Emit instruction to set float register to zero.
383 void emitFMov0(const MachineInstr &MI);
384 void emitFMov0AsFMov(const MachineInstr &MI, Register DestReg);
385
386 using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
387
388 MInstToMCSymbol LOHInstToLabel;
389
390 bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const override {
391 return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags;
392 }
393
394 const MCSubtargetInfo *getIFuncMCSubtargetInfo() const override {
395 assert(STI);
396 return STI;
397 }
398 void emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI,
399 MCSymbol *LazyPointer) override;
400 void emitMachOIFuncStubHelperBody(Module &M, const GlobalIFunc &GI,
401 MCSymbol *LazyPointer) override;
402
403 /// Checks if this instruction is part of a sequence that is eligle for import
404 /// call optimization and, if so, records it to be emitted in the import call
405 /// section.
406 void recordIfImportCall(const MachineInstr *BranchInst);
407};
408
409} // end anonymous namespace
410
411// Get boolean module flag (0 or 1), treating absent flag as having value 0.
412static bool getOptionalBooleanModuleFlag(Module &M, StringRef Name) {
413 Metadata *Flag = M.getModuleFlag(Key: Name);
414 if (!Flag)
415 return false;
416
417 uint64_t Value = mdconst::extract<ConstantInt>(MD&: Flag)->getZExtValue();
418 assert((Value == 0 || Value == 1) && "Boolean flag is expected, if present");
419 return Value;
420}
421
422void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
423 const Triple &TT = M.getTargetTriple();
424
425 if (TT.isOSBinFormatCOFF()) {
426 emitCOFFFeatureSymbol(M);
427 emitCOFFReplaceableFunctionData(M);
428
429 if (M.getModuleFlag(Key: "import-call-optimization"))
430 EnableImportCallOptimization = true;
431 }
432
433 PtrauthInitFini = getOptionalBooleanModuleFlag(M, Name: "ptrauth-init-fini");
434 PtrauthInitFiniAddressDisc = getOptionalBooleanModuleFlag(
435 M, Name: "ptrauth-init-fini-address-discrimination");
436
437 if (!TT.isOSBinFormatELF())
438 return;
439
440 // For emitting build attributes and .note.gnu.property section
441 auto *TS =
442 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
443 // Assemble feature flags that may require creation of build attributes and a
444 // note section.
445 unsigned BAFlags = 0;
446 unsigned GNUFlags = 0;
447 if (const auto *BTE = mdconst::extract_or_null<ConstantInt>(
448 MD: M.getModuleFlag(Key: "branch-target-enforcement"))) {
449 if (!BTE->isZero()) {
450 BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_BTI_Flag;
451 GNUFlags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
452 }
453 }
454
455 if (const auto *GCS = mdconst::extract_or_null<ConstantInt>(
456 MD: M.getModuleFlag(Key: "guarded-control-stack"))) {
457 if (!GCS->isZero()) {
458 BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_GCS_Flag;
459 GNUFlags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_GCS;
460 }
461 }
462
463 if (const auto *Sign = mdconst::extract_or_null<ConstantInt>(
464 MD: M.getModuleFlag(Key: "sign-return-address"))) {
465 if (!Sign->isZero()) {
466 BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_PAC_Flag;
467 GNUFlags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
468 }
469 }
470
471 uint64_t PAuthABIPlatform = -1;
472 if (const auto *PAP = mdconst::extract_or_null<ConstantInt>(
473 MD: M.getModuleFlag(Key: "aarch64-elf-pauthabi-platform"))) {
474 PAuthABIPlatform = PAP->getZExtValue();
475 }
476
477 uint64_t PAuthABIVersion = -1;
478 if (const auto *PAV = mdconst::extract_or_null<ConstantInt>(
479 MD: M.getModuleFlag(Key: "aarch64-elf-pauthabi-version"))) {
480 PAuthABIVersion = PAV->getZExtValue();
481 }
482
483 // For LLVM_LINUX experimental platform, version value of 0 means no PAuth
484 // support. Do not emit corresponding PAuthABI GNU property note and AArch64
485 // build attributes for this case to keep Linux binaries not using PAuth
486 // unaffected.
487 if (PAuthABIPlatform == ELF::AARCH64_PAUTH_PLATFORM_LLVM_LINUX &&
488 PAuthABIVersion == 0) {
489 PAuthABIPlatform = uint64_t(-1);
490 PAuthABIVersion = uint64_t(-1);
491 }
492
493 // Emit AArch64 Build Attributes
494 emitAttributes(Flags: BAFlags, PAuthABIPlatform, PAuthABIVersion, TS);
495 // Emit a .note.gnu.property section with the flags.
496 TS->emitNoteSection(Flags: GNUFlags, PAuthABIPlatform, PAuthABIVersion);
497}
498
499void AArch64AsmPrinter::emitFunctionHeaderComment() {
500 const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>();
501 std::optional<std::string> OutlinerString = FI->getOutliningStyle();
502 if (OutlinerString != std::nullopt)
503 OutStreamer->getCommentOS() << ' ' << OutlinerString;
504}
505
506void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
507{
508 const Function &F = MF->getFunction();
509 if (F.hasFnAttribute(Kind: "patchable-function-entry")) {
510 unsigned Num;
511 if (F.getFnAttribute(Kind: "patchable-function-entry")
512 .getValueAsString()
513 .getAsInteger(Radix: 10, Result&: Num))
514 return;
515 emitNops(N: Num);
516 return;
517 }
518
519 emitSled(MI, Kind: SledKind::FUNCTION_ENTER);
520}
521
522void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI) {
523 emitSled(MI, Kind: SledKind::FUNCTION_EXIT);
524}
525
526void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) {
527 emitSled(MI, Kind: SledKind::TAIL_CALL);
528}
529
530void AArch64AsmPrinter::emitSled(const MachineInstr &MI, SledKind Kind) {
531 static const int8_t NoopsInSledCount = 7;
532 // We want to emit the following pattern:
533 //
534 // .Lxray_sled_N:
535 // ALIGN
536 // B #32
537 // ; 7 NOP instructions (28 bytes)
538 // .tmpN
539 //
540 // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
541 // over the full 32 bytes (8 instructions) with the following pattern:
542 //
543 // STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
544 // LDR W17, #12 ; W17 := function ID
545 // LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
546 // BLR X16 ; call the tracing trampoline
547 // ;DATA: 32 bits of function ID
548 // ;DATA: lower 32 bits of the address of the trampoline
549 // ;DATA: higher 32 bits of the address of the trampoline
550 // LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
551 //
552 OutStreamer->emitCodeAlignment(Alignment: Align(4), STI: getSubtargetInfo());
553 auto CurSled = OutContext.createTempSymbol(Name: "xray_sled_", AlwaysAddSuffix: true);
554 OutStreamer->emitLabel(Symbol: CurSled);
555 auto Target = OutContext.createTempSymbol();
556
557 // Emit "B #32" instruction, which jumps over the next 28 bytes.
558 // The operand has to be the number of 4-byte instructions to jump over,
559 // including the current instruction.
560 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::B).addImm(Val: 8));
561
562 for (int8_t I = 0; I < NoopsInSledCount; I++)
563 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::NOP));
564
565 OutStreamer->emitLabel(Symbol: Target);
566 recordSled(Sled: CurSled, MI, Kind, Version: 2);
567}
568
569void AArch64AsmPrinter::emitAttributes(unsigned Flags,
570 uint64_t PAuthABIPlatform,
571 uint64_t PAuthABIVersion,
572 AArch64TargetStreamer *TS) {
573
574 PAuthABIPlatform = (uint64_t(-1) == PAuthABIPlatform) ? 0 : PAuthABIPlatform;
575 PAuthABIVersion = (uint64_t(-1) == PAuthABIVersion) ? 0 : PAuthABIVersion;
576
577 if (PAuthABIPlatform || PAuthABIVersion) {
578 TS->emitAttributesSubsection(
579 VendorName: AArch64BuildAttributes::getVendorName(
580 Vendor: AArch64BuildAttributes::AEABI_PAUTHABI),
581 IsOptional: AArch64BuildAttributes::SubsectionOptional::REQUIRED,
582 ParameterType: AArch64BuildAttributes::SubsectionType::ULEB128);
583 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
584 Vendor: AArch64BuildAttributes::AEABI_PAUTHABI),
585 Tag: AArch64BuildAttributes::TAG_PAUTH_PLATFORM,
586 Value: PAuthABIPlatform, String: "");
587 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
588 Vendor: AArch64BuildAttributes::AEABI_PAUTHABI),
589 Tag: AArch64BuildAttributes::TAG_PAUTH_SCHEMA, Value: PAuthABIVersion,
590 String: "");
591 }
592
593 unsigned BTIValue =
594 (Flags & AArch64BuildAttributes::Feature_BTI_Flag) ? 1 : 0;
595 unsigned PACValue =
596 (Flags & AArch64BuildAttributes::Feature_PAC_Flag) ? 1 : 0;
597 unsigned GCSValue =
598 (Flags & AArch64BuildAttributes::Feature_GCS_Flag) ? 1 : 0;
599
600 if (BTIValue || PACValue || GCSValue) {
601 TS->emitAttributesSubsection(
602 VendorName: AArch64BuildAttributes::getVendorName(
603 Vendor: AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
604 IsOptional: AArch64BuildAttributes::SubsectionOptional::OPTIONAL,
605 ParameterType: AArch64BuildAttributes::SubsectionType::ULEB128);
606 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
607 Vendor: AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
608 Tag: AArch64BuildAttributes::TAG_FEATURE_BTI, Value: BTIValue, String: "");
609 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
610 Vendor: AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
611 Tag: AArch64BuildAttributes::TAG_FEATURE_PAC, Value: PACValue, String: "");
612 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
613 Vendor: AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
614 Tag: AArch64BuildAttributes::TAG_FEATURE_GCS, Value: GCSValue, String: "");
615 }
616}
617
618// Emit the following code for Intrinsic::{xray_customevent,xray_typedevent}
619// (built-in functions __xray_customevent/__xray_typedevent).
620//
621// .Lxray_event_sled_N:
622// b 1f
623// save x0 and x1 (and also x2 for TYPED_EVENT_CALL)
624// set up x0 and x1 (and also x2 for TYPED_EVENT_CALL)
625// bl __xray_CustomEvent or __xray_TypedEvent
626// restore x0 and x1 (and also x2 for TYPED_EVENT_CALL)
627// 1:
628//
629// There are 6 instructions for EVENT_CALL and 9 for TYPED_EVENT_CALL.
630//
631// Then record a sled of kind CUSTOM_EVENT or TYPED_EVENT.
632// After patching, b .+N will become a nop.
633void AArch64AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI,
634 bool Typed) {
635 auto &O = *OutStreamer;
636 MCSymbol *CurSled = OutContext.createTempSymbol(Name: "xray_sled_", AlwaysAddSuffix: true);
637 O.emitLabel(Symbol: CurSled);
638 bool MachO = MMI->getModule()->getTargetTriple().isOSBinFormatMachO();
639 auto *Sym = MCSymbolRefExpr::create(
640 Symbol: OutContext.getOrCreateSymbol(
641 Name: Twine(MachO ? "_" : "") +
642 (Typed ? "__xray_TypedEvent" : "__xray_CustomEvent")),
643 Ctx&: OutContext);
644 if (Typed) {
645 O.AddComment(T: "Begin XRay typed event");
646 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::B).addImm(Val: 9));
647 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::STPXpre)
648 .addReg(Reg: AArch64::SP)
649 .addReg(Reg: AArch64::X0)
650 .addReg(Reg: AArch64::X1)
651 .addReg(Reg: AArch64::SP)
652 .addImm(Val: -4));
653 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::STRXui)
654 .addReg(Reg: AArch64::X2)
655 .addReg(Reg: AArch64::SP)
656 .addImm(Val: 2));
657 emitMovXReg(Dest: AArch64::X0, Src: MI.getOperand(i: 0).getReg());
658 emitMovXReg(Dest: AArch64::X1, Src: MI.getOperand(i: 1).getReg());
659 emitMovXReg(Dest: AArch64::X2, Src: MI.getOperand(i: 2).getReg());
660 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::BL).addExpr(Val: Sym));
661 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::LDRXui)
662 .addReg(Reg: AArch64::X2)
663 .addReg(Reg: AArch64::SP)
664 .addImm(Val: 2));
665 O.AddComment(T: "End XRay typed event");
666 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::LDPXpost)
667 .addReg(Reg: AArch64::SP)
668 .addReg(Reg: AArch64::X0)
669 .addReg(Reg: AArch64::X1)
670 .addReg(Reg: AArch64::SP)
671 .addImm(Val: 4));
672
673 recordSled(Sled: CurSled, MI, Kind: SledKind::TYPED_EVENT, Version: 2);
674 } else {
675 O.AddComment(T: "Begin XRay custom event");
676 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::B).addImm(Val: 6));
677 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::STPXpre)
678 .addReg(Reg: AArch64::SP)
679 .addReg(Reg: AArch64::X0)
680 .addReg(Reg: AArch64::X1)
681 .addReg(Reg: AArch64::SP)
682 .addImm(Val: -2));
683 emitMovXReg(Dest: AArch64::X0, Src: MI.getOperand(i: 0).getReg());
684 emitMovXReg(Dest: AArch64::X1, Src: MI.getOperand(i: 1).getReg());
685 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::BL).addExpr(Val: Sym));
686 O.AddComment(T: "End XRay custom event");
687 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::LDPXpost)
688 .addReg(Reg: AArch64::SP)
689 .addReg(Reg: AArch64::X0)
690 .addReg(Reg: AArch64::X1)
691 .addReg(Reg: AArch64::SP)
692 .addImm(Val: 2));
693
694 recordSled(Sled: CurSled, MI, Kind: SledKind::CUSTOM_EVENT, Version: 2);
695 }
696}
697
698void AArch64AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
699 Register AddrReg = MI.getOperand(i: 0).getReg();
700 assert(std::next(MI.getIterator())->isCall() &&
701 "KCFI_CHECK not followed by a call instruction");
702 assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg &&
703 "KCFI_CHECK call target doesn't match call operand");
704
705 // Default to using the intra-procedure-call temporary registers for
706 // comparing the hashes.
707 unsigned ScratchRegs[] = {AArch64::W16, AArch64::W17};
708 if (AddrReg == AArch64::XZR) {
709 // Checking XZR makes no sense. Instead of emitting a load, zero
710 // ScratchRegs[0] and use it for the ESR AddrIndex below.
711 AddrReg = getXRegFromWReg(Reg: ScratchRegs[0]);
712 emitMovXReg(Dest: AddrReg, Src: AArch64::XZR);
713 } else {
714 // If one of the scratch registers is used for the call target (e.g.
715 // with AArch64::TCRETURNriBTI), we can clobber another caller-saved
716 // temporary register instead (in this case, AArch64::W9) as the check
717 // is immediately followed by the call instruction.
718 for (auto &Reg : ScratchRegs) {
719 if (Reg == getWRegFromXReg(Reg: AddrReg)) {
720 Reg = AArch64::W9;
721 break;
722 }
723 }
724 assert(ScratchRegs[0] != AddrReg && ScratchRegs[1] != AddrReg &&
725 "Invalid scratch registers for KCFI_CHECK");
726
727 // Adjust the offset for patchable-function-prefix. This assumes that
728 // patchable-function-prefix is the same for all functions.
729 int64_t PrefixNops =
730 MI.getMF()->getFunction().getFnAttributeAsParsedInteger(
731 Kind: "patchable-function-prefix");
732
733 // Load the target function type hash.
734 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::LDURWi)
735 .addReg(Reg: ScratchRegs[0])
736 .addReg(Reg: AddrReg)
737 .addImm(Val: -(PrefixNops * 4 + 4)));
738 }
739
740 // Load the expected type hash.
741 const int64_t Type = MI.getOperand(i: 1).getImm();
742 emitMOVK(Dest: ScratchRegs[1], Imm: Type & 0xFFFF, Shift: 0);
743 emitMOVK(Dest: ScratchRegs[1], Imm: (Type >> 16) & 0xFFFF, Shift: 16);
744
745 // Compare the hashes and trap if there's a mismatch.
746 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::SUBSWrs)
747 .addReg(Reg: AArch64::WZR)
748 .addReg(Reg: ScratchRegs[0])
749 .addReg(Reg: ScratchRegs[1])
750 .addImm(Val: 0));
751
752 MCSymbol *Pass = OutContext.createTempSymbol();
753 EmitToStreamer(S&: *OutStreamer,
754 Inst: MCInstBuilder(AArch64::Bcc)
755 .addImm(Val: AArch64CC::EQ)
756 .addExpr(Val: MCSymbolRefExpr::create(Symbol: Pass, Ctx&: OutContext)));
757
758 // The base ESR is 0x8000 and the register information is encoded in bits
759 // 0-9 as follows:
760 // - 0-4: n, where the register Xn contains the target address
761 // - 5-9: m, where the register Wm contains the expected type hash
762 // Where n, m are in [0, 30].
763 unsigned TypeIndex = ScratchRegs[1] - AArch64::W0;
764 unsigned AddrIndex;
765 switch (AddrReg) {
766 default:
767 AddrIndex = AddrReg - AArch64::X0;
768 break;
769 case AArch64::FP:
770 AddrIndex = 29;
771 break;
772 case AArch64::LR:
773 AddrIndex = 30;
774 break;
775 }
776
777 assert(AddrIndex < 31 && TypeIndex < 31);
778
779 unsigned ESR = 0x8000 | ((TypeIndex & 31) << 5) | (AddrIndex & 31);
780 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::BRK).addImm(Val: ESR));
781 OutStreamer->emitLabel(Symbol: Pass);
782}
783
784void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
785 Register Reg = MI.getOperand(i: 0).getReg();
786
787 // The HWASan pass won't emit a CHECK_MEMACCESS intrinsic with a pointer
788 // statically known to be zero. However, conceivably, the HWASan pass may
789 // encounter a "cannot currently statically prove to be null" pointer (and is
790 // therefore unable to omit the intrinsic) that later optimization passes
791 // convert into a statically known-null pointer.
792 if (Reg == AArch64::XZR)
793 return;
794
795 bool IsShort =
796 ((MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES) ||
797 (MI.getOpcode() ==
798 AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW));
799 uint32_t AccessInfo = MI.getOperand(i: 1).getImm();
800 bool IsFixedShadow =
801 ((MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_FIXEDSHADOW) ||
802 (MI.getOpcode() ==
803 AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW));
804 uint64_t FixedShadowOffset = IsFixedShadow ? MI.getOperand(i: 2).getImm() : 0;
805
806 MCSymbol *&Sym = HwasanMemaccessSymbols[HwasanMemaccessTuple(
807 Reg, IsShort, AccessInfo, IsFixedShadow, FixedShadowOffset)];
808 if (!Sym) {
809 // FIXME: Make this work on non-ELF.
810 if (!TM.getTargetTriple().isOSBinFormatELF())
811 report_fatal_error(reason: "llvm.hwasan.check.memaccess only supported on ELF");
812
813 std::string SymName = "__hwasan_check_x" + utostr(X: Reg - AArch64::X0) + "_" +
814 utostr(X: AccessInfo);
815 if (IsFixedShadow)
816 SymName += "_fixed_" + utostr(X: FixedShadowOffset);
817 if (IsShort)
818 SymName += "_short_v2";
819 Sym = OutContext.getOrCreateSymbol(Name: SymName);
820 }
821
822 EmitToStreamer(S&: *OutStreamer,
823 Inst: MCInstBuilder(AArch64::BL)
824 .addExpr(Val: MCSymbolRefExpr::create(Symbol: Sym, Ctx&: OutContext)));
825}
826
827void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) {
828 if (HwasanMemaccessSymbols.empty())
829 return;
830
831 const Triple &TT = M.getTargetTriple();
832 assert(TT.isOSBinFormatELF());
833 // AArch64Subtarget is huge, so heap allocate it so we don't run out of stack
834 // space.
835 auto STI = std::make_unique<AArch64Subtarget>(
836 args: TT, args: TM.getTargetCPU(), args: TM.getTargetCPU(), args: TM.getTargetFeatureString(), args&: TM,
837 args: true);
838 this->STI = STI.get();
839
840 MCSymbol *HwasanTagMismatchV1Sym =
841 OutContext.getOrCreateSymbol(Name: "__hwasan_tag_mismatch");
842 MCSymbol *HwasanTagMismatchV2Sym =
843 OutContext.getOrCreateSymbol(Name: "__hwasan_tag_mismatch_v2");
844
845 const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
846 MCSymbolRefExpr::create(Symbol: HwasanTagMismatchV1Sym, Ctx&: OutContext);
847 const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
848 MCSymbolRefExpr::create(Symbol: HwasanTagMismatchV2Sym, Ctx&: OutContext);
849
850 for (auto &P : HwasanMemaccessSymbols) {
851 unsigned Reg = std::get<0>(t: P.first);
852 bool IsShort = std::get<1>(t: P.first);
853 uint32_t AccessInfo = std::get<2>(t: P.first);
854 bool IsFixedShadow = std::get<3>(t: P.first);
855 uint64_t FixedShadowOffset = std::get<4>(t: P.first);
856 const MCSymbolRefExpr *HwasanTagMismatchRef =
857 IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
858 MCSymbol *Sym = P.second;
859
860 bool HasMatchAllTag =
861 (AccessInfo >> HWASanAccessInfo::HasMatchAllShift) & 1;
862 uint8_t MatchAllTag =
863 (AccessInfo >> HWASanAccessInfo::MatchAllShift) & 0xff;
864 unsigned Size =
865 1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf);
866 bool CompileKernel =
867 (AccessInfo >> HWASanAccessInfo::CompileKernelShift) & 1;
868
869 OutStreamer->switchSection(Section: OutContext.getELFSection(
870 Section: ".text.hot", Type: ELF::SHT_PROGBITS,
871 Flags: ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, EntrySize: 0, Group: Sym->getName(),
872 /*IsComdat=*/true));
873
874 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_ELF_TypeFunction);
875 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Weak);
876 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Hidden);
877 OutStreamer->emitLabel(Symbol: Sym);
878
879 EmitToStreamer(Inst: MCInstBuilder(AArch64::SBFMXri)
880 .addReg(Reg: AArch64::X16)
881 .addReg(Reg)
882 .addImm(Val: 4)
883 .addImm(Val: 55));
884
885 if (IsFixedShadow) {
886 // Aarch64 makes it difficult to embed large constants in the code.
887 // Fortuitously, kShadowBaseAlignment == 32, so we use the 32-bit
888 // left-shift option in the MOV instruction. Combined with the 16-bit
889 // immediate, this is enough to represent any offset up to 2**48.
890 emitMOVZ(Dest: AArch64::X17, Imm: FixedShadowOffset >> 32, Shift: 32);
891 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRBBroX)
892 .addReg(Reg: AArch64::W16)
893 .addReg(Reg: AArch64::X17)
894 .addReg(Reg: AArch64::X16)
895 .addImm(Val: 0)
896 .addImm(Val: 0));
897 } else {
898 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRBBroX)
899 .addReg(Reg: AArch64::W16)
900 .addReg(Reg: IsShort ? AArch64::X20 : AArch64::X9)
901 .addReg(Reg: AArch64::X16)
902 .addImm(Val: 0)
903 .addImm(Val: 0));
904 }
905
906 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSXrs)
907 .addReg(Reg: AArch64::XZR)
908 .addReg(Reg: AArch64::X16)
909 .addReg(Reg)
910 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSR, Imm: 56)));
911 MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
912 EmitToStreamer(Inst: MCInstBuilder(AArch64::Bcc)
913 .addImm(Val: AArch64CC::NE)
914 .addExpr(Val: MCSymbolRefExpr::create(
915 Symbol: HandleMismatchOrPartialSym, Ctx&: OutContext)));
916 MCSymbol *ReturnSym = OutContext.createTempSymbol();
917 OutStreamer->emitLabel(Symbol: ReturnSym);
918 EmitToStreamer(Inst: MCInstBuilder(AArch64::RET).addReg(Reg: AArch64::LR));
919 OutStreamer->emitLabel(Symbol: HandleMismatchOrPartialSym);
920
921 if (HasMatchAllTag) {
922 EmitToStreamer(Inst: MCInstBuilder(AArch64::UBFMXri)
923 .addReg(Reg: AArch64::X17)
924 .addReg(Reg)
925 .addImm(Val: 56)
926 .addImm(Val: 63));
927 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSXri)
928 .addReg(Reg: AArch64::XZR)
929 .addReg(Reg: AArch64::X17)
930 .addImm(Val: MatchAllTag)
931 .addImm(Val: 0));
932 EmitToStreamer(
933 Inst: MCInstBuilder(AArch64::Bcc)
934 .addImm(Val: AArch64CC::EQ)
935 .addExpr(Val: MCSymbolRefExpr::create(Symbol: ReturnSym, Ctx&: OutContext)));
936 }
937
938 if (IsShort) {
939 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSWri)
940 .addReg(Reg: AArch64::WZR)
941 .addReg(Reg: AArch64::W16)
942 .addImm(Val: 15)
943 .addImm(Val: 0));
944 MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
945 EmitToStreamer(
946 Inst: MCInstBuilder(AArch64::Bcc)
947 .addImm(Val: AArch64CC::HI)
948 .addExpr(Val: MCSymbolRefExpr::create(Symbol: HandleMismatchSym, Ctx&: OutContext)));
949
950 EmitToStreamer(Inst: MCInstBuilder(AArch64::ANDXri)
951 .addReg(Reg: AArch64::X17)
952 .addReg(Reg)
953 .addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: 0xf, regSize: 64)));
954 if (Size != 1)
955 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
956 .addReg(Reg: AArch64::X17)
957 .addReg(Reg: AArch64::X17)
958 .addImm(Val: Size - 1)
959 .addImm(Val: 0));
960 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSWrs)
961 .addReg(Reg: AArch64::WZR)
962 .addReg(Reg: AArch64::W16)
963 .addReg(Reg: AArch64::W17)
964 .addImm(Val: 0));
965 EmitToStreamer(
966 Inst: MCInstBuilder(AArch64::Bcc)
967 .addImm(Val: AArch64CC::LS)
968 .addExpr(Val: MCSymbolRefExpr::create(Symbol: HandleMismatchSym, Ctx&: OutContext)));
969
970 EmitToStreamer(Inst: MCInstBuilder(AArch64::ORRXri)
971 .addReg(Reg: AArch64::X16)
972 .addReg(Reg)
973 .addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: 0xf, regSize: 64)));
974 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRBBui)
975 .addReg(Reg: AArch64::W16)
976 .addReg(Reg: AArch64::X16)
977 .addImm(Val: 0));
978 EmitToStreamer(
979 Inst: MCInstBuilder(AArch64::SUBSXrs)
980 .addReg(Reg: AArch64::XZR)
981 .addReg(Reg: AArch64::X16)
982 .addReg(Reg)
983 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSR, Imm: 56)));
984 EmitToStreamer(
985 Inst: MCInstBuilder(AArch64::Bcc)
986 .addImm(Val: AArch64CC::EQ)
987 .addExpr(Val: MCSymbolRefExpr::create(Symbol: ReturnSym, Ctx&: OutContext)));
988
989 OutStreamer->emitLabel(Symbol: HandleMismatchSym);
990 }
991
992 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPXpre)
993 .addReg(Reg: AArch64::SP)
994 .addReg(Reg: AArch64::X0)
995 .addReg(Reg: AArch64::X1)
996 .addReg(Reg: AArch64::SP)
997 .addImm(Val: -32));
998 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPXi)
999 .addReg(Reg: AArch64::FP)
1000 .addReg(Reg: AArch64::LR)
1001 .addReg(Reg: AArch64::SP)
1002 .addImm(Val: 29));
1003
1004 if (Reg != AArch64::X0)
1005 emitMovXReg(Dest: AArch64::X0, Src: Reg);
1006 emitMOVZ(Dest: AArch64::X1, Imm: AccessInfo & HWASanAccessInfo::RuntimeMask, Shift: 0);
1007
1008 if (CompileKernel) {
1009 // The Linux kernel's dynamic loader doesn't support GOT relative
1010 // relocations, but it doesn't support late binding either, so just call
1011 // the function directly.
1012 EmitToStreamer(Inst: MCInstBuilder(AArch64::B).addExpr(Val: HwasanTagMismatchRef));
1013 } else {
1014 // Intentionally load the GOT entry and branch to it, rather than possibly
1015 // late binding the function, which may clobber the registers before we
1016 // have a chance to save them.
1017 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADRP)
1018 .addReg(Reg: AArch64::X16)
1019 .addExpr(Val: MCSpecifierExpr::create(Expr: HwasanTagMismatchRef,
1020 S: AArch64::S_GOT_PAGE,
1021 Ctx&: OutContext)));
1022 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
1023 .addReg(Reg: AArch64::X16)
1024 .addReg(Reg: AArch64::X16)
1025 .addExpr(Val: MCSpecifierExpr::create(Expr: HwasanTagMismatchRef,
1026 S: AArch64::S_GOT_LO12,
1027 Ctx&: OutContext)));
1028 EmitToStreamer(Inst: MCInstBuilder(AArch64::BR).addReg(Reg: AArch64::X16));
1029 }
1030 }
1031 this->STI = nullptr;
1032}
1033
1034static void emitAuthenticatedPointer(MCStreamer &OutStreamer,
1035 MCSymbol *StubLabel,
1036 const MCExpr *StubAuthPtrRef) {
1037 // sym$auth_ptr$key$disc:
1038 OutStreamer.emitLabel(Symbol: StubLabel);
1039 OutStreamer.emitValue(Value: StubAuthPtrRef, /*size=*/Size: 8);
1040}
1041
1042void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
1043 emitHwasanMemaccessSymbols(M);
1044
1045 const Triple &TT = M.getTargetTriple();
1046 if (TT.isOSBinFormatMachO()) {
1047 // Output authenticated pointers as indirect symbols, if we have any.
1048 MachineModuleInfoMachO &MMIMacho =
1049 MMI->getObjFileInfo<MachineModuleInfoMachO>();
1050
1051 auto Stubs = MMIMacho.getAuthGVStubList();
1052
1053 if (!Stubs.empty()) {
1054 // Switch to the "__auth_ptr" section.
1055 OutStreamer->switchSection(
1056 Section: OutContext.getMachOSection(Segment: "__DATA", Section: "__auth_ptr", TypeAndAttributes: MachO::S_REGULAR,
1057 K: SectionKind::getMetadata()));
1058 emitAlignment(Alignment: Align(8));
1059
1060 for (const auto &Stub : Stubs)
1061 emitAuthenticatedPointer(OutStreamer&: *OutStreamer, StubLabel: Stub.first, StubAuthPtrRef: Stub.second);
1062
1063 OutStreamer->addBlankLine();
1064 }
1065
1066 // Funny Darwin hack: This flag tells the linker that no global symbols
1067 // contain code that falls through to other global symbols (e.g. the obvious
1068 // implementation of multiple entry points). If this doesn't occur, the
1069 // linker can safely perform dead code stripping. Since LLVM never
1070 // generates code that does this, it is always safe to set.
1071 OutStreamer->emitSubsectionsViaSymbols();
1072 }
1073
1074 if (TT.isOSBinFormatELF()) {
1075 // Output authenticated pointers as indirect symbols, if we have any.
1076 MachineModuleInfoELF &MMIELF = MMI->getObjFileInfo<MachineModuleInfoELF>();
1077
1078 auto Stubs = MMIELF.getAuthGVStubList();
1079
1080 if (!Stubs.empty()) {
1081 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
1082 OutStreamer->switchSection(Section: TLOF.getDataSection());
1083 emitAlignment(Alignment: Align(8));
1084
1085 for (const auto &Stub : Stubs)
1086 emitAuthenticatedPointer(OutStreamer&: *OutStreamer, StubLabel: Stub.first, StubAuthPtrRef: Stub.second);
1087
1088 OutStreamer->addBlankLine();
1089 }
1090
1091 // With signed ELF GOT enabled, the linker looks at the symbol type to
1092 // choose between keys IA (for STT_FUNC) and DA (for other types). Symbols
1093 // for functions not defined in the module have STT_NOTYPE type by default.
1094 // This makes linker to emit signing schema with DA key (instead of IA) for
1095 // corresponding R_AARCH64_AUTH_GLOB_DAT dynamic reloc. To avoid that, force
1096 // all function symbols used in the module to have STT_FUNC type. See
1097 // https://github.com/ARM-software/abi-aa/blob/main/pauthabielf64/pauthabielf64.rst#default-signing-schema
1098 const auto *PtrAuthELFGOTFlag = mdconst::extract_or_null<ConstantInt>(
1099 MD: M.getModuleFlag(Key: "ptrauth-elf-got"));
1100 if (PtrAuthELFGOTFlag && PtrAuthELFGOTFlag->getZExtValue() == 1)
1101 for (const GlobalValue &GV : M.global_values())
1102 if (!GV.use_empty() && isa<Function>(Val: GV) &&
1103 !GV.getName().starts_with(Prefix: "llvm."))
1104 OutStreamer->emitSymbolAttribute(Symbol: getSymbol(GV: &GV),
1105 Attribute: MCSA_ELF_TypeFunction);
1106 }
1107
1108 // Emit stack and fault map information.
1109 FM.serializeToFaultMapSection();
1110
1111 // If import call optimization is enabled, emit the appropriate section.
1112 // We do this whether or not we recorded any import calls.
1113 if (EnableImportCallOptimization && TT.isOSBinFormatCOFF()) {
1114 OutStreamer->switchSection(Section: getObjFileLowering().getImportCallSection());
1115
1116 // Section always starts with some magic.
1117 constexpr char ImpCallMagic[12] = "Imp_Call_V1";
1118 OutStreamer->emitBytes(Data: StringRef{ImpCallMagic, sizeof(ImpCallMagic)});
1119
1120 // Layout of this section is:
1121 // Per section that contains calls to imported functions:
1122 // uint32_t SectionSize: Size in bytes for information in this section.
1123 // uint32_t Section Number
1124 // Per call to imported function in section:
1125 // uint32_t Kind: the kind of imported function.
1126 // uint32_t BranchOffset: the offset of the branch instruction in its
1127 // parent section.
1128 // uint32_t TargetSymbolId: the symbol id of the called function.
1129 for (auto &[Section, CallsToImportedFuncs] :
1130 SectionToImportedFunctionCalls) {
1131 unsigned SectionSize =
1132 sizeof(uint32_t) * (2 + 3 * CallsToImportedFuncs.size());
1133 OutStreamer->emitInt32(Value: SectionSize);
1134 OutStreamer->emitCOFFSecNumber(Symbol: Section->getBeginSymbol());
1135 for (auto &[CallsiteSymbol, CalledSymbol] : CallsToImportedFuncs) {
1136 // Kind is always IMAGE_REL_ARM64_DYNAMIC_IMPORT_CALL (0x13).
1137 OutStreamer->emitInt32(Value: 0x13);
1138 OutStreamer->emitCOFFSecOffset(Symbol: CallsiteSymbol);
1139 OutStreamer->emitCOFFSymbolIndex(Symbol: CalledSymbol);
1140 }
1141 }
1142 }
1143}
1144
1145void AArch64AsmPrinter::emitLOHs() {
1146 SmallVector<MCSymbol *, 3> MCArgs;
1147
1148 for (const auto &D : AArch64FI->getLOHContainer()) {
1149 for (const MachineInstr *MI : D.getArgs()) {
1150 MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(x: MI);
1151 assert(LabelIt != LOHInstToLabel.end() &&
1152 "Label hasn't been inserted for LOH related instruction");
1153 MCArgs.push_back(Elt: LabelIt->second);
1154 }
1155 OutStreamer->emitLOHDirective(Kind: D.getKind(), Args: MCArgs);
1156 MCArgs.clear();
1157 }
1158}
1159
1160// Unwinding can get confused if the last instruction in an exception-handling
1161// region (function, funclet, try block, etc.) is a call, because the return
1162// address then points to the *next* region (see
1163// X86AsmPrinter::maybeEmitNopAfterCallForWindowsEH for the X86 equivalent).
1164// Trap after a noreturn call that would end a function or funclet, and elide
1165// the trap when the next instruction is in the same function.
1166void AArch64AsmPrinter::emitBasicBlockEnd(const MachineBasicBlock &MBB) {
1167 if (MBB.succ_empty() && MF->usesWindowsUnwindInfo()) {
1168 auto Next = std::next(x: MBB.getIterator());
1169 if (Next == MF->end() || Next->isEHFuncletEntry()) {
1170 for (const MachineInstr &MI : reverse(C: MBB)) {
1171 if (MI.isMetaInstruction() || AArch64InstrInfo::isSEHInstruction(MI))
1172 continue;
1173 if (MI.isCall() && !MI.isReturn())
1174 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::BRK).addImm(Val: 1));
1175 break;
1176 }
1177 }
1178 }
1179
1180 AsmPrinter::emitBasicBlockEnd(MBB);
1181}
1182
1183void AArch64AsmPrinter::emitFunctionBodyEnd() {
1184 if (!AArch64FI->getLOHRelated().empty())
1185 emitLOHs();
1186}
1187
1188/// GetCPISymbol - Return the symbol for the specified constant pool entry.
1189MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
1190 // Darwin uses a linker-private symbol name for constant-pools (to
1191 // avoid addends on the relocation?), ELF has no such concept and
1192 // uses a normal private symbol.
1193 if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
1194 return OutContext.getOrCreateSymbol(
1195 Name: Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
1196 Twine(getFunctionNumber()) + "_" + Twine(CPID));
1197
1198 return AsmPrinter::GetCPISymbol(CPID);
1199}
1200
1201void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
1202 raw_ostream &O) {
1203 const MachineOperand &MO = MI->getOperand(i: OpNum);
1204 switch (MO.getType()) {
1205 default:
1206 llvm_unreachable("<unknown operand type>");
1207 case MachineOperand::MO_Register: {
1208 Register Reg = MO.getReg();
1209 assert(Reg.isPhysical());
1210 assert(!MO.getSubReg() && "Subregs should be eliminated!");
1211 O << AArch64InstPrinter::getRegisterName(Reg);
1212 break;
1213 }
1214 case MachineOperand::MO_Immediate: {
1215 O << MO.getImm();
1216 break;
1217 }
1218 case MachineOperand::MO_GlobalAddress: {
1219 PrintSymbolOperand(MO, OS&: O);
1220 break;
1221 }
1222 case MachineOperand::MO_BlockAddress: {
1223 MCSymbol *Sym = GetBlockAddressSymbol(BA: MO.getBlockAddress());
1224 Sym->print(OS&: O, MAI);
1225 break;
1226 }
1227 }
1228}
1229
1230bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
1231 raw_ostream &O) {
1232 Register Reg = MO.getReg();
1233 switch (Mode) {
1234 default:
1235 return true; // Unknown mode.
1236 case 'w':
1237 Reg = getWRegFromXReg(Reg);
1238 break;
1239 case 'x':
1240 Reg = getXRegFromWReg(Reg);
1241 break;
1242 case 't':
1243 Reg = getXRegFromXRegTuple(RegTuple: Reg);
1244 break;
1245 }
1246
1247 O << AArch64InstPrinter::getRegisterName(Reg);
1248 return false;
1249}
1250
1251// Prints the register in MO using class RC using the offset in the
1252// new register class. This should not be used for cross class
1253// printing.
1254bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
1255 const TargetRegisterClass *RC,
1256 unsigned AltName, raw_ostream &O) {
1257 assert(MO.isReg() && "Should only get here with a register!");
1258 const TargetRegisterInfo *RI = STI->getRegisterInfo();
1259 Register Reg = MO.getReg();
1260 MCRegister RegToPrint = RC->getRegister(i: RI->getEncodingValue(Reg));
1261 if (!RI->regsOverlap(RegA: RegToPrint, RegB: Reg))
1262 return true;
1263 O << AArch64InstPrinter::getRegisterName(Reg: RegToPrint, AltIdx: AltName);
1264 return false;
1265}
1266
1267bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
1268 const char *ExtraCode, raw_ostream &O) {
1269 const MachineOperand &MO = MI->getOperand(i: OpNum);
1270
1271 // First try the generic code, which knows about modifiers like 'c' and 'n'.
1272 if (!AsmPrinter::PrintAsmOperand(MI, OpNo: OpNum, ExtraCode, OS&: O))
1273 return false;
1274
1275 // Does this asm operand have a single letter operand modifier?
1276 if (ExtraCode && ExtraCode[0]) {
1277 if (ExtraCode[1] != 0)
1278 return true; // Unknown modifier.
1279
1280 switch (ExtraCode[0]) {
1281 default:
1282 return true; // Unknown modifier.
1283 case 'w': // Print W register
1284 case 'x': // Print X register
1285 if (MO.isReg())
1286 return printAsmMRegister(MO, Mode: ExtraCode[0], O);
1287 if (MO.isImm() && MO.getImm() == 0) {
1288 unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
1289 O << AArch64InstPrinter::getRegisterName(Reg);
1290 return false;
1291 }
1292 printOperand(MI, OpNum, O);
1293 return false;
1294 case 'b': // Print B register.
1295 case 'h': // Print H register.
1296 case 's': // Print S register.
1297 case 'd': // Print D register.
1298 case 'q': // Print Q register.
1299 case 'z': // Print Z register.
1300 if (MO.isReg()) {
1301 const TargetRegisterClass *RC;
1302 switch (ExtraCode[0]) {
1303 case 'b':
1304 RC = &AArch64::FPR8RegClass;
1305 break;
1306 case 'h':
1307 RC = &AArch64::FPR16RegClass;
1308 break;
1309 case 's':
1310 RC = &AArch64::FPR32RegClass;
1311 break;
1312 case 'd':
1313 RC = &AArch64::FPR64RegClass;
1314 break;
1315 case 'q':
1316 RC = &AArch64::FPR128RegClass;
1317 break;
1318 case 'z':
1319 RC = &AArch64::ZPRRegClass;
1320 break;
1321 default:
1322 return true;
1323 }
1324 return printAsmRegInClass(MO, RC, AltName: AArch64::NoRegAltName, O);
1325 }
1326 printOperand(MI, OpNum, O);
1327 return false;
1328 }
1329 }
1330
1331 // According to ARM, we should emit x and v registers unless we have a
1332 // modifier.
1333 if (MO.isReg()) {
1334 Register Reg = MO.getReg();
1335
1336 // If this is a w or x register, print an x register.
1337 if (AArch64::GPR32allRegClass.contains(Reg) ||
1338 AArch64::GPR64allRegClass.contains(Reg))
1339 return printAsmMRegister(MO, Mode: 'x', O);
1340
1341 // If this is an x register tuple, print an x register.
1342 if (AArch64::GPR64x8ClassRegClass.contains(Reg))
1343 return printAsmMRegister(MO, Mode: 't', O);
1344
1345 unsigned AltName = AArch64::NoRegAltName;
1346 const TargetRegisterClass *RegClass;
1347 if (AArch64::ZPRRegClass.contains(Reg)) {
1348 RegClass = &AArch64::ZPRRegClass;
1349 } else if (AArch64::PPRRegClass.contains(Reg)) {
1350 RegClass = &AArch64::PPRRegClass;
1351 } else if (AArch64::PNRRegClass.contains(Reg)) {
1352 RegClass = &AArch64::PNRRegClass;
1353 } else {
1354 RegClass = &AArch64::FPR128RegClass;
1355 AltName = AArch64::vreg;
1356 }
1357
1358 // If this is a b, h, s, d, or q register, print it as a v register.
1359 return printAsmRegInClass(MO, RC: RegClass, AltName, O);
1360 }
1361
1362 printOperand(MI, OpNum, O);
1363 return false;
1364}
1365
1366bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
1367 unsigned OpNum,
1368 const char *ExtraCode,
1369 raw_ostream &O) {
1370 if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
1371 return true; // Unknown modifier.
1372
1373 const MachineOperand &MO = MI->getOperand(i: OpNum);
1374 assert(MO.isReg() && "unexpected inline asm memory operand");
1375 O << "[" << AArch64InstPrinter::getRegisterName(Reg: MO.getReg()) << "]";
1376 return false;
1377}
1378
1379void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
1380 raw_ostream &OS) {
1381 unsigned NOps = MI->getNumOperands();
1382 assert(NOps == 4);
1383 OS << '\t' << MAI.getCommentString() << "DEBUG_VALUE: ";
1384 // cast away const; DIetc do not take const operands for some reason.
1385 OS << MI->getDebugVariable()->getName();
1386 OS << " <- ";
1387 // Frame address. Currently handles register +- offset only.
1388 assert(MI->isIndirectDebugValue());
1389 OS << '[';
1390 for (unsigned I = 0, E = llvm::size(Range: MI->debug_operands()); I < E; ++I) {
1391 if (I != 0)
1392 OS << ", ";
1393 printOperand(MI, OpNum: I, O&: OS);
1394 }
1395 OS << ']';
1396 OS << "+";
1397 printOperand(MI, OpNum: NOps - 2, O&: OS);
1398}
1399
1400void AArch64AsmPrinter::emitJumpTableImpl(const MachineJumpTableInfo &MJTI,
1401 ArrayRef<unsigned> JumpTableIndices) {
1402 // Fast return if there is nothing to emit to avoid creating empty sections.
1403 if (JumpTableIndices.empty())
1404 return;
1405 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
1406 const auto &F = MF->getFunction();
1407 ArrayRef<MachineJumpTableEntry> JT = MJTI.getJumpTables();
1408
1409 MCSection *ReadOnlySec = nullptr;
1410 if (TM.Options.EnableStaticDataPartitioning) {
1411 ReadOnlySec =
1412 TLOF.getSectionForJumpTable(F, TM, JTE: &JT[JumpTableIndices.front()]);
1413 } else {
1414 ReadOnlySec = TLOF.getSectionForJumpTable(F, TM);
1415 }
1416 OutStreamer->switchSection(Section: ReadOnlySec);
1417
1418 auto AFI = MF->getInfo<AArch64FunctionInfo>();
1419 for (unsigned JTI : JumpTableIndices) {
1420 const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
1421
1422 // If this jump table was deleted, ignore it.
1423 if (JTBBs.empty()) continue;
1424
1425 unsigned Size = AFI->getJumpTableEntrySize(Idx: JTI);
1426 emitAlignment(Alignment: Align(Size));
1427 OutStreamer->emitLabel(Symbol: GetJTISymbol(JTID: JTI));
1428
1429 const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(Idx: JTI);
1430 const MCExpr *Base = MCSymbolRefExpr::create(Symbol: BaseSym, Ctx&: OutContext);
1431
1432 for (auto *JTBB : JTBBs) {
1433 const MCExpr *Value =
1434 MCSymbolRefExpr::create(Symbol: JTBB->getSymbol(), Ctx&: OutContext);
1435
1436 // Each entry is:
1437 // .byte/.hword (LBB - Lbase)>>2
1438 // or plain:
1439 // .word LBB - Lbase
1440 Value = MCBinaryExpr::createSub(LHS: Value, RHS: Base, Ctx&: OutContext);
1441 if (Size != 4)
1442 Value = MCBinaryExpr::createLShr(
1443 LHS: Value, RHS: MCConstantExpr::create(Value: 2, Ctx&: OutContext), Ctx&: OutContext);
1444
1445 OutStreamer->emitValue(Value, Size);
1446 }
1447 }
1448}
1449
1450std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
1451 codeview::JumpTableEntrySize>
1452AArch64AsmPrinter::getCodeViewJumpTableInfo(int JTI,
1453 const MachineInstr *BranchInstr,
1454 const MCSymbol *BranchLabel) const {
1455 const auto AFI = MF->getInfo<AArch64FunctionInfo>();
1456 const auto Base = AArch64FI->getJumpTableEntryPCRelSymbol(Idx: JTI);
1457 codeview::JumpTableEntrySize EntrySize;
1458 switch (AFI->getJumpTableEntrySize(Idx: JTI)) {
1459 case 1:
1460 EntrySize = codeview::JumpTableEntrySize::UInt8ShiftLeft;
1461 break;
1462 case 2:
1463 EntrySize = codeview::JumpTableEntrySize::UInt16ShiftLeft;
1464 break;
1465 case 4:
1466 EntrySize = codeview::JumpTableEntrySize::Int32;
1467 break;
1468 default:
1469 llvm_unreachable("Unexpected jump table entry size");
1470 }
1471 return std::make_tuple(args: Base, args: 0, args&: BranchLabel, args&: EntrySize);
1472}
1473
1474void AArch64AsmPrinter::emitFunctionEntryLabel() {
1475 const Triple &TT = TM.getTargetTriple();
1476 if (TT.isOSBinFormatELF() &&
1477 (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall ||
1478 MF->getFunction().getCallingConv() ==
1479 CallingConv::AArch64_SVE_VectorCall ||
1480 MF->getInfo<AArch64FunctionInfo>()->isSVECC())) {
1481 auto *TS =
1482 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
1483 TS->emitDirectiveVariantPCS(Symbol: CurrentFnSym);
1484 }
1485
1486 AsmPrinter::emitFunctionEntryLabel();
1487
1488 if (TT.isWindowsArm64EC() && !MF->getFunction().hasLocalLinkage()) {
1489 // For ARM64EC targets, a function definition's name is mangled differently
1490 // from the normal symbol, emit required aliases here.
1491 auto emitFunctionAlias = [&](MCSymbol *Src, MCSymbol *Dst) {
1492 OutStreamer->emitSymbolAttribute(Symbol: Src, Attribute: MCSA_WeakAntiDep);
1493 OutStreamer->emitAssignment(
1494 Symbol: Src, Value: MCSymbolRefExpr::create(Symbol: Dst, Ctx&: MMI->getContext()));
1495 };
1496
1497 auto getSymbolFromMetadata = [&](StringRef Name) {
1498 MCSymbol *Sym = nullptr;
1499 if (MDNode *Node = MF->getFunction().getMetadata(Kind: Name)) {
1500 StringRef NameStr = cast<MDString>(Val: Node->getOperand(I: 0))->getString();
1501 Sym = MMI->getContext().getOrCreateSymbol(Name: NameStr);
1502 }
1503 return Sym;
1504 };
1505
1506 SmallVector<MDNode *> UnmangledNames;
1507 MF->getFunction().getMetadata(Kind: "arm64ec_unmangled_name", MDs&: UnmangledNames);
1508 for (MDNode *Node : UnmangledNames) {
1509 StringRef NameStr = cast<MDString>(Val: Node->getOperand(I: 0))->getString();
1510 MCSymbol *UnmangledSym = MMI->getContext().getOrCreateSymbol(Name: NameStr);
1511 if (std::optional<std::string> MangledName =
1512 getArm64ECMangledFunctionName(Name: UnmangledSym->getName())) {
1513 MCSymbol *ECMangledSym =
1514 MMI->getContext().getOrCreateSymbol(Name: *MangledName);
1515 emitFunctionAlias(UnmangledSym, ECMangledSym);
1516 }
1517 }
1518 if (MCSymbol *ECMangledSym =
1519 getSymbolFromMetadata("arm64ec_ecmangled_name"))
1520 emitFunctionAlias(ECMangledSym, CurrentFnSym);
1521 }
1522}
1523
1524void AArch64AsmPrinter::emitXXStructor(const DataLayout &DL,
1525 const Constant *CV) {
1526 LLVMContext &C = CV->getContext();
1527 assert(!isa<ConstantPtrAuth>(CV) &&
1528 "ctors/dtors are to be signed by asm printer");
1529
1530 if (PtrauthInitFini) {
1531 IntegerType *Int32Ty = IntegerType::get(C, NumBits: 32);
1532 IntegerType *Int64Ty = IntegerType::get(C, NumBits: 64);
1533 PointerType *PtrTy = PointerType::get(C, AddressSpace: 0);
1534
1535 ConstantInt *Key = ConstantInt::get(Ty: Int32Ty, V: AArch64PAuth::InitFiniKey);
1536 ConstantInt *IntDisc = ConstantInt::get(
1537 Ty: Int64Ty, V: AArch64PAuth::InitFiniPointerConstantDiscriminator);
1538 Constant *Null = ConstantPointerNull::get(T: PtrTy);
1539 Constant *AddressDisc = Null;
1540 if (PtrauthInitFiniAddressDisc) {
1541 uint64_t Marker = ConstantPtrAuth::AddrDiscriminator_CtorsDtors;
1542 AddressDisc =
1543 ConstantExpr::getIntToPtr(C: ConstantInt::get(Ty: Int64Ty, V: Marker), Ty: PtrTy);
1544 }
1545
1546 CV = ConstantPtrAuth::get(Ptr: const_cast<Constant *>(CV), Key, Disc: IntDisc,
1547 AddrDisc: AddressDisc, /*DeactivationSymbol=*/Null);
1548 }
1549
1550 // Signed pointers will be lowered by AArch64AsmPrinter::lowerConstantPtrAuth.
1551 AsmPrinter::emitXXStructor(DL, CV);
1552}
1553
1554void AArch64AsmPrinter::emitGlobalAlias(const Module &M,
1555 const GlobalAlias &GA) {
1556 if (auto F = dyn_cast_or_null<Function>(Val: GA.getAliasee())) {
1557 // Global aliases must point to a definition, but unmangled patchable
1558 // symbols are special and need to point to an undefined symbol with "EXP+"
1559 // prefix. Such undefined symbol is resolved by the linker by creating
1560 // x86 thunk that jumps back to the actual EC target.
1561 if (MDNode *Node = F->getMetadata(Kind: "arm64ec_exp_name")) {
1562 StringRef ExpStr = cast<MDString>(Val: Node->getOperand(I: 0))->getString();
1563 MCSymbol *ExpSym = MMI->getContext().getOrCreateSymbol(Name: ExpStr);
1564 MCSymbol *Sym = MMI->getContext().getOrCreateSymbol(Name: GA.getName());
1565
1566 OutStreamer->beginCOFFSymbolDef(Symbol: ExpSym);
1567 OutStreamer->emitCOFFSymbolStorageClass(StorageClass: COFF::IMAGE_SYM_CLASS_EXTERNAL);
1568 OutStreamer->emitCOFFSymbolType(Type: COFF::IMAGE_SYM_DTYPE_FUNCTION
1569 << COFF::SCT_COMPLEX_TYPE_SHIFT);
1570 OutStreamer->endCOFFSymbolDef();
1571
1572 OutStreamer->beginCOFFSymbolDef(Symbol: Sym);
1573 OutStreamer->emitCOFFSymbolStorageClass(StorageClass: COFF::IMAGE_SYM_CLASS_EXTERNAL);
1574 OutStreamer->emitCOFFSymbolType(Type: COFF::IMAGE_SYM_DTYPE_FUNCTION
1575 << COFF::SCT_COMPLEX_TYPE_SHIFT);
1576 OutStreamer->endCOFFSymbolDef();
1577 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Weak);
1578 OutStreamer->emitAssignment(
1579 Symbol: Sym, Value: MCSymbolRefExpr::create(Symbol: ExpSym, Ctx&: MMI->getContext()));
1580 return;
1581 }
1582 }
1583 AsmPrinter::emitGlobalAlias(M, GA);
1584}
1585
1586/// Small jump tables contain an unsigned byte or half, representing the offset
1587/// from the lowest-addressed possible destination to the desired basic
1588/// block. Since all instructions are 4-byte aligned, this is further compressed
1589/// by counting in instructions rather than bytes (i.e. divided by 4). So, to
1590/// materialize the correct destination we need:
1591///
1592/// adr xDest, .LBB0_0
1593/// ldrb wScratch, [xTable, xEntry] (with "lsl #1" for ldrh).
1594/// add xDest, xDest, xScratch (with "lsl #2" for smaller entries)
1595void AArch64AsmPrinter::LowerJumpTableDest(llvm::MCStreamer &OutStreamer,
1596 const llvm::MachineInstr &MI) {
1597 Register DestReg = MI.getOperand(i: 0).getReg();
1598 Register ScratchReg = MI.getOperand(i: 1).getReg();
1599 Register ScratchRegW =
1600 STI->getRegisterInfo()->getSubReg(Reg: ScratchReg, Idx: AArch64::sub_32);
1601 Register TableReg = MI.getOperand(i: 2).getReg();
1602 Register EntryReg = MI.getOperand(i: 3).getReg();
1603 int JTIdx = MI.getOperand(i: 4).getIndex();
1604 int Size = AArch64FI->getJumpTableEntrySize(Idx: JTIdx);
1605
1606 // This has to be first because the compression pass based its reachability
1607 // calculations on the start of the JumpTableDest instruction.
1608 auto Label =
1609 MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(Idx: JTIdx);
1610
1611 // If we don't already have a symbol to use as the base, use the ADR
1612 // instruction itself.
1613 if (!Label) {
1614 Label = MF->getContext().createTempSymbol();
1615 AArch64FI->setJumpTableEntryInfo(Idx: JTIdx, Size, PCRelSym: Label);
1616 OutStreamer.emitLabel(Symbol: Label);
1617 }
1618
1619 auto LabelExpr = MCSymbolRefExpr::create(Symbol: Label, Ctx&: MF->getContext());
1620 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::ADR)
1621 .addReg(Reg: DestReg)
1622 .addExpr(Val: LabelExpr));
1623
1624 // Load the number of instruction-steps to offset from the label.
1625 unsigned LdrOpcode;
1626 switch (Size) {
1627 case 1: LdrOpcode = AArch64::LDRBBroX; break;
1628 case 2: LdrOpcode = AArch64::LDRHHroX; break;
1629 case 4: LdrOpcode = AArch64::LDRSWroX; break;
1630 default:
1631 llvm_unreachable("Unknown jump table size");
1632 }
1633
1634 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(LdrOpcode)
1635 .addReg(Reg: Size == 4 ? ScratchReg : ScratchRegW)
1636 .addReg(Reg: TableReg)
1637 .addReg(Reg: EntryReg)
1638 .addImm(Val: 0)
1639 .addImm(Val: Size == 1 ? 0 : 1));
1640
1641 // Add to the already materialized base label address, multiplying by 4 if
1642 // compressed.
1643 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::ADDXrs)
1644 .addReg(Reg: DestReg)
1645 .addReg(Reg: DestReg)
1646 .addReg(Reg: ScratchReg)
1647 .addImm(Val: Size == 4 ? 0 : 2));
1648}
1649
1650void AArch64AsmPrinter::LowerHardenedBRJumpTable(const MachineInstr &MI) {
1651 const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
1652 assert(MJTI && "Can't lower jump-table dispatch without JTI");
1653
1654 const std::vector<MachineJumpTableEntry> &JTs = MJTI->getJumpTables();
1655 assert(!JTs.empty() && "Invalid JT index for jump-table dispatch");
1656
1657 // Emit:
1658 // mov x17, #<size of table> ; depending on table size, with MOVKs
1659 // cmp x16, x17 ; or #imm if table size fits in 12-bit
1660 // csel x16, x16, xzr, ls ; check for index overflow
1661 //
1662 // adrp x17, Ltable@PAGE ; materialize table address
1663 // add x17, Ltable@PAGEOFF
1664 // ldrsw x16, [x17, x16, lsl #2] ; load table entry
1665 //
1666 // Lanchor:
1667 // adr x17, Lanchor ; compute target address
1668 // add x16, x17, x16
1669 // br x16 ; branch to target
1670
1671 MachineOperand JTOp = MI.getOperand(i: 0);
1672
1673 unsigned JTI = JTOp.getIndex();
1674 assert(!AArch64FI->getJumpTableEntryPCRelSymbol(JTI) &&
1675 "unsupported compressed jump table");
1676
1677 const uint64_t NumTableEntries = JTs[JTI].MBBs.size();
1678
1679 // cmp only supports a 12-bit immediate. If we need more, materialize the
1680 // immediate, using x17 as a scratch register.
1681 uint64_t MaxTableEntry = NumTableEntries - 1;
1682 if (isUInt<12>(x: MaxTableEntry)) {
1683 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::SUBSXri)
1684 .addReg(Reg: AArch64::XZR)
1685 .addReg(Reg: AArch64::X16)
1686 .addImm(Val: MaxTableEntry)
1687 .addImm(Val: 0));
1688 } else {
1689 emitMOVZ(Dest: AArch64::X17, Imm: static_cast<uint16_t>(MaxTableEntry), Shift: 0);
1690 // It's sad that we have to manually materialize instructions, but we can't
1691 // trivially reuse the main pseudo expansion logic.
1692 // A MOVK sequence is easy enough to generate and handles the general case.
1693 for (int Offset = 16; Offset < 64; Offset += 16) {
1694 if ((MaxTableEntry >> Offset) == 0)
1695 break;
1696 emitMOVK(Dest: AArch64::X17, Imm: static_cast<uint16_t>(MaxTableEntry >> Offset),
1697 Shift: Offset);
1698 }
1699 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::SUBSXrs)
1700 .addReg(Reg: AArch64::XZR)
1701 .addReg(Reg: AArch64::X16)
1702 .addReg(Reg: AArch64::X17)
1703 .addImm(Val: 0));
1704 }
1705
1706 // This picks entry #0 on failure.
1707 // We might want to trap instead.
1708 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::CSELXr)
1709 .addReg(Reg: AArch64::X16)
1710 .addReg(Reg: AArch64::X16)
1711 .addReg(Reg: AArch64::XZR)
1712 .addImm(Val: AArch64CC::LS));
1713
1714 // Prepare the @PAGE/@PAGEOFF low/high operands.
1715 MachineOperand JTMOHi(JTOp), JTMOLo(JTOp);
1716 MCOperand JTMCHi, JTMCLo;
1717
1718 JTMOHi.setTargetFlags(AArch64II::MO_PAGE);
1719 JTMOLo.setTargetFlags(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
1720
1721 MCInstLowering.lowerOperand(MO: JTMOHi, MCOp&: JTMCHi);
1722 MCInstLowering.lowerOperand(MO: JTMOLo, MCOp&: JTMCLo);
1723
1724 EmitToStreamer(
1725 S&: *OutStreamer,
1726 Inst: MCInstBuilder(AArch64::ADRP).addReg(Reg: AArch64::X17).addOperand(Op: JTMCHi));
1727
1728 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ADDXri)
1729 .addReg(Reg: AArch64::X17)
1730 .addReg(Reg: AArch64::X17)
1731 .addOperand(Op: JTMCLo)
1732 .addImm(Val: 0));
1733
1734 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::LDRSWroX)
1735 .addReg(Reg: AArch64::X16)
1736 .addReg(Reg: AArch64::X17)
1737 .addReg(Reg: AArch64::X16)
1738 .addImm(Val: 0)
1739 .addImm(Val: 1));
1740
1741 MCSymbol *AdrLabel = MF->getContext().createTempSymbol();
1742 const auto *AdrLabelE = MCSymbolRefExpr::create(Symbol: AdrLabel, Ctx&: MF->getContext());
1743 AArch64FI->setJumpTableEntryInfo(Idx: JTI, Size: 4, PCRelSym: AdrLabel);
1744
1745 OutStreamer->emitLabel(Symbol: AdrLabel);
1746 EmitToStreamer(
1747 S&: *OutStreamer,
1748 Inst: MCInstBuilder(AArch64::ADR).addReg(Reg: AArch64::X17).addExpr(Val: AdrLabelE));
1749
1750 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ADDXrs)
1751 .addReg(Reg: AArch64::X16)
1752 .addReg(Reg: AArch64::X17)
1753 .addReg(Reg: AArch64::X16)
1754 .addImm(Val: 0));
1755
1756 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::BR).addReg(Reg: AArch64::X16));
1757}
1758
1759void AArch64AsmPrinter::LowerMOPS(llvm::MCStreamer &OutStreamer,
1760 const llvm::MachineInstr &MI) {
1761 unsigned Opcode = MI.getOpcode();
1762 assert(STI->hasMOPS());
1763 assert(STI->hasMTE() || Opcode != AArch64::MOPSMemorySetTaggingPseudo);
1764
1765 const auto Ops = [Opcode]() -> std::array<unsigned, 3> {
1766 if (Opcode == AArch64::MOPSMemoryCopyPseudo)
1767 return {AArch64::CPYFP, AArch64::CPYFM, AArch64::CPYFE};
1768 if (Opcode == AArch64::MOPSMemoryMovePseudo)
1769 return {AArch64::CPYP, AArch64::CPYM, AArch64::CPYE};
1770 if (Opcode == AArch64::MOPSMemorySetPseudo)
1771 return {AArch64::SETP, AArch64::SETM, AArch64::SETE};
1772 if (Opcode == AArch64::MOPSMemorySetTaggingPseudo)
1773 return {AArch64::SETGP, AArch64::SETGM, AArch64::MOPSSETGE};
1774 llvm_unreachable("Unhandled memory operation pseudo");
1775 }();
1776 const bool IsSet = Opcode == AArch64::MOPSMemorySetPseudo ||
1777 Opcode == AArch64::MOPSMemorySetTaggingPseudo;
1778
1779 for (auto Op : Ops) {
1780 int i = 0;
1781 auto MCIB = MCInstBuilder(Op);
1782 // Destination registers
1783 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1784 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1785 if (!IsSet)
1786 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1787 // Input registers
1788 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1789 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1790 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1791
1792 EmitToStreamer(S&: OutStreamer, Inst: MCIB);
1793 }
1794}
1795
1796void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
1797 const MachineInstr &MI) {
1798 unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
1799
1800 auto &Ctx = OutStreamer.getContext();
1801 MCSymbol *MILabel = Ctx.createTempSymbol();
1802 OutStreamer.emitLabel(Symbol: MILabel);
1803
1804 SM.recordStackMap(L: *MILabel, MI);
1805 assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1806
1807 // Scan ahead to trim the shadow.
1808 const MachineBasicBlock &MBB = *MI.getParent();
1809 MachineBasicBlock::const_iterator MII(MI);
1810 ++MII;
1811 while (NumNOPBytes > 0) {
1812 if (MII != MBB.end() && MII->isDebugInstr()) {
1813 ++MII;
1814 continue;
1815 }
1816 if (MII == MBB.end() || MII->isCall() ||
1817 MII->getOpcode() == TargetOpcode::PATCHPOINT ||
1818 MII->getOpcode() == TargetOpcode::STACKMAP)
1819 break;
1820 ++MII;
1821 NumNOPBytes -= 4;
1822 }
1823
1824 // Emit nops.
1825 for (unsigned i = 0; i < NumNOPBytes; i += 4)
1826 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::NOP));
1827}
1828
1829// Lower a patchpoint of the form:
1830// [<def>], <id>, <numBytes>, <target>, <numArgs>
1831void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1832 const MachineInstr &MI) {
1833 auto &Ctx = OutStreamer.getContext();
1834 MCSymbol *MILabel = Ctx.createTempSymbol();
1835 OutStreamer.emitLabel(Symbol: MILabel);
1836 SM.recordPatchPoint(L: *MILabel, MI);
1837
1838 PatchPointOpers Opers(&MI);
1839
1840 int64_t CallTarget = Opers.getCallTarget().getImm();
1841 unsigned EncodedBytes = 0;
1842 if (CallTarget) {
1843 assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
1844 "High 16 bits of call target should be zero.");
1845 Register ScratchReg = MI.getOperand(i: Opers.getNextScratchIdx()).getReg();
1846 EncodedBytes = 16;
1847 // Materialize the jump address:
1848 emitMOVZ(Dest: ScratchReg, Imm: (CallTarget >> 32) & 0xFFFF, Shift: 32);
1849 emitMOVK(Dest: ScratchReg, Imm: (CallTarget >> 16) & 0xFFFF, Shift: 16);
1850 emitMOVK(Dest: ScratchReg, Imm: CallTarget & 0xFFFF, Shift: 0);
1851 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::BLR).addReg(Reg: ScratchReg));
1852 }
1853 // Emit padding.
1854 unsigned NumBytes = Opers.getNumPatchBytes();
1855 assert(NumBytes >= EncodedBytes &&
1856 "Patchpoint can't request size less than the length of a call.");
1857 assert((NumBytes - EncodedBytes) % 4 == 0 &&
1858 "Invalid number of NOP bytes requested!");
1859 for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
1860 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::NOP));
1861}
1862
1863void AArch64AsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1864 const MachineInstr &MI) {
1865 StatepointOpers SOpers(&MI);
1866 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
1867 assert(PatchBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1868 for (unsigned i = 0; i < PatchBytes; i += 4)
1869 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::NOP));
1870 } else {
1871 // Lower call target and choose correct opcode
1872 const MachineOperand &CallTarget = SOpers.getCallTarget();
1873 MCOperand CallTargetMCOp;
1874 unsigned CallOpcode;
1875 switch (CallTarget.getType()) {
1876 case MachineOperand::MO_GlobalAddress:
1877 case MachineOperand::MO_ExternalSymbol:
1878 MCInstLowering.lowerOperand(MO: CallTarget, MCOp&: CallTargetMCOp);
1879 CallOpcode = AArch64::BL;
1880 break;
1881 case MachineOperand::MO_Immediate:
1882 CallTargetMCOp = MCOperand::createImm(Val: CallTarget.getImm());
1883 CallOpcode = AArch64::BL;
1884 break;
1885 case MachineOperand::MO_Register:
1886 CallTargetMCOp = MCOperand::createReg(Reg: CallTarget.getReg());
1887 CallOpcode = AArch64::BLR;
1888 break;
1889 default:
1890 llvm_unreachable("Unsupported operand type in statepoint call target");
1891 break;
1892 }
1893
1894 EmitToStreamer(S&: OutStreamer,
1895 Inst: MCInstBuilder(CallOpcode).addOperand(Op: CallTargetMCOp));
1896 }
1897
1898 auto &Ctx = OutStreamer.getContext();
1899 MCSymbol *MILabel = Ctx.createTempSymbol();
1900 OutStreamer.emitLabel(Symbol: MILabel);
1901 SM.recordStatepoint(L: *MILabel, MI);
1902}
1903
1904void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) {
1905 // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
1906 // <opcode>, <operands>
1907
1908 Register DefRegister = FaultingMI.getOperand(i: 0).getReg();
1909 FaultMaps::FaultKind FK =
1910 static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(i: 1).getImm());
1911 MCSymbol *HandlerLabel = FaultingMI.getOperand(i: 2).getMBB()->getSymbol();
1912 unsigned Opcode = FaultingMI.getOperand(i: 3).getImm();
1913 unsigned OperandsBeginIdx = 4;
1914
1915 auto &Ctx = OutStreamer->getContext();
1916 MCSymbol *FaultingLabel = Ctx.createTempSymbol();
1917 OutStreamer->emitLabel(Symbol: FaultingLabel);
1918
1919 assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
1920 FM.recordFaultingOp(FaultTy: FK, FaultingLabel, HandlerLabel);
1921
1922 MCInst MI;
1923 MI.setOpcode(Opcode);
1924
1925 if (DefRegister != (Register)0)
1926 MI.addOperand(Op: MCOperand::createReg(Reg: DefRegister));
1927
1928 for (const MachineOperand &MO :
1929 llvm::drop_begin(RangeOrContainer: FaultingMI.operands(), N: OperandsBeginIdx)) {
1930 MCOperand Dest;
1931 lowerOperand(MO, MCOp&: Dest);
1932 MI.addOperand(Op: Dest);
1933 }
1934
1935 OutStreamer->AddComment(T: "on-fault: " + HandlerLabel->getName());
1936 EmitToStreamer(Inst: MI);
1937}
1938
1939void AArch64AsmPrinter::emitMovXReg(Register Dest, Register Src) {
1940 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ORRXrs)
1941 .addReg(Reg: Dest)
1942 .addReg(Reg: AArch64::XZR)
1943 .addReg(Reg: Src)
1944 .addImm(Val: 0));
1945}
1946
1947void AArch64AsmPrinter::emitMOVZ(Register Dest, uint64_t Imm, unsigned Shift) {
1948 bool Is64Bit = AArch64::GPR64RegClass.contains(Reg: Dest);
1949 EmitToStreamer(S&: *OutStreamer,
1950 Inst: MCInstBuilder(Is64Bit ? AArch64::MOVZXi : AArch64::MOVZWi)
1951 .addReg(Reg: Dest)
1952 .addImm(Val: Imm)
1953 .addImm(Val: Shift));
1954}
1955
1956void AArch64AsmPrinter::emitMOVK(Register Dest, uint64_t Imm, unsigned Shift) {
1957 bool Is64Bit = AArch64::GPR64RegClass.contains(Reg: Dest);
1958 EmitToStreamer(S&: *OutStreamer,
1959 Inst: MCInstBuilder(Is64Bit ? AArch64::MOVKXi : AArch64::MOVKWi)
1960 .addReg(Reg: Dest)
1961 .addReg(Reg: Dest)
1962 .addImm(Val: Imm)
1963 .addImm(Val: Shift));
1964}
1965
1966void AArch64AsmPrinter::emitAUT(AArch64PACKey::ID Key, Register Pointer,
1967 Register Disc) {
1968 bool IsZeroDisc = Disc == AArch64::XZR;
1969 unsigned Opcode = getAUTOpcodeForKey(K: Key, Zero: IsZeroDisc);
1970
1971 // autiza x16 ; if IsZeroDisc
1972 // autia x16, x17 ; if !IsZeroDisc
1973 MCInst AUTInst;
1974 AUTInst.setOpcode(Opcode);
1975 AUTInst.addOperand(Op: MCOperand::createReg(Reg: Pointer));
1976 AUTInst.addOperand(Op: MCOperand::createReg(Reg: Pointer));
1977 if (!IsZeroDisc)
1978 AUTInst.addOperand(Op: MCOperand::createReg(Reg: Disc));
1979
1980 EmitToStreamer(Inst: AUTInst);
1981}
1982
1983void AArch64AsmPrinter::emitPAC(AArch64PACKey::ID Key, Register Pointer,
1984 Register Disc) {
1985 bool IsZeroDisc = Disc == AArch64::XZR;
1986 unsigned Opcode = getPACOpcodeForKey(K: Key, Zero: IsZeroDisc);
1987
1988 // paciza x16 ; if IsZeroDisc
1989 // pacia x16, x17 ; if !IsZeroDisc
1990 MCInst PACInst;
1991 PACInst.setOpcode(Opcode);
1992 PACInst.addOperand(Op: MCOperand::createReg(Reg: Pointer));
1993 PACInst.addOperand(Op: MCOperand::createReg(Reg: Pointer));
1994 if (!IsZeroDisc)
1995 PACInst.addOperand(Op: MCOperand::createReg(Reg: Disc));
1996
1997 EmitToStreamer(Inst: PACInst);
1998}
1999
2000void AArch64AsmPrinter::emitBLRA(bool IsCall, AArch64PACKey::ID Key,
2001 Register Target, Register Disc) {
2002 bool IsZeroDisc = Disc == AArch64::XZR;
2003 unsigned Opcode = getBranchOpcodeForKey(IsCall, K: Key, Zero: IsZeroDisc);
2004
2005 // blraaz x16 ; if IsZeroDisc
2006 // blraa x16, x17 ; if !IsZeroDisc
2007 MCInst Inst;
2008 Inst.setOpcode(Opcode);
2009 Inst.addOperand(Op: MCOperand::createReg(Reg: Target));
2010 if (!IsZeroDisc)
2011 Inst.addOperand(Op: MCOperand::createReg(Reg: Disc));
2012 EmitToStreamer(Inst);
2013}
2014
2015void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) {
2016 Register DestReg = MI.getOperand(i: 0).getReg();
2017 if (!STI->hasZeroCycleZeroingFPWorkaround() && STI->isNeonAvailable()) {
2018 if (STI->hasZeroCycleZeroingFPR64()) {
2019 // Convert H/S register to corresponding D register
2020 const AArch64RegisterInfo *TRI = STI->getRegisterInfo();
2021 if (AArch64::FPR16RegClass.contains(Reg: DestReg))
2022 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
2023 RC: &AArch64::FPR64RegClass);
2024 else if (AArch64::FPR32RegClass.contains(Reg: DestReg))
2025 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::ssub,
2026 RC: &AArch64::FPR64RegClass);
2027 else
2028 assert(AArch64::FPR64RegClass.contains(DestReg));
2029
2030 MCInst MOVI;
2031 MOVI.setOpcode(AArch64::MOVID);
2032 MOVI.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2033 MOVI.addOperand(Op: MCOperand::createImm(Val: 0));
2034 EmitToStreamer(S&: *OutStreamer, Inst: MOVI);
2035 ++NumZCZeroingInstrsFPR;
2036 } else if (STI->hasZeroCycleZeroingFPR128()) {
2037 // Convert H/S/D register to corresponding Q register
2038 const AArch64RegisterInfo *TRI = STI->getRegisterInfo();
2039 if (AArch64::FPR16RegClass.contains(Reg: DestReg)) {
2040 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
2041 RC: &AArch64::FPR128RegClass);
2042 } else if (AArch64::FPR32RegClass.contains(Reg: DestReg)) {
2043 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::ssub,
2044 RC: &AArch64::FPR128RegClass);
2045 } else {
2046 assert(AArch64::FPR64RegClass.contains(DestReg));
2047 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::dsub,
2048 RC: &AArch64::FPR128RegClass);
2049 }
2050
2051 MCInst MOVI;
2052 MOVI.setOpcode(AArch64::MOVIv2d_ns);
2053 MOVI.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2054 MOVI.addOperand(Op: MCOperand::createImm(Val: 0));
2055 EmitToStreamer(S&: *OutStreamer, Inst: MOVI);
2056 ++NumZCZeroingInstrsFPR;
2057 } else {
2058 emitFMov0AsFMov(MI, DestReg);
2059 }
2060 } else {
2061 emitFMov0AsFMov(MI, DestReg);
2062 }
2063}
2064
2065void AArch64AsmPrinter::emitFMov0AsFMov(const MachineInstr &MI,
2066 Register DestReg) {
2067 MCInst FMov;
2068 switch (MI.getOpcode()) {
2069 default:
2070 llvm_unreachable("Unexpected opcode");
2071 case AArch64::FMOVH0:
2072 FMov.setOpcode(STI->hasFullFP16() ? AArch64::FMOVWHr : AArch64::FMOVWSr);
2073 if (!STI->hasFullFP16())
2074 DestReg = (AArch64::S0 + (DestReg - AArch64::H0));
2075 FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2076 FMov.addOperand(Op: MCOperand::createReg(Reg: AArch64::WZR));
2077 break;
2078 case AArch64::FMOVS0:
2079 FMov.setOpcode(AArch64::FMOVWSr);
2080 FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2081 FMov.addOperand(Op: MCOperand::createReg(Reg: AArch64::WZR));
2082 break;
2083 case AArch64::FMOVD0:
2084 FMov.setOpcode(AArch64::FMOVXDr);
2085 FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2086 FMov.addOperand(Op: MCOperand::createReg(Reg: AArch64::XZR));
2087 break;
2088 }
2089 EmitToStreamer(S&: *OutStreamer, Inst: FMov);
2090}
2091
2092Register AArch64AsmPrinter::emitPtrauthDiscriminator(uint64_t Disc,
2093 Register AddrDisc,
2094 Register ScratchReg,
2095 bool MayClobberAddrDisc) {
2096 assert(isPtrauthRegSafe(ScratchReg) &&
2097 "Safe scratch register must be provided by the caller");
2098 assert(isUInt<16>(Disc) && "Constant discriminator is too wide");
2099
2100 // So far we've used NoRegister in pseudos. Now we need real encodings.
2101 if (!AddrDisc.isValid())
2102 AddrDisc = AArch64::XZR;
2103
2104 // If there is no constant discriminator, there's no blend involved:
2105 // just use the address discriminator register as-is (XZR or not).
2106 if (!Disc)
2107 return AddrDisc;
2108
2109 // If there's only a constant discriminator, MOV it into the scratch register.
2110 if (AddrDisc == AArch64::XZR) {
2111 emitMOVZ(Dest: ScratchReg, Imm: Disc, Shift: 0);
2112 return ScratchReg;
2113 }
2114
2115 // If there are both, emit a blend into the scratch register.
2116
2117 // Check if we can save one MOV instruction.
2118 if (MayClobberAddrDisc && isPtrauthRegSafe(Reg: AddrDisc)) {
2119 ScratchReg = AddrDisc;
2120 } else {
2121 emitMovXReg(Dest: ScratchReg, Src: AddrDisc);
2122 assert(ScratchReg != AddrDisc &&
2123 "Forbidden to clobber AddrDisc, but have to");
2124 }
2125
2126 emitMOVK(Dest: ScratchReg, Imm: Disc, Shift: 48);
2127 return ScratchReg;
2128}
2129
2130/// Emit a code sequence to check an authenticated pointer value.
2131///
2132/// This function emits a sequence of instructions that checks if TestedReg was
2133/// authenticated successfully. On success, execution continues at the next
2134/// instruction after the sequence.
2135///
2136/// The action performed on failure depends on the OnFailure argument:
2137/// * if OnFailure is not nullptr, control is transferred to that label after
2138/// clearing the PAC field
2139/// * otherwise, BRK instruction is emitted to generate an error
2140void AArch64AsmPrinter::emitPtrauthCheckAuthenticatedValue(
2141 Register TestedReg, Register ScratchReg, AArch64PACKey::ID Key,
2142 AArch64PAuth::AuthCheckMethod Method, const MCSymbol *OnFailure) {
2143 // Insert a sequence to check if authentication of TestedReg succeeded,
2144 // such as:
2145 //
2146 // - checked and clearing:
2147 // ; x16 is TestedReg, x17 is ScratchReg
2148 // mov x17, x16
2149 // xpaci x17
2150 // cmp x16, x17
2151 // b.eq Lsuccess
2152 // mov x16, x17
2153 // b Lend
2154 // Lsuccess:
2155 // ; skipped if authentication failed
2156 // Lend:
2157 // ...
2158 //
2159 // - checked and trapping:
2160 // mov x17, x16
2161 // xpaci x17
2162 // cmp x16, x17
2163 // b.eq Lsuccess
2164 // brk #<0xc470 + aut key>
2165 // Lsuccess:
2166 // ...
2167 //
2168 // See the documentation on AuthCheckMethod enumeration constants for
2169 // the specific code sequences that can be used to perform the check.
2170 using AArch64PAuth::AuthCheckMethod;
2171
2172 if (Method == AuthCheckMethod::None)
2173 return;
2174 if (Method == AuthCheckMethod::DummyLoad) {
2175 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRWui)
2176 .addReg(Reg: getWRegFromXReg(Reg: ScratchReg))
2177 .addReg(Reg: TestedReg)
2178 .addImm(Val: 0));
2179 assert(!OnFailure && "DummyLoad always traps on error");
2180 return;
2181 }
2182
2183 MCSymbol *SuccessSym = createTempSymbol(Name: "auth_success_");
2184 if (Method == AuthCheckMethod::XPAC || Method == AuthCheckMethod::XPACHint) {
2185 // mov Xscratch, Xtested
2186 emitMovXReg(Dest: ScratchReg, Src: TestedReg);
2187
2188 if (Method == AuthCheckMethod::XPAC) {
2189 // xpac(i|d) Xscratch
2190 unsigned XPACOpc = getXPACOpcodeForKey(K: Key);
2191 EmitToStreamer(
2192 Inst: MCInstBuilder(XPACOpc).addReg(Reg: ScratchReg).addReg(Reg: ScratchReg));
2193 } else {
2194 // xpaclri
2195
2196 // Note that this method applies XPAC to TestedReg instead of ScratchReg.
2197 assert(TestedReg == AArch64::LR &&
2198 "XPACHint mode is only compatible with checking the LR register");
2199 assert((Key == AArch64PACKey::IA || Key == AArch64PACKey::IB) &&
2200 "XPACHint mode is only compatible with I-keys");
2201 EmitToStreamer(Inst: MCInstBuilder(AArch64::XPACLRI));
2202 }
2203
2204 // cmp Xtested, Xscratch
2205 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSXrs)
2206 .addReg(Reg: AArch64::XZR)
2207 .addReg(Reg: TestedReg)
2208 .addReg(Reg: ScratchReg)
2209 .addImm(Val: 0));
2210
2211 // b.eq Lsuccess
2212 EmitToStreamer(
2213 Inst: MCInstBuilder(AArch64::Bcc)
2214 .addImm(Val: AArch64CC::EQ)
2215 .addExpr(Val: MCSymbolRefExpr::create(Symbol: SuccessSym, Ctx&: OutContext)));
2216 } else if (Method == AuthCheckMethod::HighBitsNoTBI) {
2217 // eor Xscratch, Xtested, Xtested, lsl #1
2218 EmitToStreamer(Inst: MCInstBuilder(AArch64::EORXrs)
2219 .addReg(Reg: ScratchReg)
2220 .addReg(Reg: TestedReg)
2221 .addReg(Reg: TestedReg)
2222 .addImm(Val: 1));
2223 // tbz Xscratch, #62, Lsuccess
2224 EmitToStreamer(
2225 Inst: MCInstBuilder(AArch64::TBZX)
2226 .addReg(Reg: ScratchReg)
2227 .addImm(Val: 62)
2228 .addExpr(Val: MCSymbolRefExpr::create(Symbol: SuccessSym, Ctx&: OutContext)));
2229 } else {
2230 llvm_unreachable("Unsupported check method");
2231 }
2232
2233 if (!OnFailure) {
2234 // Trapping sequences do a 'brk'.
2235 // brk #<0xc470 + aut key>
2236 EmitToStreamer(Inst: MCInstBuilder(AArch64::BRK).addImm(Val: 0xc470 | Key));
2237 } else {
2238 // Non-trapping checked sequences return the stripped result in TestedReg,
2239 // skipping over success-only code (such as re-signing the pointer) by
2240 // jumping to OnFailure label.
2241 // Note that this can introduce an authentication oracle (such as based on
2242 // the high bits of the re-signed value).
2243
2244 // FIXME: The XPAC method can be optimized by applying XPAC to TestedReg
2245 // instead of ScratchReg, thus eliminating one `mov` instruction.
2246 // Both XPAC and XPACHint can be further optimized by not using a
2247 // conditional branch jumping over an unconditional one.
2248
2249 switch (Method) {
2250 case AuthCheckMethod::XPACHint:
2251 // LR is already XPAC-ed at this point.
2252 break;
2253 case AuthCheckMethod::XPAC:
2254 // mov Xtested, Xscratch
2255 emitMovXReg(Dest: TestedReg, Src: ScratchReg);
2256 break;
2257 default:
2258 // If Xtested was not XPAC-ed so far, emit XPAC here.
2259 // xpac(i|d) Xtested
2260 unsigned XPACOpc = getXPACOpcodeForKey(K: Key);
2261 EmitToStreamer(
2262 Inst: MCInstBuilder(XPACOpc).addReg(Reg: TestedReg).addReg(Reg: TestedReg));
2263 }
2264
2265 // b Lend
2266 const auto *OnFailureExpr = MCSymbolRefExpr::create(Symbol: OnFailure, Ctx&: OutContext);
2267 EmitToStreamer(Inst: MCInstBuilder(AArch64::B).addExpr(Val: OnFailureExpr));
2268 }
2269
2270 // If the auth check succeeds, we can continue.
2271 // Lsuccess:
2272 OutStreamer->emitLabel(Symbol: SuccessSym);
2273}
2274
2275// With Pointer Authentication, it may be needed to explicitly check the
2276// authenticated value in LR before performing a tail call.
2277// Otherwise, the callee may re-sign the invalid return address,
2278// introducing a signing oracle.
2279void AArch64AsmPrinter::emitPtrauthTailCallHardening(const MachineInstr *TC) {
2280 if (!AArch64FI->shouldSignReturnAddress(MF: *MF))
2281 return;
2282
2283 auto LRCheckMethod = STI->getAuthenticatedLRCheckMethod(MF: *MF);
2284 if (LRCheckMethod == AArch64PAuth::AuthCheckMethod::None)
2285 return;
2286
2287 const AArch64RegisterInfo *TRI = STI->getRegisterInfo();
2288 Register ScratchReg =
2289 TC->readsRegister(Reg: AArch64::X16, TRI) ? AArch64::X17 : AArch64::X16;
2290 assert(!TC->readsRegister(ScratchReg, TRI) &&
2291 "Neither x16 nor x17 is available as a scratch register");
2292 AArch64PACKey::ID Key =
2293 AArch64FI->shouldSignWithBKey() ? AArch64PACKey::IB : AArch64PACKey::IA;
2294 emitPtrauthCheckAuthenticatedValue(TestedReg: AArch64::LR, ScratchReg, Key,
2295 Method: LRCheckMethod);
2296}
2297
2298bool AArch64AsmPrinter::emitDeactivationSymbolRelocation(Value *DS) {
2299 if (!DS)
2300 return false;
2301
2302 if (isa<GlobalAlias>(Val: DS)) {
2303 // Just emit the nop directly.
2304 EmitToStreamer(Inst: MCInstBuilder(AArch64::NOP));
2305 return true;
2306 }
2307 MCSymbol *Dot = OutContext.createTempSymbol();
2308 OutStreamer->emitLabel(Symbol: Dot);
2309 const MCExpr *DeactDotExpr = MCSymbolRefExpr::create(Symbol: Dot, Ctx&: OutContext);
2310
2311 const MCExpr *DSExpr = MCSymbolRefExpr::create(
2312 Symbol: OutContext.getOrCreateSymbol(Name: DS->getName()), Ctx&: OutContext);
2313 OutStreamer->emitRelocDirective(Offset: *DeactDotExpr, Name: "R_AARCH64_PATCHINST", Expr: DSExpr,
2314 Loc: SMLoc());
2315 return false;
2316}
2317
2318AArch64AsmPrinter::PtrAuthSchema AArch64AsmPrinter::PtrAuthSchema::CreateImmReg(
2319 AArch64PACKey::ID Key, uint64_t IntDisc, const MachineOperand &AddrDiscOp) {
2320 PtrAuthSchema Schema;
2321 Schema.Key = Key;
2322 Schema.IntDisc = IntDisc;
2323 Schema.AddrDisc = AddrDiscOp.getReg();
2324 Schema.AddrDiscIsKilled = AddrDiscOp.isKill();
2325 Schema.PCDisc = Register();
2326 return Schema;
2327}
2328
2329AArch64AsmPrinter::PtrAuthSchema AArch64AsmPrinter::PtrAuthSchema::CreateRegReg(
2330 AArch64PACKey::ID Key, Register AddrDisc, Register PCDisc) {
2331 assert(PCDisc.isValid() && "Use CreateImmReg for non-PC schemas");
2332 PtrAuthSchema Schema;
2333 Schema.Key = Key;
2334 Schema.IntDisc = 0;
2335 Schema.AddrDisc = AddrDisc;
2336 Schema.AddrDiscIsKilled = false;
2337 Schema.PCDisc = PCDisc;
2338 return Schema;
2339}
2340
2341void AArch64AsmPrinter::emitPtrauthApplyIndirectAddend(Register Pointer,
2342 Register Scratch,
2343 int64_t Addend) {
2344 if (isInt<9>(x: Addend)) {
2345 // ldrsw Scratch, [Pointer, #Addend]! ; note: Pointer+Addend is used later.
2346 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRSWpre)
2347 .addReg(Reg: Pointer)
2348 .addReg(Reg: Scratch)
2349 .addReg(Reg: Pointer)
2350 .addImm(/*simm9:*/ Val: Addend));
2351 } else {
2352 // Pointer += Addend computation has 2 variants
2353 if (isUInt<24>(x: Addend)) {
2354 // Variant 1: add Pointer, Pointer, (Addend >> shift12) lsl shift12
2355 // This can take up to 2 instructions.
2356 for (int BitPos = 0; BitPos != 24 && (Addend >> BitPos); BitPos += 12) {
2357 EmitToStreamer(
2358 Inst: MCInstBuilder(AArch64::ADDXri)
2359 .addReg(Reg: Pointer)
2360 .addReg(Reg: Pointer)
2361 .addImm(Val: (Addend >> BitPos) & 0xfff)
2362 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: BitPos)));
2363 }
2364 } else {
2365 // Variant 2: accumulate constant in Scratch 16 bits at a time,
2366 // and add it to Pointer. This can take 2-5 instructions.
2367 emitMOVZ(Dest: Scratch, Imm: Addend & 0xffff, Shift: 0);
2368 for (int Offset = 16; Offset < 64; Offset += 16) {
2369 if (unsigned Fragment = (Addend >> Offset) & 0xffff)
2370 emitMOVK(Dest: Scratch, Imm: Fragment, Shift: Offset);
2371 }
2372
2373 // add Pointer, Pointer, Scratch
2374 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXrs)
2375 .addReg(Reg: Pointer)
2376 .addReg(Reg: Pointer)
2377 .addReg(Reg: Scratch)
2378 .addImm(Val: 0));
2379 }
2380 // ldrsw Scratch, [Pointer]
2381 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRSWui)
2382 .addReg(Reg: Scratch)
2383 .addReg(Reg: Pointer)
2384 .addImm(Val: 0));
2385 }
2386 // add Pointer, Pointer, Scratch
2387 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXrs)
2388 .addReg(Reg: Pointer)
2389 .addReg(Reg: Pointer)
2390 .addReg(Reg: Scratch)
2391 .addImm(Val: 0));
2392}
2393
2394static PtrauthCheckMode getCheckMode(const MachineFunction *MF) {
2395 const AArch64Subtarget &STI = MF->getSubtarget<AArch64Subtarget>();
2396
2397 // If an override is passed via command line argument, just use that value.
2398 if (std::optional<PtrauthCheckMode> Mode =
2399 STI.getCLOpts().ptrauth_auth_checks)
2400 return *Mode;
2401
2402 // Otherwise, on an FPAC CPU, you get traps whether you want them or not:
2403 // there's no point in emitting checks or traps.
2404 if (STI.hasFPAC())
2405 return PtrauthCheckMode::Unchecked;
2406
2407 bool ShouldTrap = MF->getFunction().hasFnAttribute(Kind: "ptrauth-auth-traps");
2408 return ShouldTrap ? PtrauthCheckMode::Trap : PtrauthCheckMode::Poison;
2409}
2410
2411// We expand non-signing AUT* pseudo instructions into a sequence of the form
2412//
2413// ; 1. Authenticate Pointer
2414//
2415// or
2416//
2417// ; 1. Authenticate Pointer
2418// ; 2. Check that Pointer is valid, trap otherwise
2419//
2420// We expand AUT*PAC pseudo instructions into a sequence of the form
2421// (with addend only applied if Addend argument is given):
2422//
2423// ; 1. Authenticate Pointer
2424// ; 3. Apply addend and sign Pointer
2425//
2426// or
2427//
2428// ; 1. Authenticate Pointer
2429// ; 2. Check that Pointer is valid, trap otherwise
2430// ; 3. Apply addend and sign Pointer
2431//
2432// or
2433//
2434// ; 1. Authenticate Pointer
2435// ; 2. Check that Pointer is valid, jump to .Lon_failure otherwise
2436// ; 3. Apply addend and sign Pointer
2437// .Lon_failure:
2438//
2439void AArch64AsmPrinter::emitPtrauthAuthResign(
2440 Register Pointer, Register Scratch, PtrAuthSchema AuthSchema,
2441 std::optional<PtrAuthSchema> SignSchema, std::optional<int64_t> Addend,
2442 Value *DS) {
2443 const PtrauthCheckMode CheckMode = getCheckMode(MF);
2444 const bool IsAuthWithPC = AuthSchema.PCDisc.isValid();
2445 assert(!SignSchema || !SignSchema->PCDisc.isValid());
2446
2447 Register SignAddrDiscOrNone = SignSchema ? SignSchema->AddrDisc : Register();
2448
2449 // 1. Authenticate Pointer - this is the only common step.
2450 // It is more complex than signing because AUTI[AB]171615 may be used.
2451
2452 if (IsAuthWithPC) {
2453 assert(Pointer == AArch64::X17 && Scratch == AArch64::X16 &&
2454 "AUTPCPAC must use x17/x16 as Pointer/Scratch");
2455
2456 assert(AuthSchema.AddrDisc == AArch64::X16 &&
2457 "AUTPCPAC requires address discriminator in X16");
2458
2459 assert(AuthSchema.PCDisc == AArch64::X15 &&
2460 "AUTPCPAC requires PC discriminator in X15");
2461
2462 assert(AuthSchema.IntDisc == 0 && "AUTPCPAC does not support IntDisc");
2463
2464 assert((AuthSchema.Key == AArch64PACKey::IB ||
2465 AuthSchema.Key == AArch64PACKey::IA) &&
2466 "AUTPCPAC only supports AUT-ing with IA/IB");
2467
2468 if (!emitDeactivationSymbolRelocation(DS)) {
2469 unsigned AutOpc = (AuthSchema.Key == AArch64PACKey::IB)
2470 ? AArch64::AUTIB171615
2471 : AArch64::AUTIA171615;
2472 EmitToStreamer(Inst: MCInstBuilder(AutOpc));
2473 }
2474 } else {
2475 // emitPtrauthDiscriminator is allowed to clobber AuthSchema.AddrDisc as
2476 // long as it is not used past this point neither externally (the register
2477 // operand is "killed"), nor internally (it does not alias anything being
2478 // used later by this pseudo instruction).
2479 //
2480 // Note that, while rather unlikely, it is technically possible to use the
2481 // Pointer to compute its own discriminator.
2482 Register AUTDiscReg = emitPtrauthDiscriminator(
2483 Disc: AuthSchema.IntDisc, AddrDisc: AuthSchema.AddrDisc, ScratchReg: Scratch,
2484 MayClobberAddrDisc: AuthSchema.addrDiscIsKilledAndNoneOf(Regs: {Pointer, SignAddrDiscOrNone}));
2485 if (!emitDeactivationSymbolRelocation(DS))
2486 emitAUT(Key: AuthSchema.Key, Pointer, Disc: AUTDiscReg);
2487 }
2488
2489 // The other two steps are optional, define lambdas for them:
2490 // 2. Check that Pointer is valid, on failure jump to label or trap.
2491 auto EmitCheck = [&](MCSymbol *OnFailure = nullptr) {
2492 emitPtrauthCheckAuthenticatedValue(TestedReg: Pointer, ScratchReg: Scratch, Key: AuthSchema.Key,
2493 Method: AArch64PAuth::AuthCheckMethod::XPAC,
2494 OnFailure);
2495 };
2496 // 3. Apply addend and sign Pointer.
2497 auto EmitResignOnSuccess = [&]() {
2498 if (Addend.has_value())
2499 emitPtrauthApplyIndirectAddend(Pointer, Scratch, Addend: *Addend);
2500
2501 assert(Pointer != SignSchema->AddrDisc && "Pointer is early-clobbered");
2502 Register PACDiscReg =
2503 emitPtrauthDiscriminator(Disc: SignSchema->IntDisc, AddrDisc: SignSchema->AddrDisc,
2504 ScratchReg: Scratch, MayClobberAddrDisc: SignSchema->AddrDiscIsKilled);
2505 emitPAC(Key: SignSchema->Key, Pointer, Disc: PACDiscReg);
2506 };
2507
2508 // Emit checking and resigning as needed.
2509
2510 if (!SignSchema) {
2511 if (CheckMode == PtrauthCheckMode::Trap)
2512 EmitCheck();
2513 // For authentication-only pseudos, Poison is demoted to Unchecked.
2514 return;
2515 }
2516
2517 switch (CheckMode) {
2518 case PtrauthCheckMode::Unchecked:
2519 EmitResignOnSuccess();
2520 break;
2521 case PtrauthCheckMode::Trap:
2522 EmitCheck();
2523 EmitResignOnSuccess();
2524 break;
2525 case PtrauthCheckMode::Poison:
2526 MCSymbol *OnFailure = createTempSymbol(Name: "resign_end_");
2527 EmitCheck(OnFailure);
2528 EmitResignOnSuccess();
2529 OutStreamer->emitLabel(Symbol: OnFailure);
2530 break;
2531 }
2532}
2533
2534void AArch64AsmPrinter::emitPtrauthSign(const MachineInstr *MI) {
2535 Register Val = MI->getOperand(i: 1).getReg();
2536 auto Key = (AArch64PACKey::ID)MI->getOperand(i: 2).getImm();
2537 uint64_t Disc = MI->getOperand(i: 3).getImm();
2538 Register AddrDisc = MI->getOperand(i: 4).getReg();
2539 bool AddrDiscKilled = MI->getOperand(i: 4).isKill();
2540
2541 // As long as at least one of Val and AddrDisc is in GPR64noip, a scratch
2542 // register is available.
2543 Register ScratchReg = Val == AArch64::X16 ? AArch64::X17 : AArch64::X16;
2544 assert(ScratchReg != AddrDisc &&
2545 "Neither X16 nor X17 is available as a scratch register");
2546
2547 // Compute pac discriminator
2548 Register DiscReg = emitPtrauthDiscriminator(
2549 Disc, AddrDisc, ScratchReg, /*MayClobberAddrDisc=*/AddrDiscKilled);
2550
2551 if (emitDeactivationSymbolRelocation(DS: MI->getDeactivationSymbol()))
2552 return;
2553
2554 emitPAC(Key, Pointer: Val, Disc: DiscReg);
2555}
2556
2557void AArch64AsmPrinter::emitPtrauthBranch(const MachineInstr *MI) {
2558 bool IsCall = MI->getOpcode() == AArch64::BLRA;
2559 unsigned BrTarget = MI->getOperand(i: 0).getReg();
2560
2561 auto Key = (AArch64PACKey::ID)MI->getOperand(i: 1).getImm();
2562 uint64_t Disc = MI->getOperand(i: 2).getImm();
2563
2564 unsigned AddrDisc = MI->getOperand(i: 3).getReg();
2565
2566 // Make sure AddrDisc is solely used to compute the discriminator.
2567 // While hardly meaningful, it is still possible to describe an authentication
2568 // of a pointer against its own value (instead of storage address) with
2569 // intrinsics, so use report_fatal_error instead of assert.
2570 if (BrTarget == AddrDisc)
2571 report_fatal_error(reason: "Branch target is signed with its own value");
2572
2573 // If we are printing BLRA pseudo, try to save one MOV by making use of the
2574 // fact that x16 and x17 are described as clobbered by the MI instruction and
2575 // AddrDisc is not used as any other input.
2576 //
2577 // Back in the day, emitPtrauthDiscriminator was restricted to only returning
2578 // either x16 or x17, meaning the returned register is always among the
2579 // implicit-def'ed registers of BLRA pseudo. Now this property can be violated
2580 // if isX16X17Safer predicate is false, thus manually check if AddrDisc is
2581 // among x16 and x17 to prevent clobbering unexpected registers.
2582 //
2583 // Unlike BLRA, BRA pseudo is used to perform computed goto, and thus not
2584 // declared as clobbering x16/x17.
2585 //
2586 // FIXME: Make use of `killed` flags and register masks instead.
2587 bool AddrDiscIsImplicitDef =
2588 IsCall && (AddrDisc == AArch64::X16 || AddrDisc == AArch64::X17);
2589 Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, ScratchReg: AArch64::X17,
2590 MayClobberAddrDisc: AddrDiscIsImplicitDef);
2591 emitBLRA(IsCall, Key, Target: BrTarget, Disc: DiscReg);
2592}
2593
2594void AArch64AsmPrinter::emitAddImm(MCRegister Reg, int64_t Addend,
2595 MCRegister Tmp) {
2596 if (Addend != 0) {
2597 const uint64_t AbsOffset = (Addend > 0 ? Addend : -((uint64_t)Addend));
2598 const bool IsNeg = Addend < 0;
2599 if (isUInt<24>(x: AbsOffset)) {
2600 for (int BitPos = 0; BitPos != 24 && (AbsOffset >> BitPos);
2601 BitPos += 12) {
2602 EmitToStreamer(
2603 Inst: MCInstBuilder(IsNeg ? AArch64::SUBXri : AArch64::ADDXri)
2604 .addReg(Reg)
2605 .addReg(Reg)
2606 .addImm(Val: (AbsOffset >> BitPos) & 0xfff)
2607 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: BitPos)));
2608 }
2609 } else {
2610 const uint64_t UAddend = Addend;
2611 EmitToStreamer(Inst: MCInstBuilder(IsNeg ? AArch64::MOVNXi : AArch64::MOVZXi)
2612 .addReg(Reg: Tmp)
2613 .addImm(Val: (IsNeg ? ~UAddend : UAddend) & 0xffff)
2614 .addImm(/*shift=*/Val: 0));
2615 auto NeedMovk = [IsNeg, UAddend](int BitPos) -> bool {
2616 assert(BitPos == 16 || BitPos == 32 || BitPos == 48);
2617 uint64_t Shifted = UAddend >> BitPos;
2618 if (!IsNeg)
2619 return Shifted != 0;
2620 for (int I = 0; I != 64 - BitPos; I += 16)
2621 if (((Shifted >> I) & 0xffff) != 0xffff)
2622 return true;
2623 return false;
2624 };
2625 for (int BitPos = 16; BitPos != 64 && NeedMovk(BitPos); BitPos += 16)
2626 emitMOVK(Dest: Tmp, Imm: (UAddend >> BitPos) & 0xffff, Shift: BitPos);
2627
2628 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXrs)
2629 .addReg(Reg)
2630 .addReg(Reg)
2631 .addReg(Reg: Tmp)
2632 .addImm(/*shift=*/Val: 0));
2633 }
2634 }
2635}
2636
2637void AArch64AsmPrinter::emitAddress(MCRegister Reg, const MCExpr *Expr,
2638 MCRegister Tmp, bool DSOLocal,
2639 const MCSubtargetInfo &STI) {
2640 MCValue Val;
2641 if (!Expr->evaluateAsRelocatable(Res&: Val, Asm: nullptr))
2642 report_fatal_error(reason: "emitAddress could not evaluate");
2643 if (DSOLocal) {
2644 EmitToStreamer(
2645 Inst: MCInstBuilder(AArch64::ADRP)
2646 .addReg(Reg)
2647 .addExpr(Val: MCSpecifierExpr::create(Expr, S: AArch64::S_ABS_PAGE,
2648 Ctx&: OutStreamer->getContext())));
2649 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
2650 .addReg(Reg)
2651 .addReg(Reg)
2652 .addExpr(Val: MCSpecifierExpr::create(
2653 Expr, S: AArch64::S_LO12, Ctx&: OutStreamer->getContext()))
2654 .addImm(Val: 0));
2655 } else {
2656 auto *SymRef =
2657 MCSymbolRefExpr::create(Symbol: Val.getAddSym(), Ctx&: OutStreamer->getContext());
2658 EmitToStreamer(
2659 Inst: MCInstBuilder(AArch64::ADRP)
2660 .addReg(Reg)
2661 .addExpr(Val: MCSpecifierExpr::create(Expr: SymRef, S: AArch64::S_GOT_PAGE,
2662 Ctx&: OutStreamer->getContext())));
2663 EmitToStreamer(
2664 Inst: MCInstBuilder(AArch64::LDRXui)
2665 .addReg(Reg)
2666 .addReg(Reg)
2667 .addExpr(Val: MCSpecifierExpr::create(Expr: SymRef, S: AArch64::S_GOT_LO12,
2668 Ctx&: OutStreamer->getContext())));
2669 emitAddImm(Reg, Addend: Val.getConstant(), Tmp);
2670 }
2671}
2672
2673static bool targetSupportsIRelativeRelocation(const Triple &TT) {
2674 // IFUNCs are ELF-only.
2675 if (!TT.isOSBinFormatELF())
2676 return false;
2677
2678 // IFUNCs are supported on glibc, bionic, and some but not all of the BSDs.
2679 return TT.isOSGlibc() || TT.isAndroid() || TT.isOSFreeBSD() ||
2680 TT.isOSDragonFly() || TT.isOSNetBSD();
2681}
2682
2683// Emit an ifunc resolver that returns a signed pointer to the specified target,
2684// and return a FUNCINIT reference to the resolver. In the linked binary, this
2685// function becomes the target of an IRELATIVE relocation. This resolver is used
2686// to relocate signed pointers in global variable initializers in special cases
2687// where the standard R_AARCH64_AUTH_ABS64 relocation would not work.
2688//
2689// Example (signed null pointer, not address discriminated):
2690//
2691// .8byte .Lpauth_ifunc0
2692// .pushsection .text.startup,"ax",@progbits
2693// .Lpauth_ifunc0:
2694// mov x0, #0
2695// mov x1, #12345
2696// b __emupac_pacda
2697//
2698// Example (signed null pointer, address discriminated):
2699//
2700// .Ltmp:
2701// .8byte .Lpauth_ifunc0
2702// .pushsection .text.startup,"ax",@progbits
2703// .Lpauth_ifunc0:
2704// mov x0, #0
2705// adrp x1, .Ltmp
2706// add x1, x1, :lo12:.Ltmp
2707// b __emupac_pacda
2708// .popsection
2709//
2710// Example (signed pointer to symbol, not address discriminated):
2711//
2712// .Ltmp:
2713// .8byte .Lpauth_ifunc0
2714// .pushsection .text.startup,"ax",@progbits
2715// .Lpauth_ifunc0:
2716// adrp x0, symbol
2717// add x0, x0, :lo12:symbol
2718// mov x1, #12345
2719// b __emupac_pacda
2720// .popsection
2721//
2722// Example (signed null pointer, not address discriminated, with deactivation
2723// symbol ds):
2724//
2725// .8byte .Lpauth_ifunc0
2726// .pushsection .text.startup,"ax",@progbits
2727// .Lpauth_ifunc0:
2728// mov x0, #0
2729// mov x1, #12345
2730// .reloc ., R_AARCH64_PATCHINST, ds
2731// b __emupac_pacda
2732// ret
2733// .popsection
2734const MCExpr *AArch64AsmPrinter::emitPAuthRelocationAsIRelative(
2735 const MCExpr *Target, uint64_t Disc, AArch64PACKey::ID KeyID,
2736 bool HasAddressDiversity, bool IsDSOLocal, const MCExpr *DSExpr) {
2737 const Triple &TT = TM.getTargetTriple();
2738
2739 // We only emit an IRELATIVE relocation if the target supports IRELATIVE.
2740 if (!targetSupportsIRelativeRelocation(TT))
2741 return nullptr;
2742
2743 // For now, only the DA key is supported.
2744 if (KeyID != AArch64PACKey::DA)
2745 return nullptr;
2746
2747 // AArch64Subtarget is huge, so heap allocate it so we don't run out of stack
2748 // space.
2749 auto STI = std::make_unique<AArch64Subtarget>(
2750 args: TT, args: TM.getTargetCPU(), args: TM.getTargetCPU(), args: TM.getTargetFeatureString(), args&: TM,
2751 args: true);
2752 // This may be called while emitting a function, whose subtarget is restored.
2753 SaveAndRestore<const AArch64Subtarget *> SavedSTI(this->STI, STI.get());
2754
2755 MCSymbol *Place = OutStreamer->getContext().createTempSymbol();
2756 OutStreamer->emitLabel(Symbol: Place);
2757 OutStreamer->pushSection();
2758
2759 const MCSymbolELF *Group =
2760 static_cast<MCSectionELF *>(OutStreamer->getCurrentSectionOnly())
2761 ->getGroup();
2762 auto Flags = ELF::SHF_ALLOC | ELF::SHF_EXECINSTR;
2763 if (Group)
2764 Flags |= ELF::SHF_GROUP;
2765 OutStreamer->switchSection(Section: OutStreamer->getContext().getELFSection(
2766 Section: ".text.startup", Type: ELF::SHT_PROGBITS, Flags, EntrySize: 0, Group, IsComdat: true,
2767 UniqueID: Group ? MCSection::NonUniqueID : PAuthIFuncNextUniqueID++, LinkedToSym: nullptr));
2768
2769 MCSymbol *IRelativeSym =
2770 OutStreamer->getContext().createLinkerPrivateSymbol(Name: "pauth_ifunc");
2771 OutStreamer->emitLabel(Symbol: IRelativeSym);
2772 if (isa<MCConstantExpr>(Val: Target)) {
2773 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::MOVZXi)
2774 .addReg(Reg: AArch64::X0)
2775 .addExpr(Val: Target)
2776 .addImm(Val: 0),
2777 STI: *STI);
2778 } else {
2779 emitAddress(Reg: AArch64::X0, Expr: Target, Tmp: AArch64::X16, DSOLocal: IsDSOLocal, STI: *STI);
2780 }
2781 if (HasAddressDiversity) {
2782 auto *PlacePlusDisc = MCBinaryExpr::createAdd(
2783 LHS: MCSymbolRefExpr::create(Symbol: Place, Ctx&: OutStreamer->getContext()),
2784 RHS: MCConstantExpr::create(Value: Disc, Ctx&: OutStreamer->getContext()),
2785 Ctx&: OutStreamer->getContext());
2786 emitAddress(Reg: AArch64::X1, Expr: PlacePlusDisc, Tmp: AArch64::X16, /*IsDSOLocal=*/DSOLocal: true,
2787 STI: *STI);
2788 } else {
2789 if (!isUInt<16>(x: Disc)) {
2790 OutContext.reportError(L: SMLoc(), Msg: "AArch64 PAC Discriminator '" +
2791 Twine(Disc) +
2792 "' out of range [0, 0xFFFF]");
2793 }
2794 emitMOVZ(Dest: AArch64::X1, Imm: Disc, Shift: 0);
2795 }
2796
2797 if (DSExpr) {
2798 MCSymbol *PrePACInst = OutStreamer->getContext().createTempSymbol();
2799 OutStreamer->emitLabel(Symbol: PrePACInst);
2800
2801 auto *PrePACInstExpr =
2802 MCSymbolRefExpr::create(Symbol: PrePACInst, Ctx&: OutStreamer->getContext());
2803 OutStreamer->emitRelocDirective(Offset: *PrePACInstExpr, Name: "R_AARCH64_PATCHINST",
2804 Expr: DSExpr, Loc: SMLoc());
2805 }
2806
2807 // We don't know the subtarget because this is being emitted for a global
2808 // initializer. Because the performance of IFUNC resolvers is unimportant, we
2809 // always call the EmuPAC runtime, which will end up using the PAC instruction
2810 // if the target supports PAC.
2811 MCSymbol *EmuPAC =
2812 OutStreamer->getContext().getOrCreateSymbol(Name: "__emupac_pacda");
2813 const MCSymbolRefExpr *EmuPACRef =
2814 MCSymbolRefExpr::create(Symbol: EmuPAC, Ctx&: OutStreamer->getContext());
2815 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::B).addExpr(Val: EmuPACRef),
2816 STI: *STI);
2817
2818 // We need a RET despite the above tail call because the deactivation symbol
2819 // may replace the tail call with a NOP.
2820 if (DSExpr)
2821 OutStreamer->emitInstruction(
2822 Inst: MCInstBuilder(AArch64::RET).addReg(Reg: AArch64::LR), STI: *STI);
2823 OutStreamer->popSection();
2824
2825 return MCSpecifierExpr::create(
2826 Expr: MCSymbolRefExpr::create(Symbol: IRelativeSym, Ctx&: OutStreamer->getContext()),
2827 S: AArch64::S_FUNCINIT, Ctx&: OutStreamer->getContext());
2828}
2829
2830const MCExpr *
2831AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) {
2832 MCContext &Ctx = OutContext;
2833
2834 // Figure out the base symbol and the addend, if any.
2835 APInt Offset(64, 0);
2836 const Value *BaseGV = CPA.getPointer()->stripAndAccumulateConstantOffsets(
2837 DL: getDataLayout(), Offset, /*AllowNonInbounds=*/true);
2838
2839 auto *BaseGVB = dyn_cast<GlobalValue>(Val: BaseGV);
2840
2841 const MCExpr *Sym;
2842 if (BaseGVB) {
2843 // If there is an addend, turn that into the appropriate MCExpr.
2844 Sym = MCSymbolRefExpr::create(Symbol: getSymbol(GV: BaseGVB), Ctx);
2845 if (Offset.sgt(RHS: 0))
2846 Sym = MCBinaryExpr::createAdd(
2847 LHS: Sym, RHS: MCConstantExpr::create(Value: Offset.getSExtValue(), Ctx), Ctx);
2848 else if (Offset.slt(RHS: 0))
2849 Sym = MCBinaryExpr::createSub(
2850 LHS: Sym, RHS: MCConstantExpr::create(Value: (-Offset).getSExtValue(), Ctx), Ctx);
2851 } else if (isa<ConstantPointerNull>(Val: BaseGV)) {
2852 Sym = MCConstantExpr::create(Value: Offset.getSExtValue(), Ctx);
2853 } else {
2854 reportFatalUsageError(reason: "unsupported constant expression in ptrauth pointer");
2855 }
2856
2857 const MCExpr *DSExpr = nullptr;
2858 if (auto *DS = dyn_cast<GlobalValue>(Val: CPA.getDeactivationSymbol())) {
2859 if (isa<GlobalAlias>(Val: DS))
2860 return Sym;
2861 DSExpr = MCSymbolRefExpr::create(Symbol: getSymbol(GV: DS), Ctx);
2862 }
2863
2864 uint64_t KeyID = CPA.getKey()->getZExtValue();
2865 // We later rely on valid KeyID value in AArch64PACKeyIDToString call from
2866 // AArch64AuthMCExpr::printImpl, so fail fast.
2867 if (KeyID > AArch64PACKey::LAST) {
2868 CPA.getContext().emitError(ErrorStr: "AArch64 PAC Key ID '" + Twine(KeyID) +
2869 "' out of range [0, " +
2870 Twine((unsigned)AArch64PACKey::LAST) + "]");
2871 KeyID = 0;
2872 }
2873
2874 uint64_t Disc = CPA.getDiscriminator()->getZExtValue();
2875
2876 // Check if we can represent this with an IRELATIVE and emit it if so.
2877 if (auto *IFuncSym = emitPAuthRelocationAsIRelative(
2878 Target: Sym, Disc, KeyID: AArch64PACKey::ID(KeyID), HasAddressDiversity: CPA.hasAddressDiscriminator(),
2879 IsDSOLocal: BaseGVB && BaseGVB->isDSOLocal(), DSExpr))
2880 return IFuncSym;
2881
2882 if (!isUInt<16>(x: Disc)) {
2883 CPA.getContext().emitError(ErrorStr: "AArch64 PAC Discriminator '" + Twine(Disc) +
2884 "' out of range [0, 0xFFFF]");
2885 Disc = 0;
2886 }
2887
2888 if (DSExpr)
2889 report_fatal_error(reason: "deactivation symbols unsupported in constant "
2890 "expressions on this target");
2891
2892 // Finally build the complete @AUTH expr.
2893 return AArch64AuthMCExpr::create(Expr: Sym, Discriminator: Disc, Key: AArch64PACKey::ID(KeyID),
2894 HasAddressDiversity: CPA.hasAddressDiscriminator(), Ctx);
2895}
2896
2897void AArch64AsmPrinter::LowerLOADauthptrstatic(const MachineInstr &MI) {
2898 unsigned DstReg = MI.getOperand(i: 0).getReg();
2899 const MachineOperand &GAOp = MI.getOperand(i: 1);
2900 const uint64_t KeyC = MI.getOperand(i: 2).getImm();
2901 assert(KeyC <= AArch64PACKey::LAST &&
2902 "key is out of range [0, AArch64PACKey::LAST]");
2903 const auto Key = (AArch64PACKey::ID)KeyC;
2904 const uint64_t Disc = MI.getOperand(i: 3).getImm();
2905 assert(isUInt<16>(Disc) &&
2906 "constant discriminator is out of range [0, 0xffff]");
2907
2908 // Emit instruction sequence like the following:
2909 // ADRP x16, symbol$auth_ptr$key$disc
2910 // LDR x16, [x16, :lo12:symbol$auth_ptr$key$disc]
2911 //
2912 // Where the $auth_ptr$ symbol is the stub slot containing the signed pointer
2913 // to symbol.
2914 MCSymbol *AuthPtrStubSym;
2915 if (TM.getTargetTriple().isOSBinFormatELF()) {
2916 const auto &TLOF =
2917 static_cast<const AArch64_ELFTargetObjectFile &>(getObjFileLowering());
2918
2919 assert(GAOp.getOffset() == 0 &&
2920 "non-zero offset for $auth_ptr$ stub slots is not supported");
2921 const MCSymbol *GASym = TM.getSymbol(GV: GAOp.getGlobal());
2922 AuthPtrStubSym = TLOF.getAuthPtrSlotSymbol(TM, MMI, RawSym: GASym, Key, Discriminator: Disc);
2923 } else {
2924 assert(TM.getTargetTriple().isOSBinFormatMachO() &&
2925 "LOADauthptrstatic is implemented only for MachO/ELF");
2926
2927 const auto &TLOF = static_cast<const AArch64_MachoTargetObjectFile &>(
2928 getObjFileLowering());
2929
2930 assert(GAOp.getOffset() == 0 &&
2931 "non-zero offset for $auth_ptr$ stub slots is not supported");
2932 const MCSymbol *GASym = TM.getSymbol(GV: GAOp.getGlobal());
2933 AuthPtrStubSym = TLOF.getAuthPtrSlotSymbol(TM, MMI, RawSym: GASym, Key, Discriminator: Disc);
2934 }
2935
2936 MachineOperand StubMOHi =
2937 MachineOperand::CreateMCSymbol(Sym: AuthPtrStubSym, TargetFlags: AArch64II::MO_PAGE);
2938 MachineOperand StubMOLo = MachineOperand::CreateMCSymbol(
2939 Sym: AuthPtrStubSym, TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2940 MCOperand StubMCHi, StubMCLo;
2941
2942 MCInstLowering.lowerOperand(MO: StubMOHi, MCOp&: StubMCHi);
2943 MCInstLowering.lowerOperand(MO: StubMOLo, MCOp&: StubMCLo);
2944
2945 EmitToStreamer(
2946 S&: *OutStreamer,
2947 Inst: MCInstBuilder(AArch64::ADRP).addReg(Reg: DstReg).addOperand(Op: StubMCHi));
2948
2949 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::LDRXui)
2950 .addReg(Reg: DstReg)
2951 .addReg(Reg: DstReg)
2952 .addOperand(Op: StubMCLo));
2953}
2954
2955void AArch64AsmPrinter::LowerMOVaddrPAC(const MachineInstr &MI) {
2956 const bool IsGOTLoad = MI.getOpcode() == AArch64::LOADgotPAC;
2957 const bool IsELFSignedGOT = MI.getParent()
2958 ->getParent()
2959 ->getInfo<AArch64FunctionInfo>()
2960 ->hasELFSignedGOT();
2961 MachineOperand GAOp = MI.getOperand(i: 0);
2962 const uint64_t KeyC = MI.getOperand(i: 1).getImm();
2963 assert(KeyC <= AArch64PACKey::LAST &&
2964 "key is out of range [0, AArch64PACKey::LAST]");
2965 const auto Key = (AArch64PACKey::ID)KeyC;
2966 const unsigned AddrDisc = MI.getOperand(i: 2).getReg();
2967 const uint64_t Disc = MI.getOperand(i: 3).getImm();
2968
2969 const int64_t Offset = GAOp.getOffset();
2970 GAOp.setOffset(0);
2971
2972 // Emit:
2973 // target materialization:
2974 // - via GOT:
2975 // - unsigned GOT:
2976 // adrp x16, :got:target
2977 // ldr x16, [x16, :got_lo12:target]
2978 // add offset to x16 if offset != 0
2979 // - ELF signed GOT:
2980 // adrp x17, :got:target
2981 // add x17, x17, :got_auth_lo12:target
2982 // ldr x16, [x17]
2983 // aut{i|d}a x16, x17
2984 // check+trap sequence (if no FPAC)
2985 // add offset to x16 if offset != 0
2986 //
2987 // - direct:
2988 // adrp x16, target
2989 // add x16, x16, :lo12:target
2990 // add offset to x16 if offset != 0
2991 //
2992 // add offset to x16:
2993 // - abs(offset) fits 24 bits:
2994 // add/sub x16, x16, #<offset>[, #lsl 12] (up to 2 instructions)
2995 // - abs(offset) does not fit 24 bits:
2996 // - offset < 0:
2997 // movn+movk sequence filling x17 register with the offset (up to 4
2998 // instructions)
2999 // add x16, x16, x17
3000 // - offset > 0:
3001 // movz+movk sequence filling x17 register with the offset (up to 4
3002 // instructions)
3003 // add x16, x16, x17
3004 //
3005 // signing:
3006 // - 0 discriminator:
3007 // paciza x16
3008 // - Non-0 discriminator, no address discriminator:
3009 // mov x17, #Disc
3010 // pacia x16, x17
3011 // - address discriminator (with potentially folded immediate discriminator):
3012 // pacia x16, xAddrDisc
3013
3014 MachineOperand GAMOHi(GAOp), GAMOLo(GAOp);
3015 MCOperand GAMCHi, GAMCLo;
3016
3017 GAMOHi.setTargetFlags(AArch64II::MO_PAGE);
3018 GAMOLo.setTargetFlags(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
3019 if (IsGOTLoad) {
3020 GAMOHi.addTargetFlag(F: AArch64II::MO_GOT);
3021 GAMOLo.addTargetFlag(F: AArch64II::MO_GOT);
3022 }
3023
3024 MCInstLowering.lowerOperand(MO: GAMOHi, MCOp&: GAMCHi);
3025 MCInstLowering.lowerOperand(MO: GAMOLo, MCOp&: GAMCLo);
3026
3027 EmitToStreamer(
3028 Inst: MCInstBuilder(AArch64::ADRP)
3029 .addReg(Reg: IsGOTLoad && IsELFSignedGOT ? AArch64::X17 : AArch64::X16)
3030 .addOperand(Op: GAMCHi));
3031
3032 if (IsGOTLoad) {
3033 if (IsELFSignedGOT) {
3034 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
3035 .addReg(Reg: AArch64::X17)
3036 .addReg(Reg: AArch64::X17)
3037 .addOperand(Op: GAMCLo)
3038 .addImm(Val: 0));
3039
3040 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
3041 .addReg(Reg: AArch64::X16)
3042 .addReg(Reg: AArch64::X17)
3043 .addImm(Val: 0));
3044
3045 assert(GAOp.isGlobal());
3046 assert(GAOp.getGlobal()->getValueType() != nullptr);
3047
3048 bool IsFunctionTy = GAOp.getGlobal()->getValueType()->isFunctionTy();
3049 auto AuthKey = IsFunctionTy ? AArch64PACKey::IA : AArch64PACKey::DA;
3050 emitAUT(Key: AuthKey, Pointer: AArch64::X16, Disc: AArch64::X17);
3051
3052 if (!STI->hasFPAC())
3053 emitPtrauthCheckAuthenticatedValue(TestedReg: AArch64::X16, ScratchReg: AArch64::X17, Key: AuthKey,
3054 Method: AArch64PAuth::AuthCheckMethod::XPAC);
3055 } else {
3056 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
3057 .addReg(Reg: AArch64::X16)
3058 .addReg(Reg: AArch64::X16)
3059 .addOperand(Op: GAMCLo));
3060 }
3061 } else {
3062 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
3063 .addReg(Reg: AArch64::X16)
3064 .addReg(Reg: AArch64::X16)
3065 .addOperand(Op: GAMCLo)
3066 .addImm(Val: 0));
3067 }
3068
3069 emitAddImm(Reg: AArch64::X16, Addend: Offset, Tmp: AArch64::X17);
3070 Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, ScratchReg: AArch64::X17);
3071
3072 emitPAC(Key, Pointer: AArch64::X16, Disc: DiscReg);
3073}
3074
3075void AArch64AsmPrinter::LowerLOADgotAUTH(const MachineInstr &MI) {
3076 Register DstReg = MI.getOperand(i: 0).getReg();
3077 Register AuthResultReg = STI->hasFPAC() ? DstReg : AArch64::X16;
3078 const MachineOperand &GAMO = MI.getOperand(i: 1);
3079 assert(GAMO.getOffset() == 0);
3080
3081 if (MI.getMF()->getTarget().getCodeModel() == CodeModel::Tiny) {
3082 MCOperand GAMC;
3083 MCInstLowering.lowerOperand(MO: GAMO, MCOp&: GAMC);
3084 EmitToStreamer(
3085 Inst: MCInstBuilder(AArch64::ADR).addReg(Reg: AArch64::X17).addOperand(Op: GAMC));
3086 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
3087 .addReg(Reg: AuthResultReg)
3088 .addReg(Reg: AArch64::X17)
3089 .addImm(Val: 0));
3090 } else {
3091 MachineOperand GAHiOp(GAMO);
3092 MachineOperand GALoOp(GAMO);
3093 GAHiOp.addTargetFlag(F: AArch64II::MO_PAGE);
3094 GALoOp.addTargetFlag(F: AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
3095
3096 MCOperand GAMCHi, GAMCLo;
3097 MCInstLowering.lowerOperand(MO: GAHiOp, MCOp&: GAMCHi);
3098 MCInstLowering.lowerOperand(MO: GALoOp, MCOp&: GAMCLo);
3099
3100 EmitToStreamer(
3101 Inst: MCInstBuilder(AArch64::ADRP).addReg(Reg: AArch64::X17).addOperand(Op: GAMCHi));
3102
3103 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
3104 .addReg(Reg: AArch64::X17)
3105 .addReg(Reg: AArch64::X17)
3106 .addOperand(Op: GAMCLo)
3107 .addImm(Val: 0));
3108
3109 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
3110 .addReg(Reg: AuthResultReg)
3111 .addReg(Reg: AArch64::X17)
3112 .addImm(Val: 0));
3113 }
3114
3115 assert(GAMO.isGlobal());
3116 MCSymbol *UndefWeakSym;
3117 if (GAMO.getGlobal()->hasExternalWeakLinkage()) {
3118 UndefWeakSym = createTempSymbol(Name: "undef_weak");
3119 EmitToStreamer(
3120 Inst: MCInstBuilder(AArch64::CBZX)
3121 .addReg(Reg: AuthResultReg)
3122 .addExpr(Val: MCSymbolRefExpr::create(Symbol: UndefWeakSym, Ctx&: OutContext)));
3123 }
3124
3125 assert(GAMO.getGlobal()->getValueType() != nullptr);
3126
3127 bool IsFunctionTy = GAMO.getGlobal()->getValueType()->isFunctionTy();
3128 auto AuthKey = IsFunctionTy ? AArch64PACKey::IA : AArch64PACKey::DA;
3129 emitAUT(Key: AuthKey, Pointer: AuthResultReg, Disc: AArch64::X17);
3130
3131 if (GAMO.getGlobal()->hasExternalWeakLinkage())
3132 OutStreamer->emitLabel(Symbol: UndefWeakSym);
3133
3134 if (!STI->hasFPAC()) {
3135 emitPtrauthCheckAuthenticatedValue(TestedReg: AuthResultReg, ScratchReg: AArch64::X17, Key: AuthKey,
3136 Method: AArch64PAuth::AuthCheckMethod::XPAC);
3137
3138 emitMovXReg(Dest: DstReg, Src: AuthResultReg);
3139 }
3140}
3141
3142const MCExpr *
3143AArch64AsmPrinter::lowerBlockAddressConstant(const BlockAddress &BA) {
3144 const MCExpr *BAE = AsmPrinter::lowerBlockAddressConstant(BA);
3145 const Function &Fn = *BA.getFunction();
3146
3147 if (std::optional<uint16_t> BADisc =
3148 STI->getPtrAuthBlockAddressDiscriminatorIfEnabled(ParentFn: Fn))
3149 return AArch64AuthMCExpr::create(Expr: BAE, Discriminator: *BADisc, Key: AArch64PACKey::IA,
3150 /*HasAddressDiversity=*/false, Ctx&: OutContext);
3151
3152 return BAE;
3153}
3154
3155void AArch64AsmPrinter::emitCBPseudoExpansion(const MachineInstr *MI) {
3156 bool IsImm = false;
3157 unsigned Width = 0;
3158
3159 switch (MI->getOpcode()) {
3160 default:
3161 llvm_unreachable("This is not a CB pseudo instruction");
3162 case AArch64::CBBAssertExt:
3163 IsImm = false;
3164 Width = 8;
3165 break;
3166 case AArch64::CBHAssertExt:
3167 IsImm = false;
3168 Width = 16;
3169 break;
3170 case AArch64::CBWPrr:
3171 Width = 32;
3172 break;
3173 case AArch64::CBXPrr:
3174 Width = 64;
3175 break;
3176 case AArch64::CBWPri:
3177 IsImm = true;
3178 Width = 32;
3179 break;
3180 case AArch64::CBXPri:
3181 IsImm = true;
3182 Width = 64;
3183 break;
3184 }
3185
3186 AArch64CC::CondCode CC =
3187 static_cast<AArch64CC::CondCode>(MI->getOperand(i: 0).getImm());
3188 bool NeedsRegSwap = false;
3189 bool NeedsImmDec = false;
3190 bool NeedsImmInc = false;
3191
3192#define GET_CB_OPC(IsImm, Width, ImmCond, RegCond) \
3193 (IsImm \
3194 ? (Width == 32 ? AArch64::CB##ImmCond##Wri : AArch64::CB##ImmCond##Xri) \
3195 : (Width == 8 \
3196 ? AArch64::CBB##RegCond##Wrr \
3197 : (Width == 16 ? AArch64::CBH##RegCond##Wrr \
3198 : (Width == 32 ? AArch64::CB##RegCond##Wrr \
3199 : AArch64::CB##RegCond##Xrr))))
3200 unsigned MCOpC;
3201
3202 // Decide if we need to either swap register operands or increment/decrement
3203 // immediate operands
3204 switch (CC) {
3205 default:
3206 llvm_unreachable("Invalid CB condition code");
3207 case AArch64CC::EQ:
3208 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ EQ, /* Reg-Reg */ EQ);
3209 break;
3210 case AArch64CC::NE:
3211 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ NE, /* Reg-Reg */ NE);
3212 break;
3213 case AArch64CC::HS:
3214 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ HI, /* Reg-Reg */ HS);
3215 NeedsImmDec = IsImm;
3216 break;
3217 case AArch64CC::LO:
3218 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ LO, /* Reg-Reg */ HI);
3219 NeedsRegSwap = !IsImm;
3220 break;
3221 case AArch64CC::HI:
3222 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ HI, /* Reg-Reg */ HI);
3223 break;
3224 case AArch64CC::LS:
3225 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ LO, /* Reg-Reg */ HS);
3226 NeedsRegSwap = !IsImm;
3227 NeedsImmInc = IsImm;
3228 break;
3229 case AArch64CC::GE:
3230 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ GT, /* Reg-Reg */ GE);
3231 NeedsImmDec = IsImm;
3232 break;
3233 case AArch64CC::LT:
3234 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ LT, /* Reg-Reg */ GT);
3235 NeedsRegSwap = !IsImm;
3236 break;
3237 case AArch64CC::GT:
3238 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ GT, /* Reg-Reg */ GT);
3239 break;
3240 case AArch64CC::LE:
3241 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ LT, /* Reg-Reg */ GE);
3242 NeedsRegSwap = !IsImm;
3243 NeedsImmInc = IsImm;
3244 break;
3245 }
3246#undef GET_CB_OPC
3247
3248 MCInst Inst;
3249 Inst.setOpcode(MCOpC);
3250
3251 MCOperand Lhs, Rhs, Trgt;
3252 lowerOperand(MO: MI->getOperand(i: 1), MCOp&: Lhs);
3253 lowerOperand(MO: MI->getOperand(i: 2), MCOp&: Rhs);
3254 lowerOperand(MO: MI->getOperand(i: 3), MCOp&: Trgt);
3255
3256 // Now swap, increment or decrement
3257 if (NeedsRegSwap) {
3258 assert(Lhs.isReg() && "Expected register operand for CB");
3259 assert(Rhs.isReg() && "Expected register operand for CB");
3260 Inst.addOperand(Op: Rhs);
3261 Inst.addOperand(Op: Lhs);
3262 } else if (NeedsImmDec) {
3263 Rhs.setImm(Rhs.getImm() - 1);
3264 Inst.addOperand(Op: Lhs);
3265 Inst.addOperand(Op: Rhs);
3266 } else if (NeedsImmInc) {
3267 Rhs.setImm(Rhs.getImm() + 1);
3268 Inst.addOperand(Op: Lhs);
3269 Inst.addOperand(Op: Rhs);
3270 } else {
3271 Inst.addOperand(Op: Lhs);
3272 Inst.addOperand(Op: Rhs);
3273 }
3274
3275 assert((!IsImm || (Rhs.getImm() >= 0 && Rhs.getImm() < 64)) &&
3276 "CB immediate operand out-of-bounds");
3277
3278 Inst.addOperand(Op: Trgt);
3279 EmitToStreamer(S&: *OutStreamer, Inst);
3280}
3281
3282void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI) {
3283
3284 unsigned StOpc;
3285 unsigned Relaxed = MI->getOperand(i: 2).getImm();
3286 assert(Relaxed < 2 && "Atomic hint relaxed immediate out-of-bounds");
3287 switch (MI->getOpcode()) {
3288 case AArch64::ATOMIC_STORE_HINT_B:
3289 StOpc = Relaxed ? AArch64::STRBBui : AArch64::STLRB;
3290 break;
3291 case AArch64::ATOMIC_STORE_HINT_H:
3292 StOpc = Relaxed ? AArch64::STRHHui : AArch64::STLRH;
3293 break;
3294 case AArch64::ATOMIC_STORE_HINT_W:
3295 StOpc = Relaxed ? AArch64::STRWui : AArch64::STLRW;
3296 break;
3297 case AArch64::ATOMIC_STORE_HINT_X:
3298 StOpc = Relaxed ? AArch64::STRXui : AArch64::STLRX;
3299 break;
3300 default:
3301 llvm_unreachable("Unexpected atomic hint size.");
3302 }
3303
3304 EmitToStreamer(
3305 Inst: MCInstBuilder(AArch64::HINT).addImm(Val: MI->getOperand(i: 3).getImm()));
3306
3307 MCInst Store;
3308 Store.setOpcode(StOpc);
3309 Store.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg()));
3310 Store.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 1).getReg()));
3311 Store.setFlags(MI->getFlags());
3312 if (Relaxed)
3313 Store.addOperand(Op: MCOperand::createImm(Val: 0));
3314 EmitToStreamer(S&: *OutStreamer, Inst: Store);
3315}
3316
3317void AArch64AsmPrinter::emitAtomicHintPseudoExpansionRO(
3318 const MachineInstr *MI) {
3319 unsigned StOpc;
3320 assert(MI->getOperand(5).getImm() == 1 &&
3321 "Atomic store addressing mode only supports relaxed stores");
3322
3323 switch (MI->getOpcode()) {
3324 case AArch64::ATOMIC_STORE_HINT_BroW:
3325 StOpc = AArch64::STRBBroW;
3326 break;
3327 case AArch64::ATOMIC_STORE_HINT_HroW:
3328 StOpc = AArch64::STRHHroW;
3329 break;
3330 case AArch64::ATOMIC_STORE_HINT_WroW:
3331 StOpc = AArch64::STRWroW;
3332 break;
3333 case AArch64::ATOMIC_STORE_HINT_XroW:
3334 StOpc = AArch64::STRXroW;
3335 break;
3336 case AArch64::ATOMIC_STORE_HINT_SroW:
3337 StOpc = AArch64::STRSroW;
3338 break;
3339 case AArch64::ATOMIC_STORE_HINT_DroW:
3340 StOpc = AArch64::STRDroW;
3341 break;
3342 case AArch64::ATOMIC_STORE_HINT_BroX:
3343 StOpc = AArch64::STRBBroX;
3344 break;
3345 case AArch64::ATOMIC_STORE_HINT_HroX:
3346 StOpc = AArch64::STRHHroX;
3347 break;
3348 case AArch64::ATOMIC_STORE_HINT_WroX:
3349 StOpc = AArch64::STRWroX;
3350 break;
3351 case AArch64::ATOMIC_STORE_HINT_XroX:
3352 StOpc = AArch64::STRXroX;
3353 break;
3354 case AArch64::ATOMIC_STORE_HINT_SroX:
3355 StOpc = AArch64::STRSroX;
3356 break;
3357 case AArch64::ATOMIC_STORE_HINT_DroX:
3358 StOpc = AArch64::STRDroX;
3359 break;
3360 default:
3361 llvm_unreachable("Unexpected atomic hint opcode.");
3362 }
3363
3364 EmitToStreamer(
3365 Inst: MCInstBuilder(AArch64::HINT).addImm(Val: MI->getOperand(i: 6).getImm()));
3366
3367 MCInst Store;
3368 Store.setOpcode(StOpc);
3369 Store.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg())); // Rn
3370 Store.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 1).getReg())); // Rm
3371 Store.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 2).getReg())); // Data
3372 Store.addOperand(Op: MCOperand::createImm(Val: MI->getOperand(i: 3).getImm())); // Signed
3373 Store.addOperand(Op: MCOperand::createImm(Val: MI->getOperand(i: 4).getImm())); // Shift
3374 Store.setFlags(MI->getFlags());
3375 EmitToStreamer(S&: *OutStreamer, Inst: Store);
3376}
3377
3378void AArch64AsmPrinter::emitAtomicHintPseudoExpansionImm(
3379 const MachineInstr *MI) {
3380 unsigned StOpc;
3381 assert(MI->getOperand(3).getImm() == 1 &&
3382 "Atomic store addressing mode only supports relaxed stores");
3383
3384 switch (MI->getOpcode()) {
3385 case AArch64::ATOMIC_STORE_HINT_Bui:
3386 StOpc = AArch64::STRBBui;
3387 break;
3388 case AArch64::ATOMIC_STORE_HINT_Hui:
3389 StOpc = AArch64::STRHHui;
3390 break;
3391 case AArch64::ATOMIC_STORE_HINT_Wui:
3392 StOpc = AArch64::STRWui;
3393 break;
3394 case AArch64::ATOMIC_STORE_HINT_Xui:
3395 StOpc = AArch64::STRXui;
3396 break;
3397 case AArch64::ATOMIC_STORE_HINT_Sui:
3398 StOpc = AArch64::STRSui;
3399 break;
3400 case AArch64::ATOMIC_STORE_HINT_Dui:
3401 StOpc = AArch64::STRDui;
3402 break;
3403 case AArch64::ATOMIC_STORE_HINT_Bi:
3404 StOpc = AArch64::STURBBi;
3405 break;
3406 case AArch64::ATOMIC_STORE_HINT_Hi:
3407 StOpc = AArch64::STURHHi;
3408 break;
3409 case AArch64::ATOMIC_STORE_HINT_Wi:
3410 StOpc = AArch64::STURWi;
3411 break;
3412 case AArch64::ATOMIC_STORE_HINT_Xi:
3413 StOpc = AArch64::STURXi;
3414 break;
3415 case AArch64::ATOMIC_STORE_HINT_Si:
3416 StOpc = AArch64::STURSi;
3417 break;
3418 case AArch64::ATOMIC_STORE_HINT_Di:
3419 StOpc = AArch64::STURDi;
3420 break;
3421 default:
3422 llvm_unreachable("Unexpected atomic hint opcode.");
3423 }
3424
3425 EmitToStreamer(
3426 Inst: MCInstBuilder(AArch64::HINT).addImm(Val: MI->getOperand(i: 4).getImm()));
3427
3428 MCInst Store;
3429 Store.setOpcode(StOpc);
3430 Store.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg())); // Rn
3431 Store.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 1).getReg())); // Data
3432 Store.addOperand(Op: MCOperand::createImm(Val: MI->getOperand(i: 2).getImm())); // Imm
3433 Store.setFlags(MI->getFlags());
3434 EmitToStreamer(S&: *OutStreamer, Inst: Store);
3435}
3436
3437void AArch64AsmPrinter::emitAtomicFetchHintPseudoExpansion(
3438 const MachineInstr *MI) {
3439 EmitToStreamer(
3440 Inst: MCInstBuilder(AArch64::HINT).addImm(Val: MI->getOperand(i: 4).getImm()));
3441
3442 MCInst Fetch;
3443 Fetch.setOpcode(MI->getOperand(i: 3).getImm());
3444 Fetch.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg()));
3445 Fetch.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 1).getReg()));
3446 Fetch.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 2).getReg()));
3447 Fetch.setFlags(MI->getFlags());
3448 EmitToStreamer(S&: *OutStreamer, Inst: Fetch);
3449}
3450
3451// Simple pseudo-instructions have their lowering (with expansion to real
3452// instructions) auto-generated.
3453#include "AArch64GenMCPseudoLowering.inc"
3454
3455void AArch64AsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) {
3456 S.emitInstruction(Inst, STI: *STI);
3457#ifndef NDEBUG
3458 ++InstsEmitted;
3459#endif
3460}
3461
3462void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
3463 AArch64_MC::verifyInstructionPredicates(Opcode: MI->getOpcode(), Features: STI->getFeatureBits());
3464
3465#ifndef NDEBUG
3466 InstsEmitted = 0;
3467 llvm::scope_exit CheckMISize([&]() {
3468 assert(STI->getInstrInfo()->getInstSizeInBytes(*MI) >= InstsEmitted * 4);
3469 });
3470#endif
3471
3472 // Do any auto-generated pseudo lowerings.
3473 if (MCInst OutInst; lowerPseudoInstExpansion(MI, Inst&: OutInst)) {
3474 EmitToStreamer(S&: *OutStreamer, Inst: OutInst);
3475 return;
3476 }
3477
3478 if (MI->getOpcode() == AArch64::ADRP) {
3479 for (auto &Opd : MI->operands()) {
3480 if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) ==
3481 "swift_async_extendedFramePointerFlags") {
3482 ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true;
3483 }
3484 }
3485 }
3486
3487 if (AArch64FI->getLOHRelated().count(Ptr: MI)) {
3488 // Generate a label for LOH related instruction
3489 MCSymbol *LOHLabel = createTempSymbol(Name: "loh");
3490 // Associate the instruction with the label
3491 LOHInstToLabel[MI] = LOHLabel;
3492 OutStreamer->emitLabel(Symbol: LOHLabel);
3493 }
3494
3495 AArch64TargetStreamer *TS =
3496 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
3497 // Do any manual lowerings.
3498 switch (MI->getOpcode()) {
3499 default:
3500 assert(!AArch64InstrInfo::isTailCallReturnInst(*MI) &&
3501 "Unhandled tail call instruction");
3502 break;
3503 case AArch64::READ_REGISTER_GPR64:
3504 // Read of a named GPR: emit "mov Xt, Xn" (ORR Xt, XZR, Xn). The source
3505 // register is encoded as an immediate operand so that earlier passes do not
3506 // see a use of an undefined physical register.
3507 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ORRXrs)
3508 .addReg(Reg: MI->getOperand(i: 0).getReg())
3509 .addReg(Reg: AArch64::XZR)
3510 .addReg(Reg: MI->getOperand(i: 1).getImm())
3511 .addImm(Val: 0));
3512 return;
3513 case AArch64::READ_REGISTER_FPR64:
3514 // Read of a named FP/SIMD d-register: emit "fmov Dt, Dn".
3515 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::FMOVDr)
3516 .addReg(Reg: MI->getOperand(i: 0).getReg())
3517 .addReg(Reg: MI->getOperand(i: 1).getImm()));
3518 return;
3519 case AArch64::HINT: {
3520 // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
3521 // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
3522 // non-empty. If MI is the initial BTI, place the
3523 // __patchable_function_entries label after BTI.
3524 if (CurrentPatchableFunctionEntrySym &&
3525 CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
3526 MI == &MF->front().front()) {
3527 int64_t Imm = MI->getOperand(i: 0).getImm();
3528 if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38) {
3529 MCInst Inst;
3530 MCInstLowering.Lower(MI, OutMI&: Inst);
3531 EmitToStreamer(S&: *OutStreamer, Inst);
3532 CurrentPatchableFunctionEntrySym = createTempSymbol(Name: "patch");
3533 OutStreamer->emitLabel(Symbol: CurrentPatchableFunctionEntrySym);
3534 return;
3535 }
3536 }
3537 break;
3538 }
3539 case AArch64::MOVMCSym: {
3540 Register DestReg = MI->getOperand(i: 0).getReg();
3541 const MachineOperand &MO_Sym = MI->getOperand(i: 1);
3542 MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
3543 MCOperand Hi_MCSym, Lo_MCSym;
3544
3545 Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
3546 Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
3547
3548 MCInstLowering.lowerOperand(MO: Hi_MOSym, MCOp&: Hi_MCSym);
3549 MCInstLowering.lowerOperand(MO: Lo_MOSym, MCOp&: Lo_MCSym);
3550
3551 MCInst MovZ;
3552 MovZ.setOpcode(AArch64::MOVZXi);
3553 MovZ.addOperand(Op: MCOperand::createReg(Reg: DestReg));
3554 MovZ.addOperand(Op: Hi_MCSym);
3555 MovZ.addOperand(Op: MCOperand::createImm(Val: 16));
3556 EmitToStreamer(S&: *OutStreamer, Inst: MovZ);
3557
3558 MCInst MovK;
3559 MovK.setOpcode(AArch64::MOVKXi);
3560 MovK.addOperand(Op: MCOperand::createReg(Reg: DestReg));
3561 MovK.addOperand(Op: MCOperand::createReg(Reg: DestReg));
3562 MovK.addOperand(Op: Lo_MCSym);
3563 MovK.addOperand(Op: MCOperand::createImm(Val: 0));
3564 EmitToStreamer(S&: *OutStreamer, Inst: MovK);
3565 return;
3566 }
3567 case AArch64::MOVIv2d_ns:
3568 // It is generally beneficial to rewrite "fmov s0, wzr" to "movi d0, #0".
3569 // as movi is more efficient across all cores. Newer cores can eliminate
3570 // fmovs early and there is no difference with movi, but this not true for
3571 // all implementations.
3572 //
3573 // The floating-point version doesn't quite work in rare cases on older
3574 // CPUs, so on those targets we lower this instruction to movi.16b instead.
3575 if (STI->hasZeroCycleZeroingFPWorkaround() &&
3576 MI->getOperand(i: 1).getImm() == 0) {
3577 MCInst TmpInst;
3578 TmpInst.setOpcode(AArch64::MOVIv16b_ns);
3579 TmpInst.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg()));
3580 TmpInst.addOperand(Op: MCOperand::createImm(Val: 0));
3581 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
3582 return;
3583 }
3584 break;
3585
3586 case AArch64::DBG_VALUE:
3587 case AArch64::DBG_VALUE_LIST:
3588 if (isVerbose() && OutStreamer->hasRawTextSupport()) {
3589 SmallString<128> TmpStr;
3590 raw_svector_ostream OS(TmpStr);
3591 PrintDebugValueComment(MI, OS);
3592 OutStreamer->emitRawText(String: StringRef(OS.str()));
3593 }
3594 return;
3595
3596 case AArch64::EMITBKEY:
3597 case AArch64::EMITNONHINTED: {
3598 ExceptionHandling ExceptionHandlingType = getModuleExceptionModel();
3599 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
3600 ExceptionHandlingType != ExceptionHandling::ARM)
3601 return;
3602
3603 if (getFunctionCFISectionType(MF: *MF) == CFISection::None)
3604 return;
3605
3606 if (MI->getOpcode() == AArch64::EMITBKEY)
3607 OutStreamer->emitCFIBKeyFrame();
3608 else
3609 OutStreamer->emitCFINonHintPAuthFrame();
3610
3611 return;
3612 }
3613
3614 case AArch64::EMITMTETAGGED: {
3615 ExceptionHandling ExceptionHandlingType = getModuleExceptionModel();
3616 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
3617 ExceptionHandlingType != ExceptionHandling::ARM)
3618 return;
3619
3620 if (getFunctionCFISectionType(MF: *MF) != CFISection::None)
3621 OutStreamer->emitCFIMTETaggedFrame();
3622 return;
3623 }
3624
3625 case AArch64::AUTx16x17: {
3626 const Register Pointer = AArch64::X16;
3627 const Register Scratch = AArch64::X17;
3628
3629 auto AuthSchema = PtrAuthSchema::CreateImmReg(
3630 Key: (AArch64PACKey::ID)MI->getOperand(i: 0).getImm(),
3631 IntDisc: MI->getOperand(i: 1).getImm(), AddrDiscOp: MI->getOperand(i: 2));
3632
3633 emitPtrauthAuthResign(Pointer, Scratch, AuthSchema, SignSchema: std::nullopt,
3634 Addend: std::nullopt, DS: MI->getDeactivationSymbol());
3635 return;
3636 }
3637
3638 case AArch64::AUTxMxN: {
3639 const Register Pointer = MI->getOperand(i: 0).getReg();
3640 const Register Scratch = MI->getOperand(i: 1).getReg();
3641
3642 auto AuthSchema = PtrAuthSchema::CreateImmReg(
3643 Key: (AArch64PACKey::ID)MI->getOperand(i: 3).getImm(),
3644 IntDisc: MI->getOperand(i: 4).getImm(), AddrDiscOp: MI->getOperand(i: 5));
3645
3646 emitPtrauthAuthResign(Pointer, Scratch, AuthSchema, SignSchema: std::nullopt,
3647 Addend: std::nullopt, DS: MI->getDeactivationSymbol());
3648 return;
3649 }
3650
3651 case AArch64::AUTPAC: {
3652 const Register Pointer = AArch64::X16;
3653 const Register Scratch = AArch64::X17;
3654
3655 auto AuthSchema = PtrAuthSchema::CreateImmReg(
3656 Key: (AArch64PACKey::ID)MI->getOperand(i: 0).getImm(),
3657 IntDisc: MI->getOperand(i: 1).getImm(), AddrDiscOp: MI->getOperand(i: 2));
3658
3659 auto SignSchema = PtrAuthSchema::CreateImmReg(
3660 Key: (AArch64PACKey::ID)MI->getOperand(i: 3).getImm(),
3661 IntDisc: MI->getOperand(i: 4).getImm(), AddrDiscOp: MI->getOperand(i: 5));
3662
3663 emitPtrauthAuthResign(Pointer, Scratch, AuthSchema, SignSchema,
3664 Addend: std::nullopt, DS: MI->getDeactivationSymbol());
3665 return;
3666 }
3667
3668 case AArch64::AUTPCPAC: {
3669 auto AuthSchema = PtrAuthSchema::CreateRegReg(
3670 Key: (AArch64PACKey::ID)MI->getOperand(i: 0).getImm(), AddrDisc: AArch64::X16,
3671 PCDisc: AArch64::X15);
3672
3673 auto SignSchema = PtrAuthSchema::CreateImmReg(
3674 Key: (AArch64PACKey::ID)MI->getOperand(i: 1).getImm(),
3675 IntDisc: MI->getOperand(i: 2).getImm(), AddrDiscOp: MI->getOperand(i: 3));
3676
3677 emitPtrauthAuthResign(/*Pointer=*/AArch64::X17, /*Scratch=*/AArch64::X16,
3678 AuthSchema, SignSchema, Addend: std::nullopt,
3679 DS: MI->getDeactivationSymbol());
3680 return;
3681 }
3682
3683 case AArch64::AUTRELLOADPAC: {
3684 const Register Pointer = AArch64::X16;
3685 const Register Scratch = AArch64::X17;
3686
3687 auto AuthSchema = PtrAuthSchema::CreateImmReg(
3688 Key: (AArch64PACKey::ID)MI->getOperand(i: 0).getImm(),
3689 IntDisc: MI->getOperand(i: 1).getImm(), AddrDiscOp: MI->getOperand(i: 2));
3690
3691 auto SignSchema = PtrAuthSchema::CreateImmReg(
3692 Key: (AArch64PACKey::ID)MI->getOperand(i: 3).getImm(),
3693 IntDisc: MI->getOperand(i: 4).getImm(), AddrDiscOp: MI->getOperand(i: 5));
3694
3695 emitPtrauthAuthResign(Pointer, Scratch, AuthSchema, SignSchema,
3696 Addend: MI->getOperand(i: 6).getImm(),
3697 DS: MI->getDeactivationSymbol());
3698
3699 return;
3700 }
3701
3702 case AArch64::PAC:
3703 emitPtrauthSign(MI);
3704 return;
3705
3706 case AArch64::LOADauthptrstatic:
3707 LowerLOADauthptrstatic(MI: *MI);
3708 return;
3709
3710 case AArch64::LOADgotPAC:
3711 case AArch64::MOVaddrPAC:
3712 LowerMOVaddrPAC(MI: *MI);
3713 return;
3714
3715 case AArch64::LOADgotAUTH:
3716 LowerLOADgotAUTH(MI: *MI);
3717 return;
3718
3719 case AArch64::BRA:
3720 case AArch64::BLRA:
3721 emitPtrauthBranch(MI);
3722 return;
3723
3724 // Tail calls use pseudo instructions so they have the proper code-gen
3725 // attributes (isCall, isReturn, etc.). We lower them to the real
3726 // instruction here.
3727 case AArch64::AUTH_TCRETURN:
3728 case AArch64::AUTH_TCRETURN_BTI: {
3729 Register Callee = MI->getOperand(i: 0).getReg();
3730 const auto Key = (AArch64PACKey::ID)MI->getOperand(i: 2).getImm();
3731 const uint64_t Disc = MI->getOperand(i: 3).getImm();
3732
3733 Register AddrDisc = MI->getOperand(i: 4).getReg();
3734
3735 Register ScratchReg = Callee == AArch64::X16 ? AArch64::X17 : AArch64::X16;
3736
3737 emitPtrauthTailCallHardening(TC: MI);
3738
3739 // See the comments in emitPtrauthBranch.
3740 if (Callee == AddrDisc)
3741 report_fatal_error(reason: "Call target is signed with its own value");
3742
3743 // After isX16X17Safer predicate was introduced, emitPtrauthDiscriminator is
3744 // no longer restricted to only reusing AddrDisc when it is X16 or X17
3745 // (which are implicit-def'ed by AUTH_TCRETURN pseudos), thus impose this
3746 // restriction manually not to clobber an unexpected register.
3747 bool AddrDiscIsImplicitDef =
3748 AddrDisc == AArch64::X16 || AddrDisc == AArch64::X17;
3749 Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, ScratchReg,
3750 MayClobberAddrDisc: AddrDiscIsImplicitDef);
3751 emitBLRA(/*IsCall*/ false, Key, Target: Callee, Disc: DiscReg);
3752 return;
3753 }
3754
3755 case AArch64::TCRETURNri:
3756 case AArch64::TCRETURNrix16x17:
3757 case AArch64::TCRETURNrix17:
3758 case AArch64::TCRETURNrinotx16:
3759 case AArch64::TCRETURNriALL: {
3760 emitPtrauthTailCallHardening(TC: MI);
3761
3762 recordIfImportCall(BranchInst: MI);
3763 MCInst TmpInst;
3764 TmpInst.setOpcode(AArch64::BR);
3765 TmpInst.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg()));
3766 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
3767 return;
3768 }
3769 case AArch64::TCRETURNdi: {
3770 emitPtrauthTailCallHardening(TC: MI);
3771
3772 MCOperand Dest;
3773 MCInstLowering.lowerOperand(MO: MI->getOperand(i: 0), MCOp&: Dest);
3774 recordIfImportCall(BranchInst: MI);
3775 MCInst TmpInst;
3776 TmpInst.setOpcode(AArch64::B);
3777 TmpInst.addOperand(Op: Dest);
3778 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
3779 return;
3780 }
3781 case AArch64::SpeculationBarrierISBDSBEndBB: {
3782 // Print DSB SYS + ISB
3783 MCInst TmpInstDSB;
3784 TmpInstDSB.setOpcode(AArch64::DSB);
3785 TmpInstDSB.addOperand(Op: MCOperand::createImm(Val: 0xf));
3786 EmitToStreamer(S&: *OutStreamer, Inst: TmpInstDSB);
3787 MCInst TmpInstISB;
3788 TmpInstISB.setOpcode(AArch64::ISB);
3789 TmpInstISB.addOperand(Op: MCOperand::createImm(Val: 0xf));
3790 EmitToStreamer(S&: *OutStreamer, Inst: TmpInstISB);
3791 return;
3792 }
3793 case AArch64::SpeculationBarrierSBEndBB: {
3794 // Print SB
3795 MCInst TmpInstSB;
3796 TmpInstSB.setOpcode(AArch64::SB);
3797 EmitToStreamer(S&: *OutStreamer, Inst: TmpInstSB);
3798 return;
3799 }
3800 case AArch64::TLSDESC_AUTH_CALLSEQ: {
3801 /// lower this to:
3802 /// adrp x0, :tlsdesc_auth:var
3803 /// ldr x16, [x0, #:tlsdesc_auth_lo12:var]
3804 /// add x0, x0, #:tlsdesc_auth_lo12:var
3805 /// .tlsauthdesccall var
3806 /// blraa x16, x0
3807 /// (TPIDR_EL0 offset now in x0)
3808 const MachineOperand &MO_Sym = MI->getOperand(i: 0);
3809 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
3810 MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
3811 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
3812 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
3813 MCInstLowering.lowerOperand(MO: MO_Sym, MCOp&: Sym);
3814 MCInstLowering.lowerOperand(MO: MO_TLSDESC_LO12, MCOp&: SymTLSDescLo12);
3815 MCInstLowering.lowerOperand(MO: MO_TLSDESC, MCOp&: SymTLSDesc);
3816
3817 MCInst Adrp;
3818 Adrp.setOpcode(AArch64::ADRP);
3819 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3820 Adrp.addOperand(Op: SymTLSDesc);
3821 EmitToStreamer(S&: *OutStreamer, Inst: Adrp);
3822
3823 MCInst Ldr;
3824 Ldr.setOpcode(AArch64::LDRXui);
3825 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
3826 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3827 Ldr.addOperand(Op: SymTLSDescLo12);
3828 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
3829 EmitToStreamer(S&: *OutStreamer, Inst: Ldr);
3830
3831 MCInst Add;
3832 Add.setOpcode(AArch64::ADDXri);
3833 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3834 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3835 Add.addOperand(Op: SymTLSDescLo12);
3836 Add.addOperand(Op: MCOperand::createImm(Val: AArch64_AM::getShiftValue(Imm: 0)));
3837 EmitToStreamer(S&: *OutStreamer, Inst: Add);
3838
3839 // Emit a relocation-annotation. This expands to no code, but requests
3840 // the following instruction gets an R_AARCH64_AUTH_TLSDESC_CALL.
3841 MCInst TLSAuthDescCall;
3842 TLSAuthDescCall.setOpcode(AArch64::TLSAUTHDESCCALL);
3843 TLSAuthDescCall.addOperand(Op: Sym);
3844 EmitToStreamer(S&: *OutStreamer, Inst: TLSAuthDescCall);
3845#ifndef NDEBUG
3846 --InstsEmitted; // no code emitted
3847#endif
3848
3849 MCInst Blraa;
3850 Blraa.setOpcode(AArch64::BLRAA);
3851 Blraa.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
3852 Blraa.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3853 EmitToStreamer(S&: *OutStreamer, Inst: Blraa);
3854
3855 return;
3856 }
3857 case AArch64::TLSDESC_CALLSEQ: {
3858 /// lower this to:
3859 /// adrp x0, :tlsdesc:var
3860 /// ldr x1, [x0, #:tlsdesc_lo12:var]
3861 /// add x0, x0, #:tlsdesc_lo12:var
3862 /// .tlsdesccall var
3863 /// blr x1
3864 /// (TPIDR_EL0 offset now in x0)
3865 const MachineOperand &MO_Sym = MI->getOperand(i: 0);
3866 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
3867 MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
3868 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
3869 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
3870 MCInstLowering.lowerOperand(MO: MO_Sym, MCOp&: Sym);
3871 MCInstLowering.lowerOperand(MO: MO_TLSDESC_LO12, MCOp&: SymTLSDescLo12);
3872 MCInstLowering.lowerOperand(MO: MO_TLSDESC, MCOp&: SymTLSDesc);
3873
3874 MCInst Adrp;
3875 Adrp.setOpcode(AArch64::ADRP);
3876 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3877 Adrp.addOperand(Op: SymTLSDesc);
3878 EmitToStreamer(S&: *OutStreamer, Inst: Adrp);
3879
3880 MCInst Ldr;
3881 if (STI->isTargetILP32()) {
3882 Ldr.setOpcode(AArch64::LDRWui);
3883 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::W1));
3884 } else {
3885 Ldr.setOpcode(AArch64::LDRXui);
3886 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X1));
3887 }
3888 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3889 Ldr.addOperand(Op: SymTLSDescLo12);
3890 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
3891 EmitToStreamer(S&: *OutStreamer, Inst: Ldr);
3892
3893 MCInst Add;
3894 if (STI->isTargetILP32()) {
3895 Add.setOpcode(AArch64::ADDWri);
3896 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::W0));
3897 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::W0));
3898 } else {
3899 Add.setOpcode(AArch64::ADDXri);
3900 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3901 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3902 }
3903 Add.addOperand(Op: SymTLSDescLo12);
3904 Add.addOperand(Op: MCOperand::createImm(Val: AArch64_AM::getShiftValue(Imm: 0)));
3905 EmitToStreamer(S&: *OutStreamer, Inst: Add);
3906
3907 // Emit a relocation-annotation. This expands to no code, but requests
3908 // the following instruction gets an R_AARCH64_TLSDESC_CALL.
3909 MCInst TLSDescCall;
3910 TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
3911 TLSDescCall.addOperand(Op: Sym);
3912 EmitToStreamer(S&: *OutStreamer, Inst: TLSDescCall);
3913#ifndef NDEBUG
3914 --InstsEmitted; // no code emitted
3915#endif
3916
3917 MCInst Blr;
3918 Blr.setOpcode(AArch64::BLR);
3919 Blr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X1));
3920 EmitToStreamer(S&: *OutStreamer, Inst: Blr);
3921
3922 return;
3923 }
3924
3925 case AArch64::JumpTableDest32:
3926 case AArch64::JumpTableDest16:
3927 case AArch64::JumpTableDest8:
3928 LowerJumpTableDest(OutStreamer&: *OutStreamer, MI: *MI);
3929 return;
3930
3931 case AArch64::BR_JumpTable:
3932 LowerHardenedBRJumpTable(MI: *MI);
3933 return;
3934
3935 case AArch64::FMOVH0:
3936 case AArch64::FMOVS0:
3937 case AArch64::FMOVD0:
3938 emitFMov0(MI: *MI);
3939 return;
3940
3941 case AArch64::MOPSMemoryCopyPseudo:
3942 case AArch64::MOPSMemoryMovePseudo:
3943 case AArch64::MOPSMemorySetPseudo:
3944 case AArch64::MOPSMemorySetTaggingPseudo:
3945 LowerMOPS(OutStreamer&: *OutStreamer, MI: *MI);
3946 return;
3947
3948 case TargetOpcode::STACKMAP:
3949 return LowerSTACKMAP(OutStreamer&: *OutStreamer, SM, MI: *MI);
3950
3951 case TargetOpcode::PATCHPOINT:
3952 return LowerPATCHPOINT(OutStreamer&: *OutStreamer, SM, MI: *MI);
3953
3954 case TargetOpcode::STATEPOINT:
3955 return LowerSTATEPOINT(OutStreamer&: *OutStreamer, SM, MI: *MI);
3956
3957 case TargetOpcode::FAULTING_OP:
3958 return LowerFAULTING_OP(FaultingMI: *MI);
3959
3960 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
3961 LowerPATCHABLE_FUNCTION_ENTER(MI: *MI);
3962 return;
3963
3964 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
3965 LowerPATCHABLE_FUNCTION_EXIT(MI: *MI);
3966 return;
3967
3968 case TargetOpcode::PATCHABLE_TAIL_CALL:
3969 LowerPATCHABLE_TAIL_CALL(MI: *MI);
3970 return;
3971 case TargetOpcode::PATCHABLE_EVENT_CALL:
3972 return LowerPATCHABLE_EVENT_CALL(MI: *MI, Typed: false);
3973 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
3974 return LowerPATCHABLE_EVENT_CALL(MI: *MI, Typed: true);
3975
3976 case AArch64::KCFI_CHECK:
3977 LowerKCFI_CHECK(MI: *MI);
3978 return;
3979
3980 case AArch64::HWASAN_CHECK_MEMACCESS:
3981 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
3982 case AArch64::HWASAN_CHECK_MEMACCESS_FIXEDSHADOW:
3983 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW:
3984 LowerHWASAN_CHECK_MEMACCESS(MI: *MI);
3985 return;
3986
3987 case AArch64::SEH_EndC:
3988 TS->emitARM64WinCFIEndC();
3989 return;
3990
3991 case AArch64::SEH_StackAlloc:
3992 TS->emitARM64WinCFIAllocStack(Size: MI->getOperand(i: 0).getImm());
3993 return;
3994
3995 case AArch64::SEH_SaveFPLR:
3996 TS->emitARM64WinCFISaveFPLR(Offset: MI->getOperand(i: 0).getImm());
3997 return;
3998
3999 case AArch64::SEH_SaveFPLR_X:
4000 assert(MI->getOperand(0).getImm() < 0 &&
4001 "Pre increment SEH opcode must have a negative offset");
4002 TS->emitARM64WinCFISaveFPLRX(Offset: -MI->getOperand(i: 0).getImm());
4003 return;
4004
4005 case AArch64::SEH_SaveReg:
4006 TS->emitARM64WinCFISaveReg(Reg: MI->getOperand(i: 0).getImm(),
4007 Offset: MI->getOperand(i: 1).getImm());
4008 return;
4009
4010 case AArch64::SEH_SaveReg_X:
4011 assert(MI->getOperand(1).getImm() < 0 &&
4012 "Pre increment SEH opcode must have a negative offset");
4013 TS->emitARM64WinCFISaveRegX(Reg: MI->getOperand(i: 0).getImm(),
4014 Offset: -MI->getOperand(i: 1).getImm());
4015 return;
4016
4017 case AArch64::SEH_SaveRegP:
4018 if (MI->getOperand(i: 1).getImm() == 30 && MI->getOperand(i: 0).getImm() >= 19 &&
4019 MI->getOperand(i: 0).getImm() <= 28) {
4020 assert((MI->getOperand(0).getImm() - 19) % 2 == 0 &&
4021 "Register paired with LR must be odd");
4022 TS->emitARM64WinCFISaveLRPair(Reg: MI->getOperand(i: 0).getImm(),
4023 Offset: MI->getOperand(i: 2).getImm());
4024 return;
4025 }
4026 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
4027 "Non-consecutive registers not allowed for save_regp");
4028 TS->emitARM64WinCFISaveRegP(Reg: MI->getOperand(i: 0).getImm(),
4029 Offset: MI->getOperand(i: 2).getImm());
4030 return;
4031
4032 case AArch64::SEH_SaveRegP_X:
4033 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
4034 "Non-consecutive registers not allowed for save_regp_x");
4035 assert(MI->getOperand(2).getImm() < 0 &&
4036 "Pre increment SEH opcode must have a negative offset");
4037 TS->emitARM64WinCFISaveRegPX(Reg: MI->getOperand(i: 0).getImm(),
4038 Offset: -MI->getOperand(i: 2).getImm());
4039 return;
4040
4041 case AArch64::SEH_SaveFReg:
4042 TS->emitARM64WinCFISaveFReg(Reg: MI->getOperand(i: 0).getImm(),
4043 Offset: MI->getOperand(i: 1).getImm());
4044 return;
4045
4046 case AArch64::SEH_SaveFReg_X:
4047 assert(MI->getOperand(1).getImm() < 0 &&
4048 "Pre increment SEH opcode must have a negative offset");
4049 TS->emitARM64WinCFISaveFRegX(Reg: MI->getOperand(i: 0).getImm(),
4050 Offset: -MI->getOperand(i: 1).getImm());
4051 return;
4052
4053 case AArch64::SEH_SaveFRegP:
4054 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
4055 "Non-consecutive registers not allowed for save_regp");
4056 TS->emitARM64WinCFISaveFRegP(Reg: MI->getOperand(i: 0).getImm(),
4057 Offset: MI->getOperand(i: 2).getImm());
4058 return;
4059
4060 case AArch64::SEH_SaveFRegP_X:
4061 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
4062 "Non-consecutive registers not allowed for save_regp_x");
4063 assert(MI->getOperand(2).getImm() < 0 &&
4064 "Pre increment SEH opcode must have a negative offset");
4065 TS->emitARM64WinCFISaveFRegPX(Reg: MI->getOperand(i: 0).getImm(),
4066 Offset: -MI->getOperand(i: 2).getImm());
4067 return;
4068
4069 case AArch64::SEH_SetFP:
4070 TS->emitARM64WinCFISetFP();
4071 return;
4072
4073 case AArch64::SEH_AddFP:
4074 TS->emitARM64WinCFIAddFP(Size: MI->getOperand(i: 0).getImm());
4075 return;
4076
4077 case AArch64::SEH_Nop:
4078 TS->emitARM64WinCFINop();
4079 return;
4080
4081 case AArch64::SEH_PrologEnd:
4082 TS->emitARM64WinCFIPrologEnd();
4083 return;
4084
4085 case AArch64::SEH_EpilogStart:
4086 TS->emitARM64WinCFIEpilogStart();
4087 return;
4088
4089 case AArch64::SEH_EpilogEnd:
4090 TS->emitARM64WinCFIEpilogEnd();
4091 return;
4092
4093 case AArch64::SEH_PACSignLR:
4094 TS->emitARM64WinCFIPACSignLR();
4095 return;
4096
4097 case AArch64::SEH_SaveAnyRegI:
4098 assert((isShiftedUInt<6, 3>((uint64_t)MI->getOperand(1).getImm())) &&
4099 "SaveAnyRegI SEH opcode offset must fit into 6 bits");
4100 TS->emitARM64WinCFISaveAnyRegI(Reg: MI->getOperand(i: 0).getImm(),
4101 Offset: MI->getOperand(i: 1).getImm());
4102 return;
4103
4104 case AArch64::SEH_SaveAnyRegIP:
4105 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
4106 "Non-consecutive registers not allowed for save_any_reg");
4107 assert((isShiftedUInt<6, 4>((uint64_t)MI->getOperand(2).getImm())) &&
4108 "SaveAnyRegIP SEH opcode offset must fit into 6 bits");
4109 TS->emitARM64WinCFISaveAnyRegIP(Reg: MI->getOperand(i: 0).getImm(),
4110 Offset: MI->getOperand(i: 2).getImm());
4111 return;
4112
4113 case AArch64::SEH_SaveAnyRegQ:
4114 assert((isShiftedUInt<6, 4>((uint64_t)MI->getOperand(1).getImm())) &&
4115 "SaveAnyRegQ SEH opcode offset must fit into 6 bits");
4116 TS->emitARM64WinCFISaveAnyRegQ(Reg: MI->getOperand(i: 0).getImm(),
4117 Offset: MI->getOperand(i: 1).getImm());
4118 return;
4119
4120 case AArch64::SEH_SaveAnyRegQX:
4121 assert((isShiftedUInt<6, 4>(-(uint64_t)MI->getOperand(1).getImm())) &&
4122 "SaveAnyRegQX SEH opcode offset must fit into 6 bits");
4123 TS->emitARM64WinCFISaveAnyRegQX(Reg: MI->getOperand(i: 0).getImm(),
4124 Offset: -MI->getOperand(i: 1).getImm());
4125 return;
4126
4127 case AArch64::SEH_SaveAnyRegQP:
4128 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
4129 "Non-consecutive registers not allowed for save_any_reg");
4130 assert((isShiftedUInt<6, 4>((uint64_t)MI->getOperand(2).getImm())) &&
4131 "SaveAnyRegQP SEH opcode offset must fit into 6 bits");
4132 TS->emitARM64WinCFISaveAnyRegQP(Reg: MI->getOperand(i: 0).getImm(),
4133 Offset: MI->getOperand(i: 2).getImm());
4134 return;
4135
4136 case AArch64::SEH_SaveAnyRegQPX:
4137 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
4138 "Non-consecutive registers not allowed for save_any_reg");
4139 assert((isShiftedUInt<6, 4>(-(uint64_t)MI->getOperand(2).getImm())) &&
4140 "SaveAnyRegQPX SEH opcode offset must fit into 6 bits");
4141 TS->emitARM64WinCFISaveAnyRegQPX(Reg: MI->getOperand(i: 0).getImm(),
4142 Offset: -MI->getOperand(i: 2).getImm());
4143 return;
4144
4145 case AArch64::SEH_AllocZ:
4146 assert(MI->getOperand(0).getImm() >= 0 &&
4147 "AllocZ SEH opcode offset must be non-negative");
4148 assert(MI->getOperand(0).getImm() <= 255 &&
4149 "AllocZ SEH opcode offset must fit into 8 bits");
4150 TS->emitARM64WinCFIAllocZ(Offset: MI->getOperand(i: 0).getImm());
4151 return;
4152
4153 case AArch64::SEH_SaveZReg:
4154 assert(MI->getOperand(1).getImm() >= 0 &&
4155 "SaveZReg SEH opcode offset must be non-negative");
4156 assert(MI->getOperand(1).getImm() <= 255 &&
4157 "SaveZReg SEH opcode offset must fit into 8 bits");
4158 TS->emitARM64WinCFISaveZReg(Reg: MI->getOperand(i: 0).getImm(),
4159 Offset: MI->getOperand(i: 1).getImm());
4160 return;
4161
4162 case AArch64::SEH_SavePReg:
4163 assert(MI->getOperand(1).getImm() >= 0 &&
4164 "SavePReg SEH opcode offset must be non-negative");
4165 assert(MI->getOperand(1).getImm() <= 255 &&
4166 "SavePReg SEH opcode offset must fit into 8 bits");
4167 TS->emitARM64WinCFISavePReg(Reg: MI->getOperand(i: 0).getImm(),
4168 Offset: MI->getOperand(i: 1).getImm());
4169 return;
4170
4171 case AArch64::BLR:
4172 case AArch64::BR: {
4173 recordIfImportCall(BranchInst: MI);
4174 MCInst TmpInst;
4175 MCInstLowering.Lower(MI, OutMI&: TmpInst);
4176 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
4177 return;
4178 }
4179 case AArch64::CBWPri:
4180 case AArch64::CBXPri:
4181 case AArch64::CBBAssertExt:
4182 case AArch64::CBHAssertExt:
4183 case AArch64::CBWPrr:
4184 case AArch64::CBXPrr:
4185 emitCBPseudoExpansion(MI);
4186 return;
4187 case AArch64::ATOMIC_STORE_HINT_B:
4188 case AArch64::ATOMIC_STORE_HINT_H:
4189 case AArch64::ATOMIC_STORE_HINT_W:
4190 case AArch64::ATOMIC_STORE_HINT_X:
4191 emitAtomicHintPseudoExpansion(MI);
4192 return;
4193 case AArch64::ATOMIC_FETCH_HINT_W:
4194 case AArch64::ATOMIC_FETCH_HINT_X:
4195 emitAtomicFetchHintPseudoExpansion(MI);
4196 return;
4197 case AArch64::ATOMIC_STORE_HINT_BroW:
4198 case AArch64::ATOMIC_STORE_HINT_HroW:
4199 case AArch64::ATOMIC_STORE_HINT_WroW:
4200 case AArch64::ATOMIC_STORE_HINT_XroW:
4201 case AArch64::ATOMIC_STORE_HINT_SroW:
4202 case AArch64::ATOMIC_STORE_HINT_DroW:
4203 case AArch64::ATOMIC_STORE_HINT_BroX:
4204 case AArch64::ATOMIC_STORE_HINT_HroX:
4205 case AArch64::ATOMIC_STORE_HINT_WroX:
4206 case AArch64::ATOMIC_STORE_HINT_XroX:
4207 case AArch64::ATOMIC_STORE_HINT_SroX:
4208 case AArch64::ATOMIC_STORE_HINT_DroX:
4209 emitAtomicHintPseudoExpansionRO(MI);
4210 return;
4211 case AArch64::ATOMIC_STORE_HINT_Bui:
4212 case AArch64::ATOMIC_STORE_HINT_Hui:
4213 case AArch64::ATOMIC_STORE_HINT_Wui:
4214 case AArch64::ATOMIC_STORE_HINT_Xui:
4215 case AArch64::ATOMIC_STORE_HINT_Sui:
4216 case AArch64::ATOMIC_STORE_HINT_Dui:
4217 case AArch64::ATOMIC_STORE_HINT_Bi:
4218 case AArch64::ATOMIC_STORE_HINT_Hi:
4219 case AArch64::ATOMIC_STORE_HINT_Wi:
4220 case AArch64::ATOMIC_STORE_HINT_Xi:
4221 case AArch64::ATOMIC_STORE_HINT_Si:
4222 case AArch64::ATOMIC_STORE_HINT_Di:
4223 emitAtomicHintPseudoExpansionImm(MI);
4224 return;
4225 }
4226
4227 if (emitDeactivationSymbolRelocation(DS: MI->getDeactivationSymbol()))
4228 return;
4229
4230 // Finally, do the automated lowerings for everything else.
4231 MCInst TmpInst;
4232 MCInstLowering.Lower(MI, OutMI&: TmpInst);
4233 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
4234}
4235
4236void AArch64AsmPrinter::recordIfImportCall(
4237 const llvm::MachineInstr *BranchInst) {
4238 if (!EnableImportCallOptimization)
4239 return;
4240
4241 auto [GV, OpFlags] = BranchInst->getMF()->tryGetCalledGlobal(MI: BranchInst);
4242 if (GV && GV->hasDLLImportStorageClass()) {
4243 auto *CallSiteSymbol = MMI->getContext().createNamedTempSymbol(Name: "impcall");
4244 OutStreamer->emitLabel(Symbol: CallSiteSymbol);
4245
4246 auto *CalledSymbol = MCInstLowering.GetGlobalValueSymbol(GV, TargetFlags: OpFlags);
4247 SectionToImportedFunctionCalls[OutStreamer->getCurrentSectionOnly()]
4248 .push_back(x: {CallSiteSymbol, CalledSymbol});
4249 }
4250}
4251
4252void AArch64AsmPrinter::emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI,
4253 MCSymbol *LazyPointer) {
4254 // _ifunc:
4255 // adrp x16, lazy_pointer@GOTPAGE
4256 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF]
4257 // ldr x16, [x16]
4258 // br x16
4259
4260 {
4261 MCInst Adrp;
4262 Adrp.setOpcode(AArch64::ADRP);
4263 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
4264 MCOperand SymPage;
4265 MCInstLowering.lowerOperand(
4266 MO: MachineOperand::CreateMCSymbol(Sym: LazyPointer,
4267 TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGE),
4268 MCOp&: SymPage);
4269 Adrp.addOperand(Op: SymPage);
4270 EmitToStreamer(Inst: Adrp);
4271 }
4272
4273 {
4274 MCInst Ldr;
4275 Ldr.setOpcode(AArch64::LDRXui);
4276 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
4277 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
4278 MCOperand SymPageOff;
4279 MCInstLowering.lowerOperand(
4280 MO: MachineOperand::CreateMCSymbol(Sym: LazyPointer, TargetFlags: AArch64II::MO_GOT |
4281 AArch64II::MO_PAGEOFF),
4282 MCOp&: SymPageOff);
4283 Ldr.addOperand(Op: SymPageOff);
4284 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
4285 EmitToStreamer(Inst: Ldr);
4286 }
4287
4288 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
4289 .addReg(Reg: AArch64::X16)
4290 .addReg(Reg: AArch64::X16)
4291 .addImm(Val: 0));
4292
4293 EmitToStreamer(Inst: MCInstBuilder(TM.getTargetTriple().isArm64e() ? AArch64::BRAAZ
4294 : AArch64::BR)
4295 .addReg(Reg: AArch64::X16));
4296}
4297
4298void AArch64AsmPrinter::emitMachOIFuncStubHelperBody(Module &M,
4299 const GlobalIFunc &GI,
4300 MCSymbol *LazyPointer) {
4301 // These stub helpers are only ever called once, so here we're optimizing for
4302 // minimum size by using the pre-indexed store variants, which saves a few
4303 // bytes of instructions to bump & restore sp.
4304
4305 // _ifunc.stub_helper:
4306 // stp fp, lr, [sp, #-16]!
4307 // mov fp, sp
4308 // stp x1, x0, [sp, #-16]!
4309 // stp x3, x2, [sp, #-16]!
4310 // stp x5, x4, [sp, #-16]!
4311 // stp x7, x6, [sp, #-16]!
4312 // stp d1, d0, [sp, #-16]!
4313 // stp d3, d2, [sp, #-16]!
4314 // stp d5, d4, [sp, #-16]!
4315 // stp d7, d6, [sp, #-16]!
4316 // bl _resolver
4317 // adrp x16, lazy_pointer@GOTPAGE
4318 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF]
4319 // str x0, [x16]
4320 // mov x16, x0
4321 // ldp d7, d6, [sp], #16
4322 // ldp d5, d4, [sp], #16
4323 // ldp d3, d2, [sp], #16
4324 // ldp d1, d0, [sp], #16
4325 // ldp x7, x6, [sp], #16
4326 // ldp x5, x4, [sp], #16
4327 // ldp x3, x2, [sp], #16
4328 // ldp x1, x0, [sp], #16
4329 // ldp fp, lr, [sp], #16
4330 // br x16
4331
4332 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPXpre)
4333 .addReg(Reg: AArch64::SP)
4334 .addReg(Reg: AArch64::FP)
4335 .addReg(Reg: AArch64::LR)
4336 .addReg(Reg: AArch64::SP)
4337 .addImm(Val: -2));
4338
4339 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
4340 .addReg(Reg: AArch64::FP)
4341 .addReg(Reg: AArch64::SP)
4342 .addImm(Val: 0)
4343 .addImm(Val: 0));
4344
4345 for (int I = 0; I != 4; ++I)
4346 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPXpre)
4347 .addReg(Reg: AArch64::SP)
4348 .addReg(Reg: AArch64::X1 + 2 * I)
4349 .addReg(Reg: AArch64::X0 + 2 * I)
4350 .addReg(Reg: AArch64::SP)
4351 .addImm(Val: -2));
4352
4353 for (int I = 0; I != 4; ++I)
4354 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPDpre)
4355 .addReg(Reg: AArch64::SP)
4356 .addReg(Reg: AArch64::D1 + 2 * I)
4357 .addReg(Reg: AArch64::D0 + 2 * I)
4358 .addReg(Reg: AArch64::SP)
4359 .addImm(Val: -2));
4360
4361 EmitToStreamer(
4362 Inst: MCInstBuilder(AArch64::BL)
4363 .addOperand(Op: MCOperand::createExpr(Val: lowerConstant(CV: GI.getResolver()))));
4364
4365 {
4366 MCInst Adrp;
4367 Adrp.setOpcode(AArch64::ADRP);
4368 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
4369 MCOperand SymPage;
4370 MCInstLowering.lowerOperand(
4371 MO: MachineOperand::CreateES(SymName: LazyPointer->getName().data() + 1,
4372 TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGE),
4373 MCOp&: SymPage);
4374 Adrp.addOperand(Op: SymPage);
4375 EmitToStreamer(Inst: Adrp);
4376 }
4377
4378 {
4379 MCInst Ldr;
4380 Ldr.setOpcode(AArch64::LDRXui);
4381 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
4382 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
4383 MCOperand SymPageOff;
4384 MCInstLowering.lowerOperand(
4385 MO: MachineOperand::CreateES(SymName: LazyPointer->getName().data() + 1,
4386 TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGEOFF),
4387 MCOp&: SymPageOff);
4388 Ldr.addOperand(Op: SymPageOff);
4389 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
4390 EmitToStreamer(Inst: Ldr);
4391 }
4392
4393 EmitToStreamer(Inst: MCInstBuilder(AArch64::STRXui)
4394 .addReg(Reg: AArch64::X0)
4395 .addReg(Reg: AArch64::X16)
4396 .addImm(Val: 0));
4397
4398 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
4399 .addReg(Reg: AArch64::X16)
4400 .addReg(Reg: AArch64::X0)
4401 .addImm(Val: 0)
4402 .addImm(Val: 0));
4403
4404 for (int I = 3; I != -1; --I)
4405 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDPDpost)
4406 .addReg(Reg: AArch64::SP)
4407 .addReg(Reg: AArch64::D1 + 2 * I)
4408 .addReg(Reg: AArch64::D0 + 2 * I)
4409 .addReg(Reg: AArch64::SP)
4410 .addImm(Val: 2));
4411
4412 for (int I = 3; I != -1; --I)
4413 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDPXpost)
4414 .addReg(Reg: AArch64::SP)
4415 .addReg(Reg: AArch64::X1 + 2 * I)
4416 .addReg(Reg: AArch64::X0 + 2 * I)
4417 .addReg(Reg: AArch64::SP)
4418 .addImm(Val: 2));
4419
4420 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDPXpost)
4421 .addReg(Reg: AArch64::SP)
4422 .addReg(Reg: AArch64::FP)
4423 .addReg(Reg: AArch64::LR)
4424 .addReg(Reg: AArch64::SP)
4425 .addImm(Val: 2));
4426
4427 EmitToStreamer(Inst: MCInstBuilder(TM.getTargetTriple().isArm64e() ? AArch64::BRAAZ
4428 : AArch64::BR)
4429 .addReg(Reg: AArch64::X16));
4430}
4431
4432const MCExpr *AArch64AsmPrinter::lowerConstant(const Constant *CV,
4433 const Constant *BaseCV,
4434 uint64_t Offset) {
4435 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: CV)) {
4436 return MCSymbolRefExpr::create(Symbol: MCInstLowering.GetGlobalValueSymbol(GV, TargetFlags: 0),
4437 Ctx&: OutContext);
4438 }
4439
4440 return AsmPrinter::lowerConstant(CV, BaseCV, Offset);
4441}
4442
4443char AArch64AsmPrinter::ID = 0;
4444
4445INITIALIZE_PASS(AArch64AsmPrinter, "aarch64-asm-printer",
4446 "AArch64 Assembly Printer", false, false)
4447
4448// Force static initialization.
4449extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
4450LLVMInitializeAArch64AsmPrinter() {
4451 RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
4452 RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
4453 RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
4454 RegisterAsmPrinter<AArch64AsmPrinter> W(getTheARM64_32Target());
4455 RegisterAsmPrinter<AArch64AsmPrinter> V(getTheAArch64_32Target());
4456}
4457
4458PreservedAnalyses AArch64AsmPrinterBeginPass::run(Module &M,
4459 ModuleAnalysisManager &MAM) {
4460 AArch64AsmPrinter &AsmPrinter = static_cast<AArch64AsmPrinter &>(
4461 MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter());
4462 setupModuleAsmPrinter(M, MAM, AsmPrinter);
4463 AsmPrinter.doInitialization(M);
4464 return PreservedAnalyses::all();
4465}
4466
4467PreservedAnalyses
4468AArch64AsmPrinterPass::run(MachineFunction &MF,
4469 MachineFunctionAnalysisManager &MFAM) {
4470 AArch64AsmPrinter &AsmPrinter = static_cast<AArch64AsmPrinter &>(
4471 MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(IR&: MF)
4472 .getCachedResult<AsmPrinterAnalysis>(IR&: *MF.getFunction().getParent())
4473 ->getPrinter());
4474 setupMachineFunctionAsmPrinter(MFAM, MF, AsmPrinter);
4475 AsmPrinter.runOnMachineFunction(MF);
4476 return PreservedAnalyses::all();
4477}
4478
4479PreservedAnalyses AArch64AsmPrinterEndPass::run(Module &M,
4480 ModuleAnalysisManager &MAM) {
4481 AArch64AsmPrinter &AsmPrinter = static_cast<AArch64AsmPrinter &>(
4482 MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter());
4483 setupModuleAsmPrinter(M, MAM, AsmPrinter);
4484 AsmPrinter.doFinalization(M);
4485 return PreservedAnalyses::all();
4486}
4487