1//===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a printer that converts from our internal representation
10// of machine-dependent LLVM code to the AArch64 assembly language.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AArch64AsmPrinter.h"
15#include "AArch64.h"
16#include "AArch64MCInstLower.h"
17#include "AArch64MachineFunctionInfo.h"
18#include "AArch64RegisterInfo.h"
19#include "AArch64Subtarget.h"
20#include "AArch64TargetObjectFile.h"
21#include "MCTargetDesc/AArch64AddressingModes.h"
22#include "MCTargetDesc/AArch64InstPrinter.h"
23#include "MCTargetDesc/AArch64MCAsmInfo.h"
24#include "MCTargetDesc/AArch64MCTargetDesc.h"
25#include "MCTargetDesc/AArch64TargetStreamer.h"
26#include "TargetInfo/AArch64TargetInfo.h"
27#include "Utils/AArch64BaseInfo.h"
28#include "llvm/ADT/ScopeExit.h"
29#include "llvm/ADT/SmallString.h"
30#include "llvm/ADT/SmallVector.h"
31#include "llvm/ADT/Statistic.h"
32#include "llvm/ADT/StringRef.h"
33#include "llvm/ADT/Twine.h"
34#include "llvm/BinaryFormat/COFF.h"
35#include "llvm/BinaryFormat/ELF.h"
36#include "llvm/BinaryFormat/MachO.h"
37#include "llvm/CodeGen/AsmPrinter.h"
38#include "llvm/CodeGen/AsmPrinterAnalysis.h"
39#include "llvm/CodeGen/FaultMaps.h"
40#include "llvm/CodeGen/MachineBasicBlock.h"
41#include "llvm/CodeGen/MachineFunction.h"
42#include "llvm/CodeGen/MachineInstr.h"
43#include "llvm/CodeGen/MachineJumpTableInfo.h"
44#include "llvm/CodeGen/MachineModuleInfoImpls.h"
45#include "llvm/CodeGen/MachineOperand.h"
46#include "llvm/CodeGen/StackMaps.h"
47#include "llvm/CodeGen/TargetRegisterInfo.h"
48#include "llvm/IR/Analysis.h"
49#include "llvm/IR/DataLayout.h"
50#include "llvm/IR/DebugInfoMetadata.h"
51#include "llvm/IR/Mangler.h"
52#include "llvm/IR/Module.h"
53#include "llvm/IR/PassManager.h"
54#include "llvm/MC/MCAsmInfo.h"
55#include "llvm/MC/MCContext.h"
56#include "llvm/MC/MCExpr.h"
57#include "llvm/MC/MCInst.h"
58#include "llvm/MC/MCInstBuilder.h"
59#include "llvm/MC/MCSectionELF.h"
60#include "llvm/MC/MCSectionMachO.h"
61#include "llvm/MC/MCStreamer.h"
62#include "llvm/MC/MCSymbol.h"
63#include "llvm/MC/MCValue.h"
64#include "llvm/MC/TargetRegistry.h"
65#include "llvm/Support/Casting.h"
66#include "llvm/Support/Compiler.h"
67#include "llvm/Support/ErrorHandling.h"
68#include "llvm/Support/raw_ostream.h"
69#include "llvm/Target/TargetMachine.h"
70#include "llvm/TargetParser/Triple.h"
71#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
72#include <cassert>
73#include <cstdint>
74#include <map>
75#include <memory>
76
77using namespace llvm;
78using AArch64PAuth::PtrauthCheckMode;
79
80#define DEBUG_TYPE "AArch64AsmPrinter"
81
82// Doesn't count FPR128 ZCZ instructions which are handled
83// by TableGen pattern matching
84STATISTIC(NumZCZeroingInstrsFPR,
85 "Number of zero-cycle FPR zeroing instructions expanded from "
86 "canonical pseudo instructions");
87
88namespace {
89
90class AArch64AsmPrinter : public AsmPrinter {
91 AArch64MCInstLower MCInstLowering;
92 FaultMaps FM;
93 const AArch64Subtarget *STI;
94 bool ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = false;
95 bool PtrauthInitFini = false;
96 bool PtrauthInitFiniAddressDisc = false;
97#ifndef NDEBUG
98 unsigned InstsEmitted;
99#endif
100 bool EnableImportCallOptimization = false;
101 MapVector<MCSection *, std::vector<std::pair<MCSymbol *, MCSymbol *>>>
102 SectionToImportedFunctionCalls;
103 unsigned PAuthIFuncNextUniqueID = 1;
104
105public:
106 static char ID;
107
108 AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
109 : AsmPrinter(TM, std::move(Streamer), ID),
110 MCInstLowering(OutContext, *this), FM(*this) {}
111
112 StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
113
114 /// Wrapper for MCInstLowering.lowerOperand() for the
115 /// tblgen'erated pseudo lowering.
116 bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
117 return MCInstLowering.lowerOperand(MO, MCOp);
118 }
119
120 const MCExpr *lowerConstantPtrAuth(const ConstantPtrAuth &CPA) override;
121
122 const MCExpr *lowerBlockAddressConstant(const BlockAddress &BA) override;
123
124 void emitStartOfAsmFile(Module &M) override;
125 void emitJumpTableImpl(const MachineJumpTableInfo &MJTI,
126 ArrayRef<unsigned> JumpTableIndices) override;
127 std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
128 codeview::JumpTableEntrySize>
129 getCodeViewJumpTableInfo(int JTI, const MachineInstr *BranchInstr,
130 const MCSymbol *BranchLabel) const override;
131
132 void emitFunctionEntryLabel() override;
133
134 void emitXXStructor(const DataLayout &DL, const Constant *CV) override;
135
136 void LowerJumpTableDest(MCStreamer &OutStreamer, const MachineInstr &MI);
137
138 void LowerHardenedBRJumpTable(const MachineInstr &MI);
139
140 void LowerMOPS(MCStreamer &OutStreamer, const MachineInstr &MI);
141
142 void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
143 const MachineInstr &MI);
144 void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
145 const MachineInstr &MI);
146 void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
147 const MachineInstr &MI);
148 void LowerFAULTING_OP(const MachineInstr &MI);
149
150 void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
151 void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
152 void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
153 void LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI, bool Typed);
154
155 typedef std::tuple<unsigned, bool, uint32_t, bool, uint64_t>
156 HwasanMemaccessTuple;
157 std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
158 void LowerKCFI_CHECK(const MachineInstr &MI);
159 void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
160 void emitHwasanMemaccessSymbols(Module &M);
161
162 void emitSled(const MachineInstr &MI, SledKind Kind);
163
164 // Returns whether Reg may be used to store sensitive temporary values when
165 // expanding PtrAuth pseudos. Some OSes may take extra care to protect a
166 // small subset of GPRs on context switches - use these registers then.
167 //
168 // If there are no preferred registers, returns true for any Reg.
169 bool isPtrauthRegSafe(Register Reg) const {
170 if (STI->isX16X17Safer())
171 return Reg == AArch64::X16 || Reg == AArch64::X17;
172
173 return true;
174 }
175
176 // Emit the sequence for BRA/BLRA (authenticate + branch/call).
177 void emitPtrauthBranch(const MachineInstr *MI);
178
179 void emitPtrauthCheckAuthenticatedValue(Register TestedReg,
180 Register ScratchReg,
181 AArch64PACKey::ID Key,
182 AArch64PAuth::AuthCheckMethod Method,
183 const MCSymbol *OnFailure = nullptr);
184
185 // Check authenticated LR before tail calling.
186 void emitPtrauthTailCallHardening(const MachineInstr *TC);
187
188 struct PtrAuthSchema {
189 static PtrAuthSchema CreateImmReg(AArch64PACKey::ID Key, uint64_t IntDisc,
190 const MachineOperand &AddrDiscOp);
191 static PtrAuthSchema CreateRegReg(AArch64PACKey::ID Key, Register AddrDisc,
192 Register PCDisc);
193
194 AArch64PACKey::ID Key;
195 uint64_t IntDisc;
196 Register AddrDisc;
197 bool AddrDiscIsKilled;
198 Register PCDisc;
199
200 bool addrDiscIsKilledAndNoneOf(std::initializer_list<Register> Regs) {
201 return AddrDiscIsKilled && !llvm::is_contained(Set: Regs, Element: AddrDisc);
202 }
203 };
204
205 // Helper for emitting AUTRELLOADPAC: increment Pointer by Addend and then by
206 // a 32-bit signed value loaded from memory. The instructions emitted are
207 //
208 // ldrsw Scratch, [Pointer, #Addend]!
209 // add Pointer, Pointer, Scratch
210 //
211 // for small Addend value, with longer sequences required for wider Addend.
212 void emitPtrauthApplyIndirectAddend(Register Pointer, Register Scratch,
213 int64_t Addend);
214
215 // Emit the sequence for AUT or AUTPAC (or their PC-blending variants).
216 // Addend is only used for AUTRELLOADPAC.
217 void emitPtrauthAuthResign(Register Pointer, Register Scratch,
218 PtrAuthSchema AuthSchema,
219 std::optional<PtrAuthSchema> SignSchema,
220 std::optional<int64_t> Addend, Value *DS);
221
222 // Emit R_AARCH64_PATCHINST, the deactivation symbol relocation. Returns true
223 // if no instruction should be emitted because the deactivation symbol is
224 // defined in the current module so this function emitted a NOP instead.
225 bool emitDeactivationSymbolRelocation(Value *DS);
226
227 // Emit the sequence for PAC.
228 void emitPtrauthSign(const MachineInstr *MI);
229
230 // Emit the sequence to compute the discriminator.
231 //
232 // The Scratch register passed to this function must be safe, as returned by
233 // isPtrauthRegSafe(ScratchReg).
234 //
235 // The returned register is either ScratchReg, AddrDisc, or XZR. Furthermore,
236 // it is guaranteed to be safe (or XZR), with the only exception of
237 // passing-through an *unmodified* unsafe AddrDisc register.
238 //
239 // If the expanded pseudo is allowed to clobber AddrDisc register, setting
240 // MayClobberAddrDisc may save one MOV instruction, provided
241 // isPtrauthRegSafe(AddrDisc) is true:
242 //
243 // mov x17, x16
244 // movk x17, #1234, lsl #48
245 // ; x16 is not used anymore
246 //
247 // can be replaced by
248 //
249 // movk x16, #1234, lsl #48
250 Register emitPtrauthDiscriminator(uint64_t Disc, Register AddrDisc,
251 Register ScratchReg,
252 bool MayClobberAddrDisc = false);
253
254 // Emit the sequence for LOADauthptrstatic
255 void LowerLOADauthptrstatic(const MachineInstr &MI);
256
257 // Emit the sequence for LOADgotPAC/MOVaddrPAC (either GOT adrp-ldr or
258 // adrp-add followed by PAC sign)
259 void LowerMOVaddrPAC(const MachineInstr &MI);
260
261 // Emit the sequence for LOADgotAUTH (load signed pointer from signed ELF GOT
262 // and authenticate it with, if FPAC bit is not set, check+trap sequence after
263 // authenticating)
264 void LowerLOADgotAUTH(const MachineInstr &MI);
265
266 void emitAddImm(MCRegister Val, int64_t Addend, MCRegister Tmp);
267 void emitAddress(MCRegister Reg, const MCExpr *Expr, MCRegister Tmp,
268 bool DSOLocal, const MCSubtargetInfo &STI);
269
270 const MCExpr *emitPAuthRelocationAsIRelative(
271 const MCExpr *Target, uint64_t Disc, AArch64PACKey::ID KeyID,
272 bool HasAddressDiversity, bool IsDSOLocal, const MCExpr *DSExpr);
273
274 /// tblgen'erated driver function for lowering simple MI->MC
275 /// pseudo instructions.
276 bool lowerPseudoInstExpansion(const MachineInstr *MI, MCInst &Inst);
277
278 // Emit Build Attributes
279 void emitAttributes(unsigned Flags, uint64_t PAuthABIPlatform,
280 uint64_t PAuthABIVersion, AArch64TargetStreamer *TS);
281
282 // Emit expansion of Compare-and-branch pseudo instructions
283 void emitCBPseudoExpansion(const MachineInstr *MI);
284
285 // Emit expansion of atomic store with hint pseudo instructions
286 void emitAtomicHintPseudoExpansion(const MachineInstr *MI);
287 void emitAtomicHintPseudoExpansionRO(const MachineInstr *MI);
288 void emitAtomicHintPseudoExpansionImm(const MachineInstr *MI);
289
290 void EmitToStreamer(MCStreamer &S, const MCInst &Inst);
291 void EmitToStreamer(const MCInst &Inst) {
292 EmitToStreamer(S&: *OutStreamer, Inst);
293 }
294
295 void emitInstruction(const MachineInstr *MI) override;
296
297 void emitFunctionHeaderComment() override;
298
299 void getAnalysisUsage(AnalysisUsage &AU) const override {
300 AsmPrinter::getAnalysisUsage(AU);
301 AU.setPreservesAll();
302 }
303
304 bool runOnMachineFunction(MachineFunction &MF) override {
305 if (auto *PSIW = getAnalysisIfAvailable<ProfileSummaryInfoWrapperPass>())
306 PSI = &PSIW->getPSI();
307 if (auto *SDPIW =
308 getAnalysisIfAvailable<StaticDataProfileInfoWrapperPass>())
309 SDPI = &SDPIW->getStaticDataProfileInfo();
310
311 AArch64FI = MF.getInfo<AArch64FunctionInfo>();
312 STI = &MF.getSubtarget<AArch64Subtarget>();
313
314 SetupMachineFunction(MF);
315
316 if (STI->isTargetCOFF()) {
317 bool Local = MF.getFunction().hasLocalLinkage();
318 COFF::SymbolStorageClass Scl =
319 Local ? COFF::IMAGE_SYM_CLASS_STATIC : COFF::IMAGE_SYM_CLASS_EXTERNAL;
320 int Type =
321 COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
322
323 OutStreamer->beginCOFFSymbolDef(Symbol: CurrentFnSym);
324 OutStreamer->emitCOFFSymbolStorageClass(StorageClass: Scl);
325 OutStreamer->emitCOFFSymbolType(Type);
326 OutStreamer->endCOFFSymbolDef();
327 }
328
329 // Emit the rest of the function body.
330 emitFunctionBody();
331
332 // Emit the XRay table for this function.
333 emitXRayTable();
334
335 // We didn't modify anything.
336 return false;
337 }
338
339 const MCExpr *lowerConstant(const Constant *CV,
340 const Constant *BaseCV = nullptr,
341 uint64_t Offset = 0) override;
342
343private:
344 void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
345 bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
346 bool printAsmRegInClass(const MachineOperand &MO,
347 const TargetRegisterClass *RC, unsigned AltName,
348 raw_ostream &O);
349
350 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
351 const char *ExtraCode, raw_ostream &O) override;
352 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
353 const char *ExtraCode, raw_ostream &O) override;
354
355 void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
356
357 void emitFunctionBodyEnd() override;
358 void emitGlobalAlias(const Module &M, const GlobalAlias &GA) override;
359
360 MCSymbol *GetCPISymbol(unsigned CPID) const override;
361 void emitEndOfAsmFile(Module &M) override;
362
363 AArch64FunctionInfo *AArch64FI = nullptr;
364
365 /// Emit the LOHs contained in AArch64FI.
366 void emitLOHs();
367
368 void emitMovXReg(Register Dest, Register Src);
369 void emitMOVZ(Register Dest, uint64_t Imm, unsigned Shift);
370 void emitMOVK(Register Dest, uint64_t Imm, unsigned Shift);
371
372 void emitAUT(AArch64PACKey::ID Key, Register Pointer, Register Disc);
373 void emitPAC(AArch64PACKey::ID Key, Register Pointer, Register Disc);
374 void emitBLRA(bool IsCall, AArch64PACKey::ID Key, Register Target,
375 Register Disc);
376
377 /// Emit instruction to set float register to zero.
378 void emitFMov0(const MachineInstr &MI);
379 void emitFMov0AsFMov(const MachineInstr &MI, Register DestReg);
380
381 using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
382
383 MInstToMCSymbol LOHInstToLabel;
384
385 bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const override {
386 return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags;
387 }
388
389 const MCSubtargetInfo *getIFuncMCSubtargetInfo() const override {
390 assert(STI);
391 return STI;
392 }
393 void emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI,
394 MCSymbol *LazyPointer) override;
395 void emitMachOIFuncStubHelperBody(Module &M, const GlobalIFunc &GI,
396 MCSymbol *LazyPointer) override;
397
398 /// Checks if this instruction is part of a sequence that is eligle for import
399 /// call optimization and, if so, records it to be emitted in the import call
400 /// section.
401 void recordIfImportCall(const MachineInstr *BranchInst);
402};
403
404} // end anonymous namespace
405
406// Get boolean module flag (0 or 1), treating absent flag as having value 0.
407static bool getOptionalBooleanModuleFlag(Module &M, StringRef Name) {
408 Metadata *Flag = M.getModuleFlag(Key: Name);
409 if (!Flag)
410 return false;
411
412 uint64_t Value = mdconst::extract<ConstantInt>(MD&: Flag)->getZExtValue();
413 assert((Value == 0 || Value == 1) && "Boolean flag is expected, if present");
414 return Value;
415}
416
417void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
418 const Triple &TT = M.getTargetTriple();
419
420 if (TT.isOSBinFormatCOFF()) {
421 emitCOFFFeatureSymbol(M);
422 emitCOFFReplaceableFunctionData(M);
423
424 if (M.getModuleFlag(Key: "import-call-optimization"))
425 EnableImportCallOptimization = true;
426 }
427
428 PtrauthInitFini = getOptionalBooleanModuleFlag(M, Name: "ptrauth-init-fini");
429 PtrauthInitFiniAddressDisc = getOptionalBooleanModuleFlag(
430 M, Name: "ptrauth-init-fini-address-discrimination");
431
432 if (!TT.isOSBinFormatELF())
433 return;
434
435 // For emitting build attributes and .note.gnu.property section
436 auto *TS =
437 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
438 // Assemble feature flags that may require creation of build attributes and a
439 // note section.
440 unsigned BAFlags = 0;
441 unsigned GNUFlags = 0;
442 if (const auto *BTE = mdconst::extract_or_null<ConstantInt>(
443 MD: M.getModuleFlag(Key: "branch-target-enforcement"))) {
444 if (!BTE->isZero()) {
445 BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_BTI_Flag;
446 GNUFlags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
447 }
448 }
449
450 if (const auto *GCS = mdconst::extract_or_null<ConstantInt>(
451 MD: M.getModuleFlag(Key: "guarded-control-stack"))) {
452 if (!GCS->isZero()) {
453 BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_GCS_Flag;
454 GNUFlags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_GCS;
455 }
456 }
457
458 if (const auto *Sign = mdconst::extract_or_null<ConstantInt>(
459 MD: M.getModuleFlag(Key: "sign-return-address"))) {
460 if (!Sign->isZero()) {
461 BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_PAC_Flag;
462 GNUFlags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
463 }
464 }
465
466 uint64_t PAuthABIPlatform = -1;
467 if (const auto *PAP = mdconst::extract_or_null<ConstantInt>(
468 MD: M.getModuleFlag(Key: "aarch64-elf-pauthabi-platform"))) {
469 PAuthABIPlatform = PAP->getZExtValue();
470 }
471
472 uint64_t PAuthABIVersion = -1;
473 if (const auto *PAV = mdconst::extract_or_null<ConstantInt>(
474 MD: M.getModuleFlag(Key: "aarch64-elf-pauthabi-version"))) {
475 PAuthABIVersion = PAV->getZExtValue();
476 }
477
478 // For LLVM_LINUX experimental platform, version value of 0 means no PAuth
479 // support. Do not emit corresponding PAuthABI GNU property note and AArch64
480 // build attributes for this case to keep Linux binaries not using PAuth
481 // unaffected.
482 if (PAuthABIPlatform == ELF::AARCH64_PAUTH_PLATFORM_LLVM_LINUX &&
483 PAuthABIVersion == 0) {
484 PAuthABIPlatform = uint64_t(-1);
485 PAuthABIVersion = uint64_t(-1);
486 }
487
488 // Emit AArch64 Build Attributes
489 emitAttributes(Flags: BAFlags, PAuthABIPlatform, PAuthABIVersion, TS);
490 // Emit a .note.gnu.property section with the flags.
491 TS->emitNoteSection(Flags: GNUFlags, PAuthABIPlatform, PAuthABIVersion);
492}
493
494void AArch64AsmPrinter::emitFunctionHeaderComment() {
495 const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>();
496 std::optional<std::string> OutlinerString = FI->getOutliningStyle();
497 if (OutlinerString != std::nullopt)
498 OutStreamer->getCommentOS() << ' ' << OutlinerString;
499}
500
501void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
502{
503 const Function &F = MF->getFunction();
504 if (F.hasFnAttribute(Kind: "patchable-function-entry")) {
505 unsigned Num;
506 if (F.getFnAttribute(Kind: "patchable-function-entry")
507 .getValueAsString()
508 .getAsInteger(Radix: 10, Result&: Num))
509 return;
510 emitNops(N: Num);
511 return;
512 }
513
514 emitSled(MI, Kind: SledKind::FUNCTION_ENTER);
515}
516
517void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI) {
518 emitSled(MI, Kind: SledKind::FUNCTION_EXIT);
519}
520
521void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) {
522 emitSled(MI, Kind: SledKind::TAIL_CALL);
523}
524
525void AArch64AsmPrinter::emitSled(const MachineInstr &MI, SledKind Kind) {
526 static const int8_t NoopsInSledCount = 7;
527 // We want to emit the following pattern:
528 //
529 // .Lxray_sled_N:
530 // ALIGN
531 // B #32
532 // ; 7 NOP instructions (28 bytes)
533 // .tmpN
534 //
535 // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
536 // over the full 32 bytes (8 instructions) with the following pattern:
537 //
538 // STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
539 // LDR W17, #12 ; W17 := function ID
540 // LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
541 // BLR X16 ; call the tracing trampoline
542 // ;DATA: 32 bits of function ID
543 // ;DATA: lower 32 bits of the address of the trampoline
544 // ;DATA: higher 32 bits of the address of the trampoline
545 // LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
546 //
547 OutStreamer->emitCodeAlignment(Alignment: Align(4), STI: getSubtargetInfo());
548 auto CurSled = OutContext.createTempSymbol(Name: "xray_sled_", AlwaysAddSuffix: true);
549 OutStreamer->emitLabel(Symbol: CurSled);
550 auto Target = OutContext.createTempSymbol();
551
552 // Emit "B #32" instruction, which jumps over the next 28 bytes.
553 // The operand has to be the number of 4-byte instructions to jump over,
554 // including the current instruction.
555 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::B).addImm(Val: 8));
556
557 for (int8_t I = 0; I < NoopsInSledCount; I++)
558 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::NOP));
559
560 OutStreamer->emitLabel(Symbol: Target);
561 recordSled(Sled: CurSled, MI, Kind, Version: 2);
562}
563
564void AArch64AsmPrinter::emitAttributes(unsigned Flags,
565 uint64_t PAuthABIPlatform,
566 uint64_t PAuthABIVersion,
567 AArch64TargetStreamer *TS) {
568
569 PAuthABIPlatform = (uint64_t(-1) == PAuthABIPlatform) ? 0 : PAuthABIPlatform;
570 PAuthABIVersion = (uint64_t(-1) == PAuthABIVersion) ? 0 : PAuthABIVersion;
571
572 if (PAuthABIPlatform || PAuthABIVersion) {
573 TS->emitAttributesSubsection(
574 VendorName: AArch64BuildAttributes::getVendorName(
575 Vendor: AArch64BuildAttributes::AEABI_PAUTHABI),
576 IsOptional: AArch64BuildAttributes::SubsectionOptional::REQUIRED,
577 ParameterType: AArch64BuildAttributes::SubsectionType::ULEB128);
578 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
579 Vendor: AArch64BuildAttributes::AEABI_PAUTHABI),
580 Tag: AArch64BuildAttributes::TAG_PAUTH_PLATFORM,
581 Value: PAuthABIPlatform, String: "");
582 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
583 Vendor: AArch64BuildAttributes::AEABI_PAUTHABI),
584 Tag: AArch64BuildAttributes::TAG_PAUTH_SCHEMA, Value: PAuthABIVersion,
585 String: "");
586 }
587
588 unsigned BTIValue =
589 (Flags & AArch64BuildAttributes::Feature_BTI_Flag) ? 1 : 0;
590 unsigned PACValue =
591 (Flags & AArch64BuildAttributes::Feature_PAC_Flag) ? 1 : 0;
592 unsigned GCSValue =
593 (Flags & AArch64BuildAttributes::Feature_GCS_Flag) ? 1 : 0;
594
595 if (BTIValue || PACValue || GCSValue) {
596 TS->emitAttributesSubsection(
597 VendorName: AArch64BuildAttributes::getVendorName(
598 Vendor: AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
599 IsOptional: AArch64BuildAttributes::SubsectionOptional::OPTIONAL,
600 ParameterType: AArch64BuildAttributes::SubsectionType::ULEB128);
601 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
602 Vendor: AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
603 Tag: AArch64BuildAttributes::TAG_FEATURE_BTI, Value: BTIValue, String: "");
604 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
605 Vendor: AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
606 Tag: AArch64BuildAttributes::TAG_FEATURE_PAC, Value: PACValue, String: "");
607 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
608 Vendor: AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
609 Tag: AArch64BuildAttributes::TAG_FEATURE_GCS, Value: GCSValue, String: "");
610 }
611}
612
613// Emit the following code for Intrinsic::{xray_customevent,xray_typedevent}
614// (built-in functions __xray_customevent/__xray_typedevent).
615//
616// .Lxray_event_sled_N:
617// b 1f
618// save x0 and x1 (and also x2 for TYPED_EVENT_CALL)
619// set up x0 and x1 (and also x2 for TYPED_EVENT_CALL)
620// bl __xray_CustomEvent or __xray_TypedEvent
621// restore x0 and x1 (and also x2 for TYPED_EVENT_CALL)
622// 1:
623//
624// There are 6 instructions for EVENT_CALL and 9 for TYPED_EVENT_CALL.
625//
626// Then record a sled of kind CUSTOM_EVENT or TYPED_EVENT.
627// After patching, b .+N will become a nop.
628void AArch64AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI,
629 bool Typed) {
630 auto &O = *OutStreamer;
631 MCSymbol *CurSled = OutContext.createTempSymbol(Name: "xray_sled_", AlwaysAddSuffix: true);
632 O.emitLabel(Symbol: CurSled);
633 bool MachO = MMI->getModule()->getTargetTriple().isOSBinFormatMachO();
634 auto *Sym = MCSymbolRefExpr::create(
635 Symbol: OutContext.getOrCreateSymbol(
636 Name: Twine(MachO ? "_" : "") +
637 (Typed ? "__xray_TypedEvent" : "__xray_CustomEvent")),
638 Ctx&: OutContext);
639 if (Typed) {
640 O.AddComment(T: "Begin XRay typed event");
641 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::B).addImm(Val: 9));
642 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::STPXpre)
643 .addReg(Reg: AArch64::SP)
644 .addReg(Reg: AArch64::X0)
645 .addReg(Reg: AArch64::X1)
646 .addReg(Reg: AArch64::SP)
647 .addImm(Val: -4));
648 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::STRXui)
649 .addReg(Reg: AArch64::X2)
650 .addReg(Reg: AArch64::SP)
651 .addImm(Val: 2));
652 emitMovXReg(Dest: AArch64::X0, Src: MI.getOperand(i: 0).getReg());
653 emitMovXReg(Dest: AArch64::X1, Src: MI.getOperand(i: 1).getReg());
654 emitMovXReg(Dest: AArch64::X2, Src: MI.getOperand(i: 2).getReg());
655 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::BL).addExpr(Val: Sym));
656 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::LDRXui)
657 .addReg(Reg: AArch64::X2)
658 .addReg(Reg: AArch64::SP)
659 .addImm(Val: 2));
660 O.AddComment(T: "End XRay typed event");
661 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::LDPXpost)
662 .addReg(Reg: AArch64::SP)
663 .addReg(Reg: AArch64::X0)
664 .addReg(Reg: AArch64::X1)
665 .addReg(Reg: AArch64::SP)
666 .addImm(Val: 4));
667
668 recordSled(Sled: CurSled, MI, Kind: SledKind::TYPED_EVENT, Version: 2);
669 } else {
670 O.AddComment(T: "Begin XRay custom event");
671 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::B).addImm(Val: 6));
672 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::STPXpre)
673 .addReg(Reg: AArch64::SP)
674 .addReg(Reg: AArch64::X0)
675 .addReg(Reg: AArch64::X1)
676 .addReg(Reg: AArch64::SP)
677 .addImm(Val: -2));
678 emitMovXReg(Dest: AArch64::X0, Src: MI.getOperand(i: 0).getReg());
679 emitMovXReg(Dest: AArch64::X1, Src: MI.getOperand(i: 1).getReg());
680 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::BL).addExpr(Val: Sym));
681 O.AddComment(T: "End XRay custom event");
682 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::LDPXpost)
683 .addReg(Reg: AArch64::SP)
684 .addReg(Reg: AArch64::X0)
685 .addReg(Reg: AArch64::X1)
686 .addReg(Reg: AArch64::SP)
687 .addImm(Val: 2));
688
689 recordSled(Sled: CurSled, MI, Kind: SledKind::CUSTOM_EVENT, Version: 2);
690 }
691}
692
693void AArch64AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
694 Register AddrReg = MI.getOperand(i: 0).getReg();
695 assert(std::next(MI.getIterator())->isCall() &&
696 "KCFI_CHECK not followed by a call instruction");
697 assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg &&
698 "KCFI_CHECK call target doesn't match call operand");
699
700 // Default to using the intra-procedure-call temporary registers for
701 // comparing the hashes.
702 unsigned ScratchRegs[] = {AArch64::W16, AArch64::W17};
703 if (AddrReg == AArch64::XZR) {
704 // Checking XZR makes no sense. Instead of emitting a load, zero
705 // ScratchRegs[0] and use it for the ESR AddrIndex below.
706 AddrReg = getXRegFromWReg(Reg: ScratchRegs[0]);
707 emitMovXReg(Dest: AddrReg, Src: AArch64::XZR);
708 } else {
709 // If one of the scratch registers is used for the call target (e.g.
710 // with AArch64::TCRETURNriBTI), we can clobber another caller-saved
711 // temporary register instead (in this case, AArch64::W9) as the check
712 // is immediately followed by the call instruction.
713 for (auto &Reg : ScratchRegs) {
714 if (Reg == getWRegFromXReg(Reg: AddrReg)) {
715 Reg = AArch64::W9;
716 break;
717 }
718 }
719 assert(ScratchRegs[0] != AddrReg && ScratchRegs[1] != AddrReg &&
720 "Invalid scratch registers for KCFI_CHECK");
721
722 // Adjust the offset for patchable-function-prefix. This assumes that
723 // patchable-function-prefix is the same for all functions.
724 int64_t PrefixNops =
725 MI.getMF()->getFunction().getFnAttributeAsParsedInteger(
726 Kind: "patchable-function-prefix");
727
728 // Load the target function type hash.
729 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::LDURWi)
730 .addReg(Reg: ScratchRegs[0])
731 .addReg(Reg: AddrReg)
732 .addImm(Val: -(PrefixNops * 4 + 4)));
733 }
734
735 // Load the expected type hash.
736 const int64_t Type = MI.getOperand(i: 1).getImm();
737 emitMOVK(Dest: ScratchRegs[1], Imm: Type & 0xFFFF, Shift: 0);
738 emitMOVK(Dest: ScratchRegs[1], Imm: (Type >> 16) & 0xFFFF, Shift: 16);
739
740 // Compare the hashes and trap if there's a mismatch.
741 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::SUBSWrs)
742 .addReg(Reg: AArch64::WZR)
743 .addReg(Reg: ScratchRegs[0])
744 .addReg(Reg: ScratchRegs[1])
745 .addImm(Val: 0));
746
747 MCSymbol *Pass = OutContext.createTempSymbol();
748 EmitToStreamer(S&: *OutStreamer,
749 Inst: MCInstBuilder(AArch64::Bcc)
750 .addImm(Val: AArch64CC::EQ)
751 .addExpr(Val: MCSymbolRefExpr::create(Symbol: Pass, Ctx&: OutContext)));
752
753 // The base ESR is 0x8000 and the register information is encoded in bits
754 // 0-9 as follows:
755 // - 0-4: n, where the register Xn contains the target address
756 // - 5-9: m, where the register Wm contains the expected type hash
757 // Where n, m are in [0, 30].
758 unsigned TypeIndex = ScratchRegs[1] - AArch64::W0;
759 unsigned AddrIndex;
760 switch (AddrReg) {
761 default:
762 AddrIndex = AddrReg - AArch64::X0;
763 break;
764 case AArch64::FP:
765 AddrIndex = 29;
766 break;
767 case AArch64::LR:
768 AddrIndex = 30;
769 break;
770 }
771
772 assert(AddrIndex < 31 && TypeIndex < 31);
773
774 unsigned ESR = 0x8000 | ((TypeIndex & 31) << 5) | (AddrIndex & 31);
775 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::BRK).addImm(Val: ESR));
776 OutStreamer->emitLabel(Symbol: Pass);
777}
778
779void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
780 Register Reg = MI.getOperand(i: 0).getReg();
781
782 // The HWASan pass won't emit a CHECK_MEMACCESS intrinsic with a pointer
783 // statically known to be zero. However, conceivably, the HWASan pass may
784 // encounter a "cannot currently statically prove to be null" pointer (and is
785 // therefore unable to omit the intrinsic) that later optimization passes
786 // convert into a statically known-null pointer.
787 if (Reg == AArch64::XZR)
788 return;
789
790 bool IsShort =
791 ((MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES) ||
792 (MI.getOpcode() ==
793 AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW));
794 uint32_t AccessInfo = MI.getOperand(i: 1).getImm();
795 bool IsFixedShadow =
796 ((MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_FIXEDSHADOW) ||
797 (MI.getOpcode() ==
798 AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW));
799 uint64_t FixedShadowOffset = IsFixedShadow ? MI.getOperand(i: 2).getImm() : 0;
800
801 MCSymbol *&Sym = HwasanMemaccessSymbols[HwasanMemaccessTuple(
802 Reg, IsShort, AccessInfo, IsFixedShadow, FixedShadowOffset)];
803 if (!Sym) {
804 // FIXME: Make this work on non-ELF.
805 if (!TM.getTargetTriple().isOSBinFormatELF())
806 report_fatal_error(reason: "llvm.hwasan.check.memaccess only supported on ELF");
807
808 std::string SymName = "__hwasan_check_x" + utostr(X: Reg - AArch64::X0) + "_" +
809 utostr(X: AccessInfo);
810 if (IsFixedShadow)
811 SymName += "_fixed_" + utostr(X: FixedShadowOffset);
812 if (IsShort)
813 SymName += "_short_v2";
814 Sym = OutContext.getOrCreateSymbol(Name: SymName);
815 }
816
817 EmitToStreamer(S&: *OutStreamer,
818 Inst: MCInstBuilder(AArch64::BL)
819 .addExpr(Val: MCSymbolRefExpr::create(Symbol: Sym, Ctx&: OutContext)));
820}
821
822void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) {
823 if (HwasanMemaccessSymbols.empty())
824 return;
825
826 const Triple &TT = M.getTargetTriple();
827 assert(TT.isOSBinFormatELF());
828 // AArch64Subtarget is huge, so heap allocate it so we don't run out of stack
829 // space.
830 auto STI = std::make_unique<AArch64Subtarget>(
831 args: TT, args: TM.getTargetCPU(), args: TM.getTargetCPU(), args: TM.getTargetFeatureString(), args&: TM,
832 args: true);
833 this->STI = STI.get();
834
835 MCSymbol *HwasanTagMismatchV1Sym =
836 OutContext.getOrCreateSymbol(Name: "__hwasan_tag_mismatch");
837 MCSymbol *HwasanTagMismatchV2Sym =
838 OutContext.getOrCreateSymbol(Name: "__hwasan_tag_mismatch_v2");
839
840 const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
841 MCSymbolRefExpr::create(Symbol: HwasanTagMismatchV1Sym, Ctx&: OutContext);
842 const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
843 MCSymbolRefExpr::create(Symbol: HwasanTagMismatchV2Sym, Ctx&: OutContext);
844
845 for (auto &P : HwasanMemaccessSymbols) {
846 unsigned Reg = std::get<0>(t: P.first);
847 bool IsShort = std::get<1>(t: P.first);
848 uint32_t AccessInfo = std::get<2>(t: P.first);
849 bool IsFixedShadow = std::get<3>(t: P.first);
850 uint64_t FixedShadowOffset = std::get<4>(t: P.first);
851 const MCSymbolRefExpr *HwasanTagMismatchRef =
852 IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
853 MCSymbol *Sym = P.second;
854
855 bool HasMatchAllTag =
856 (AccessInfo >> HWASanAccessInfo::HasMatchAllShift) & 1;
857 uint8_t MatchAllTag =
858 (AccessInfo >> HWASanAccessInfo::MatchAllShift) & 0xff;
859 unsigned Size =
860 1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf);
861 bool CompileKernel =
862 (AccessInfo >> HWASanAccessInfo::CompileKernelShift) & 1;
863
864 OutStreamer->switchSection(Section: OutContext.getELFSection(
865 Section: ".text.hot", Type: ELF::SHT_PROGBITS,
866 Flags: ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, EntrySize: 0, Group: Sym->getName(),
867 /*IsComdat=*/true));
868
869 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_ELF_TypeFunction);
870 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Weak);
871 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Hidden);
872 OutStreamer->emitLabel(Symbol: Sym);
873
874 EmitToStreamer(Inst: MCInstBuilder(AArch64::SBFMXri)
875 .addReg(Reg: AArch64::X16)
876 .addReg(Reg)
877 .addImm(Val: 4)
878 .addImm(Val: 55));
879
880 if (IsFixedShadow) {
881 // Aarch64 makes it difficult to embed large constants in the code.
882 // Fortuitously, kShadowBaseAlignment == 32, so we use the 32-bit
883 // left-shift option in the MOV instruction. Combined with the 16-bit
884 // immediate, this is enough to represent any offset up to 2**48.
885 emitMOVZ(Dest: AArch64::X17, Imm: FixedShadowOffset >> 32, Shift: 32);
886 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRBBroX)
887 .addReg(Reg: AArch64::W16)
888 .addReg(Reg: AArch64::X17)
889 .addReg(Reg: AArch64::X16)
890 .addImm(Val: 0)
891 .addImm(Val: 0));
892 } else {
893 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRBBroX)
894 .addReg(Reg: AArch64::W16)
895 .addReg(Reg: IsShort ? AArch64::X20 : AArch64::X9)
896 .addReg(Reg: AArch64::X16)
897 .addImm(Val: 0)
898 .addImm(Val: 0));
899 }
900
901 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSXrs)
902 .addReg(Reg: AArch64::XZR)
903 .addReg(Reg: AArch64::X16)
904 .addReg(Reg)
905 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSR, Imm: 56)));
906 MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
907 EmitToStreamer(Inst: MCInstBuilder(AArch64::Bcc)
908 .addImm(Val: AArch64CC::NE)
909 .addExpr(Val: MCSymbolRefExpr::create(
910 Symbol: HandleMismatchOrPartialSym, Ctx&: OutContext)));
911 MCSymbol *ReturnSym = OutContext.createTempSymbol();
912 OutStreamer->emitLabel(Symbol: ReturnSym);
913 EmitToStreamer(Inst: MCInstBuilder(AArch64::RET).addReg(Reg: AArch64::LR));
914 OutStreamer->emitLabel(Symbol: HandleMismatchOrPartialSym);
915
916 if (HasMatchAllTag) {
917 EmitToStreamer(Inst: MCInstBuilder(AArch64::UBFMXri)
918 .addReg(Reg: AArch64::X17)
919 .addReg(Reg)
920 .addImm(Val: 56)
921 .addImm(Val: 63));
922 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSXri)
923 .addReg(Reg: AArch64::XZR)
924 .addReg(Reg: AArch64::X17)
925 .addImm(Val: MatchAllTag)
926 .addImm(Val: 0));
927 EmitToStreamer(
928 Inst: MCInstBuilder(AArch64::Bcc)
929 .addImm(Val: AArch64CC::EQ)
930 .addExpr(Val: MCSymbolRefExpr::create(Symbol: ReturnSym, Ctx&: OutContext)));
931 }
932
933 if (IsShort) {
934 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSWri)
935 .addReg(Reg: AArch64::WZR)
936 .addReg(Reg: AArch64::W16)
937 .addImm(Val: 15)
938 .addImm(Val: 0));
939 MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
940 EmitToStreamer(
941 Inst: MCInstBuilder(AArch64::Bcc)
942 .addImm(Val: AArch64CC::HI)
943 .addExpr(Val: MCSymbolRefExpr::create(Symbol: HandleMismatchSym, Ctx&: OutContext)));
944
945 EmitToStreamer(Inst: MCInstBuilder(AArch64::ANDXri)
946 .addReg(Reg: AArch64::X17)
947 .addReg(Reg)
948 .addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: 0xf, regSize: 64)));
949 if (Size != 1)
950 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
951 .addReg(Reg: AArch64::X17)
952 .addReg(Reg: AArch64::X17)
953 .addImm(Val: Size - 1)
954 .addImm(Val: 0));
955 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSWrs)
956 .addReg(Reg: AArch64::WZR)
957 .addReg(Reg: AArch64::W16)
958 .addReg(Reg: AArch64::W17)
959 .addImm(Val: 0));
960 EmitToStreamer(
961 Inst: MCInstBuilder(AArch64::Bcc)
962 .addImm(Val: AArch64CC::LS)
963 .addExpr(Val: MCSymbolRefExpr::create(Symbol: HandleMismatchSym, Ctx&: OutContext)));
964
965 EmitToStreamer(Inst: MCInstBuilder(AArch64::ORRXri)
966 .addReg(Reg: AArch64::X16)
967 .addReg(Reg)
968 .addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: 0xf, regSize: 64)));
969 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRBBui)
970 .addReg(Reg: AArch64::W16)
971 .addReg(Reg: AArch64::X16)
972 .addImm(Val: 0));
973 EmitToStreamer(
974 Inst: MCInstBuilder(AArch64::SUBSXrs)
975 .addReg(Reg: AArch64::XZR)
976 .addReg(Reg: AArch64::X16)
977 .addReg(Reg)
978 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSR, Imm: 56)));
979 EmitToStreamer(
980 Inst: MCInstBuilder(AArch64::Bcc)
981 .addImm(Val: AArch64CC::EQ)
982 .addExpr(Val: MCSymbolRefExpr::create(Symbol: ReturnSym, Ctx&: OutContext)));
983
984 OutStreamer->emitLabel(Symbol: HandleMismatchSym);
985 }
986
987 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPXpre)
988 .addReg(Reg: AArch64::SP)
989 .addReg(Reg: AArch64::X0)
990 .addReg(Reg: AArch64::X1)
991 .addReg(Reg: AArch64::SP)
992 .addImm(Val: -32));
993 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPXi)
994 .addReg(Reg: AArch64::FP)
995 .addReg(Reg: AArch64::LR)
996 .addReg(Reg: AArch64::SP)
997 .addImm(Val: 29));
998
999 if (Reg != AArch64::X0)
1000 emitMovXReg(Dest: AArch64::X0, Src: Reg);
1001 emitMOVZ(Dest: AArch64::X1, Imm: AccessInfo & HWASanAccessInfo::RuntimeMask, Shift: 0);
1002
1003 if (CompileKernel) {
1004 // The Linux kernel's dynamic loader doesn't support GOT relative
1005 // relocations, but it doesn't support late binding either, so just call
1006 // the function directly.
1007 EmitToStreamer(Inst: MCInstBuilder(AArch64::B).addExpr(Val: HwasanTagMismatchRef));
1008 } else {
1009 // Intentionally load the GOT entry and branch to it, rather than possibly
1010 // late binding the function, which may clobber the registers before we
1011 // have a chance to save them.
1012 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADRP)
1013 .addReg(Reg: AArch64::X16)
1014 .addExpr(Val: MCSpecifierExpr::create(Expr: HwasanTagMismatchRef,
1015 S: AArch64::S_GOT_PAGE,
1016 Ctx&: OutContext)));
1017 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
1018 .addReg(Reg: AArch64::X16)
1019 .addReg(Reg: AArch64::X16)
1020 .addExpr(Val: MCSpecifierExpr::create(Expr: HwasanTagMismatchRef,
1021 S: AArch64::S_GOT_LO12,
1022 Ctx&: OutContext)));
1023 EmitToStreamer(Inst: MCInstBuilder(AArch64::BR).addReg(Reg: AArch64::X16));
1024 }
1025 }
1026 this->STI = nullptr;
1027}
1028
1029static void emitAuthenticatedPointer(MCStreamer &OutStreamer,
1030 MCSymbol *StubLabel,
1031 const MCExpr *StubAuthPtrRef) {
1032 // sym$auth_ptr$key$disc:
1033 OutStreamer.emitLabel(Symbol: StubLabel);
1034 OutStreamer.emitValue(Value: StubAuthPtrRef, /*size=*/Size: 8);
1035}
1036
1037void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
1038 emitHwasanMemaccessSymbols(M);
1039
1040 const Triple &TT = M.getTargetTriple();
1041 if (TT.isOSBinFormatMachO()) {
1042 // Output authenticated pointers as indirect symbols, if we have any.
1043 MachineModuleInfoMachO &MMIMacho =
1044 MMI->getObjFileInfo<MachineModuleInfoMachO>();
1045
1046 auto Stubs = MMIMacho.getAuthGVStubList();
1047
1048 if (!Stubs.empty()) {
1049 // Switch to the "__auth_ptr" section.
1050 OutStreamer->switchSection(
1051 Section: OutContext.getMachOSection(Segment: "__DATA", Section: "__auth_ptr", TypeAndAttributes: MachO::S_REGULAR,
1052 K: SectionKind::getMetadata()));
1053 emitAlignment(Alignment: Align(8));
1054
1055 for (const auto &Stub : Stubs)
1056 emitAuthenticatedPointer(OutStreamer&: *OutStreamer, StubLabel: Stub.first, StubAuthPtrRef: Stub.second);
1057
1058 OutStreamer->addBlankLine();
1059 }
1060
1061 // Funny Darwin hack: This flag tells the linker that no global symbols
1062 // contain code that falls through to other global symbols (e.g. the obvious
1063 // implementation of multiple entry points). If this doesn't occur, the
1064 // linker can safely perform dead code stripping. Since LLVM never
1065 // generates code that does this, it is always safe to set.
1066 OutStreamer->emitSubsectionsViaSymbols();
1067 }
1068
1069 if (TT.isOSBinFormatELF()) {
1070 // Output authenticated pointers as indirect symbols, if we have any.
1071 MachineModuleInfoELF &MMIELF = MMI->getObjFileInfo<MachineModuleInfoELF>();
1072
1073 auto Stubs = MMIELF.getAuthGVStubList();
1074
1075 if (!Stubs.empty()) {
1076 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
1077 OutStreamer->switchSection(Section: TLOF.getDataSection());
1078 emitAlignment(Alignment: Align(8));
1079
1080 for (const auto &Stub : Stubs)
1081 emitAuthenticatedPointer(OutStreamer&: *OutStreamer, StubLabel: Stub.first, StubAuthPtrRef: Stub.second);
1082
1083 OutStreamer->addBlankLine();
1084 }
1085
1086 // With signed ELF GOT enabled, the linker looks at the symbol type to
1087 // choose between keys IA (for STT_FUNC) and DA (for other types). Symbols
1088 // for functions not defined in the module have STT_NOTYPE type by default.
1089 // This makes linker to emit signing schema with DA key (instead of IA) for
1090 // corresponding R_AARCH64_AUTH_GLOB_DAT dynamic reloc. To avoid that, force
1091 // all function symbols used in the module to have STT_FUNC type. See
1092 // https://github.com/ARM-software/abi-aa/blob/main/pauthabielf64/pauthabielf64.rst#default-signing-schema
1093 const auto *PtrAuthELFGOTFlag = mdconst::extract_or_null<ConstantInt>(
1094 MD: M.getModuleFlag(Key: "ptrauth-elf-got"));
1095 if (PtrAuthELFGOTFlag && PtrAuthELFGOTFlag->getZExtValue() == 1)
1096 for (const GlobalValue &GV : M.global_values())
1097 if (!GV.use_empty() && isa<Function>(Val: GV) &&
1098 !GV.getName().starts_with(Prefix: "llvm."))
1099 OutStreamer->emitSymbolAttribute(Symbol: getSymbol(GV: &GV),
1100 Attribute: MCSA_ELF_TypeFunction);
1101 }
1102
1103 // Emit stack and fault map information.
1104 FM.serializeToFaultMapSection();
1105
1106 // If import call optimization is enabled, emit the appropriate section.
1107 // We do this whether or not we recorded any import calls.
1108 if (EnableImportCallOptimization && TT.isOSBinFormatCOFF()) {
1109 OutStreamer->switchSection(Section: getObjFileLowering().getImportCallSection());
1110
1111 // Section always starts with some magic.
1112 constexpr char ImpCallMagic[12] = "Imp_Call_V1";
1113 OutStreamer->emitBytes(Data: StringRef{ImpCallMagic, sizeof(ImpCallMagic)});
1114
1115 // Layout of this section is:
1116 // Per section that contains calls to imported functions:
1117 // uint32_t SectionSize: Size in bytes for information in this section.
1118 // uint32_t Section Number
1119 // Per call to imported function in section:
1120 // uint32_t Kind: the kind of imported function.
1121 // uint32_t BranchOffset: the offset of the branch instruction in its
1122 // parent section.
1123 // uint32_t TargetSymbolId: the symbol id of the called function.
1124 for (auto &[Section, CallsToImportedFuncs] :
1125 SectionToImportedFunctionCalls) {
1126 unsigned SectionSize =
1127 sizeof(uint32_t) * (2 + 3 * CallsToImportedFuncs.size());
1128 OutStreamer->emitInt32(Value: SectionSize);
1129 OutStreamer->emitCOFFSecNumber(Symbol: Section->getBeginSymbol());
1130 for (auto &[CallsiteSymbol, CalledSymbol] : CallsToImportedFuncs) {
1131 // Kind is always IMAGE_REL_ARM64_DYNAMIC_IMPORT_CALL (0x13).
1132 OutStreamer->emitInt32(Value: 0x13);
1133 OutStreamer->emitCOFFSecOffset(Symbol: CallsiteSymbol);
1134 OutStreamer->emitCOFFSymbolIndex(Symbol: CalledSymbol);
1135 }
1136 }
1137 }
1138}
1139
1140void AArch64AsmPrinter::emitLOHs() {
1141 SmallVector<MCSymbol *, 3> MCArgs;
1142
1143 for (const auto &D : AArch64FI->getLOHContainer()) {
1144 for (const MachineInstr *MI : D.getArgs()) {
1145 MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(x: MI);
1146 assert(LabelIt != LOHInstToLabel.end() &&
1147 "Label hasn't been inserted for LOH related instruction");
1148 MCArgs.push_back(Elt: LabelIt->second);
1149 }
1150 OutStreamer->emitLOHDirective(Kind: D.getKind(), Args: MCArgs);
1151 MCArgs.clear();
1152 }
1153}
1154
1155void AArch64AsmPrinter::emitFunctionBodyEnd() {
1156 if (!AArch64FI->getLOHRelated().empty())
1157 emitLOHs();
1158}
1159
1160/// GetCPISymbol - Return the symbol for the specified constant pool entry.
1161MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
1162 // Darwin uses a linker-private symbol name for constant-pools (to
1163 // avoid addends on the relocation?), ELF has no such concept and
1164 // uses a normal private symbol.
1165 if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
1166 return OutContext.getOrCreateSymbol(
1167 Name: Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
1168 Twine(getFunctionNumber()) + "_" + Twine(CPID));
1169
1170 return AsmPrinter::GetCPISymbol(CPID);
1171}
1172
1173void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
1174 raw_ostream &O) {
1175 const MachineOperand &MO = MI->getOperand(i: OpNum);
1176 switch (MO.getType()) {
1177 default:
1178 llvm_unreachable("<unknown operand type>");
1179 case MachineOperand::MO_Register: {
1180 Register Reg = MO.getReg();
1181 assert(Reg.isPhysical());
1182 assert(!MO.getSubReg() && "Subregs should be eliminated!");
1183 O << AArch64InstPrinter::getRegisterName(Reg);
1184 break;
1185 }
1186 case MachineOperand::MO_Immediate: {
1187 O << MO.getImm();
1188 break;
1189 }
1190 case MachineOperand::MO_GlobalAddress: {
1191 PrintSymbolOperand(MO, OS&: O);
1192 break;
1193 }
1194 case MachineOperand::MO_BlockAddress: {
1195 MCSymbol *Sym = GetBlockAddressSymbol(BA: MO.getBlockAddress());
1196 Sym->print(OS&: O, MAI);
1197 break;
1198 }
1199 }
1200}
1201
1202bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
1203 raw_ostream &O) {
1204 Register Reg = MO.getReg();
1205 switch (Mode) {
1206 default:
1207 return true; // Unknown mode.
1208 case 'w':
1209 Reg = getWRegFromXReg(Reg);
1210 break;
1211 case 'x':
1212 Reg = getXRegFromWReg(Reg);
1213 break;
1214 case 't':
1215 Reg = getXRegFromXRegTuple(RegTuple: Reg);
1216 break;
1217 }
1218
1219 O << AArch64InstPrinter::getRegisterName(Reg);
1220 return false;
1221}
1222
1223// Prints the register in MO using class RC using the offset in the
1224// new register class. This should not be used for cross class
1225// printing.
1226bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
1227 const TargetRegisterClass *RC,
1228 unsigned AltName, raw_ostream &O) {
1229 assert(MO.isReg() && "Should only get here with a register!");
1230 const TargetRegisterInfo *RI = STI->getRegisterInfo();
1231 Register Reg = MO.getReg();
1232 MCRegister RegToPrint = RC->getRegister(i: RI->getEncodingValue(Reg));
1233 if (!RI->regsOverlap(RegA: RegToPrint, RegB: Reg))
1234 return true;
1235 O << AArch64InstPrinter::getRegisterName(Reg: RegToPrint, AltIdx: AltName);
1236 return false;
1237}
1238
1239bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
1240 const char *ExtraCode, raw_ostream &O) {
1241 const MachineOperand &MO = MI->getOperand(i: OpNum);
1242
1243 // First try the generic code, which knows about modifiers like 'c' and 'n'.
1244 if (!AsmPrinter::PrintAsmOperand(MI, OpNo: OpNum, ExtraCode, OS&: O))
1245 return false;
1246
1247 // Does this asm operand have a single letter operand modifier?
1248 if (ExtraCode && ExtraCode[0]) {
1249 if (ExtraCode[1] != 0)
1250 return true; // Unknown modifier.
1251
1252 switch (ExtraCode[0]) {
1253 default:
1254 return true; // Unknown modifier.
1255 case 'w': // Print W register
1256 case 'x': // Print X register
1257 if (MO.isReg())
1258 return printAsmMRegister(MO, Mode: ExtraCode[0], O);
1259 if (MO.isImm() && MO.getImm() == 0) {
1260 unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
1261 O << AArch64InstPrinter::getRegisterName(Reg);
1262 return false;
1263 }
1264 printOperand(MI, OpNum, O);
1265 return false;
1266 case 'b': // Print B register.
1267 case 'h': // Print H register.
1268 case 's': // Print S register.
1269 case 'd': // Print D register.
1270 case 'q': // Print Q register.
1271 case 'z': // Print Z register.
1272 if (MO.isReg()) {
1273 const TargetRegisterClass *RC;
1274 switch (ExtraCode[0]) {
1275 case 'b':
1276 RC = &AArch64::FPR8RegClass;
1277 break;
1278 case 'h':
1279 RC = &AArch64::FPR16RegClass;
1280 break;
1281 case 's':
1282 RC = &AArch64::FPR32RegClass;
1283 break;
1284 case 'd':
1285 RC = &AArch64::FPR64RegClass;
1286 break;
1287 case 'q':
1288 RC = &AArch64::FPR128RegClass;
1289 break;
1290 case 'z':
1291 RC = &AArch64::ZPRRegClass;
1292 break;
1293 default:
1294 return true;
1295 }
1296 return printAsmRegInClass(MO, RC, AltName: AArch64::NoRegAltName, O);
1297 }
1298 printOperand(MI, OpNum, O);
1299 return false;
1300 }
1301 }
1302
1303 // According to ARM, we should emit x and v registers unless we have a
1304 // modifier.
1305 if (MO.isReg()) {
1306 Register Reg = MO.getReg();
1307
1308 // If this is a w or x register, print an x register.
1309 if (AArch64::GPR32allRegClass.contains(Reg) ||
1310 AArch64::GPR64allRegClass.contains(Reg))
1311 return printAsmMRegister(MO, Mode: 'x', O);
1312
1313 // If this is an x register tuple, print an x register.
1314 if (AArch64::GPR64x8ClassRegClass.contains(Reg))
1315 return printAsmMRegister(MO, Mode: 't', O);
1316
1317 unsigned AltName = AArch64::NoRegAltName;
1318 const TargetRegisterClass *RegClass;
1319 if (AArch64::ZPRRegClass.contains(Reg)) {
1320 RegClass = &AArch64::ZPRRegClass;
1321 } else if (AArch64::PPRRegClass.contains(Reg)) {
1322 RegClass = &AArch64::PPRRegClass;
1323 } else if (AArch64::PNRRegClass.contains(Reg)) {
1324 RegClass = &AArch64::PNRRegClass;
1325 } else {
1326 RegClass = &AArch64::FPR128RegClass;
1327 AltName = AArch64::vreg;
1328 }
1329
1330 // If this is a b, h, s, d, or q register, print it as a v register.
1331 return printAsmRegInClass(MO, RC: RegClass, AltName, O);
1332 }
1333
1334 printOperand(MI, OpNum, O);
1335 return false;
1336}
1337
1338bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
1339 unsigned OpNum,
1340 const char *ExtraCode,
1341 raw_ostream &O) {
1342 if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
1343 return true; // Unknown modifier.
1344
1345 const MachineOperand &MO = MI->getOperand(i: OpNum);
1346 assert(MO.isReg() && "unexpected inline asm memory operand");
1347 O << "[" << AArch64InstPrinter::getRegisterName(Reg: MO.getReg()) << "]";
1348 return false;
1349}
1350
1351void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
1352 raw_ostream &OS) {
1353 unsigned NOps = MI->getNumOperands();
1354 assert(NOps == 4);
1355 OS << '\t' << MAI.getCommentString() << "DEBUG_VALUE: ";
1356 // cast away const; DIetc do not take const operands for some reason.
1357 OS << MI->getDebugVariable()->getName();
1358 OS << " <- ";
1359 // Frame address. Currently handles register +- offset only.
1360 assert(MI->isIndirectDebugValue());
1361 OS << '[';
1362 for (unsigned I = 0, E = llvm::size(Range: MI->debug_operands()); I < E; ++I) {
1363 if (I != 0)
1364 OS << ", ";
1365 printOperand(MI, OpNum: I, O&: OS);
1366 }
1367 OS << ']';
1368 OS << "+";
1369 printOperand(MI, OpNum: NOps - 2, O&: OS);
1370}
1371
1372void AArch64AsmPrinter::emitJumpTableImpl(const MachineJumpTableInfo &MJTI,
1373 ArrayRef<unsigned> JumpTableIndices) {
1374 // Fast return if there is nothing to emit to avoid creating empty sections.
1375 if (JumpTableIndices.empty())
1376 return;
1377 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
1378 const auto &F = MF->getFunction();
1379 ArrayRef<MachineJumpTableEntry> JT = MJTI.getJumpTables();
1380
1381 MCSection *ReadOnlySec = nullptr;
1382 if (TM.Options.EnableStaticDataPartitioning) {
1383 ReadOnlySec =
1384 TLOF.getSectionForJumpTable(F, TM, JTE: &JT[JumpTableIndices.front()]);
1385 } else {
1386 ReadOnlySec = TLOF.getSectionForJumpTable(F, TM);
1387 }
1388 OutStreamer->switchSection(Section: ReadOnlySec);
1389
1390 auto AFI = MF->getInfo<AArch64FunctionInfo>();
1391 for (unsigned JTI : JumpTableIndices) {
1392 const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
1393
1394 // If this jump table was deleted, ignore it.
1395 if (JTBBs.empty()) continue;
1396
1397 unsigned Size = AFI->getJumpTableEntrySize(Idx: JTI);
1398 emitAlignment(Alignment: Align(Size));
1399 OutStreamer->emitLabel(Symbol: GetJTISymbol(JTID: JTI));
1400
1401 const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(Idx: JTI);
1402 const MCExpr *Base = MCSymbolRefExpr::create(Symbol: BaseSym, Ctx&: OutContext);
1403
1404 for (auto *JTBB : JTBBs) {
1405 const MCExpr *Value =
1406 MCSymbolRefExpr::create(Symbol: JTBB->getSymbol(), Ctx&: OutContext);
1407
1408 // Each entry is:
1409 // .byte/.hword (LBB - Lbase)>>2
1410 // or plain:
1411 // .word LBB - Lbase
1412 Value = MCBinaryExpr::createSub(LHS: Value, RHS: Base, Ctx&: OutContext);
1413 if (Size != 4)
1414 Value = MCBinaryExpr::createLShr(
1415 LHS: Value, RHS: MCConstantExpr::create(Value: 2, Ctx&: OutContext), Ctx&: OutContext);
1416
1417 OutStreamer->emitValue(Value, Size);
1418 }
1419 }
1420}
1421
1422std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
1423 codeview::JumpTableEntrySize>
1424AArch64AsmPrinter::getCodeViewJumpTableInfo(int JTI,
1425 const MachineInstr *BranchInstr,
1426 const MCSymbol *BranchLabel) const {
1427 const auto AFI = MF->getInfo<AArch64FunctionInfo>();
1428 const auto Base = AArch64FI->getJumpTableEntryPCRelSymbol(Idx: JTI);
1429 codeview::JumpTableEntrySize EntrySize;
1430 switch (AFI->getJumpTableEntrySize(Idx: JTI)) {
1431 case 1:
1432 EntrySize = codeview::JumpTableEntrySize::UInt8ShiftLeft;
1433 break;
1434 case 2:
1435 EntrySize = codeview::JumpTableEntrySize::UInt16ShiftLeft;
1436 break;
1437 case 4:
1438 EntrySize = codeview::JumpTableEntrySize::Int32;
1439 break;
1440 default:
1441 llvm_unreachable("Unexpected jump table entry size");
1442 }
1443 return std::make_tuple(args: Base, args: 0, args&: BranchLabel, args&: EntrySize);
1444}
1445
1446void AArch64AsmPrinter::emitFunctionEntryLabel() {
1447 const Triple &TT = TM.getTargetTriple();
1448 if (TT.isOSBinFormatELF() &&
1449 (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall ||
1450 MF->getFunction().getCallingConv() ==
1451 CallingConv::AArch64_SVE_VectorCall ||
1452 MF->getInfo<AArch64FunctionInfo>()->isSVECC())) {
1453 auto *TS =
1454 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
1455 TS->emitDirectiveVariantPCS(Symbol: CurrentFnSym);
1456 }
1457
1458 AsmPrinter::emitFunctionEntryLabel();
1459
1460 if (TT.isWindowsArm64EC() && !MF->getFunction().hasLocalLinkage()) {
1461 // For ARM64EC targets, a function definition's name is mangled differently
1462 // from the normal symbol, emit required aliases here.
1463 auto emitFunctionAlias = [&](MCSymbol *Src, MCSymbol *Dst) {
1464 OutStreamer->emitSymbolAttribute(Symbol: Src, Attribute: MCSA_WeakAntiDep);
1465 OutStreamer->emitAssignment(
1466 Symbol: Src, Value: MCSymbolRefExpr::create(Symbol: Dst, Ctx&: MMI->getContext()));
1467 };
1468
1469 auto getSymbolFromMetadata = [&](StringRef Name) {
1470 MCSymbol *Sym = nullptr;
1471 if (MDNode *Node = MF->getFunction().getMetadata(Kind: Name)) {
1472 StringRef NameStr = cast<MDString>(Val: Node->getOperand(I: 0))->getString();
1473 Sym = MMI->getContext().getOrCreateSymbol(Name: NameStr);
1474 }
1475 return Sym;
1476 };
1477
1478 SmallVector<MDNode *> UnmangledNames;
1479 MF->getFunction().getMetadata(Kind: "arm64ec_unmangled_name", MDs&: UnmangledNames);
1480 for (MDNode *Node : UnmangledNames) {
1481 StringRef NameStr = cast<MDString>(Val: Node->getOperand(I: 0))->getString();
1482 MCSymbol *UnmangledSym = MMI->getContext().getOrCreateSymbol(Name: NameStr);
1483 if (std::optional<std::string> MangledName =
1484 getArm64ECMangledFunctionName(Name: UnmangledSym->getName())) {
1485 MCSymbol *ECMangledSym =
1486 MMI->getContext().getOrCreateSymbol(Name: *MangledName);
1487 emitFunctionAlias(UnmangledSym, ECMangledSym);
1488 }
1489 }
1490 if (MCSymbol *ECMangledSym =
1491 getSymbolFromMetadata("arm64ec_ecmangled_name"))
1492 emitFunctionAlias(ECMangledSym, CurrentFnSym);
1493 }
1494}
1495
1496void AArch64AsmPrinter::emitXXStructor(const DataLayout &DL,
1497 const Constant *CV) {
1498 LLVMContext &C = CV->getContext();
1499 assert(!isa<ConstantPtrAuth>(CV) &&
1500 "ctors/dtors are to be signed by asm printer");
1501
1502 if (PtrauthInitFini) {
1503 IntegerType *Int32Ty = IntegerType::get(C, NumBits: 32);
1504 IntegerType *Int64Ty = IntegerType::get(C, NumBits: 64);
1505 PointerType *PtrTy = PointerType::get(C, AddressSpace: 0);
1506
1507 ConstantInt *Key = ConstantInt::get(Ty: Int32Ty, V: AArch64PAuth::InitFiniKey);
1508 ConstantInt *IntDisc = ConstantInt::get(
1509 Ty: Int64Ty, V: AArch64PAuth::InitFiniPointerConstantDiscriminator);
1510 Constant *Null = ConstantPointerNull::get(T: PtrTy);
1511 Constant *AddressDisc = Null;
1512 if (PtrauthInitFiniAddressDisc) {
1513 uint64_t Marker = ConstantPtrAuth::AddrDiscriminator_CtorsDtors;
1514 AddressDisc =
1515 ConstantExpr::getIntToPtr(C: ConstantInt::get(Ty: Int64Ty, V: Marker), Ty: PtrTy);
1516 }
1517
1518 CV = ConstantPtrAuth::get(Ptr: const_cast<Constant *>(CV), Key, Disc: IntDisc,
1519 AddrDisc: AddressDisc, /*DeactivationSymbol=*/Null);
1520 }
1521
1522 // Signed pointers will be lowered by AArch64AsmPrinter::lowerConstantPtrAuth.
1523 AsmPrinter::emitXXStructor(DL, CV);
1524}
1525
1526void AArch64AsmPrinter::emitGlobalAlias(const Module &M,
1527 const GlobalAlias &GA) {
1528 if (auto F = dyn_cast_or_null<Function>(Val: GA.getAliasee())) {
1529 // Global aliases must point to a definition, but unmangled patchable
1530 // symbols are special and need to point to an undefined symbol with "EXP+"
1531 // prefix. Such undefined symbol is resolved by the linker by creating
1532 // x86 thunk that jumps back to the actual EC target.
1533 if (MDNode *Node = F->getMetadata(Kind: "arm64ec_exp_name")) {
1534 StringRef ExpStr = cast<MDString>(Val: Node->getOperand(I: 0))->getString();
1535 MCSymbol *ExpSym = MMI->getContext().getOrCreateSymbol(Name: ExpStr);
1536 MCSymbol *Sym = MMI->getContext().getOrCreateSymbol(Name: GA.getName());
1537
1538 OutStreamer->beginCOFFSymbolDef(Symbol: ExpSym);
1539 OutStreamer->emitCOFFSymbolStorageClass(StorageClass: COFF::IMAGE_SYM_CLASS_EXTERNAL);
1540 OutStreamer->emitCOFFSymbolType(Type: COFF::IMAGE_SYM_DTYPE_FUNCTION
1541 << COFF::SCT_COMPLEX_TYPE_SHIFT);
1542 OutStreamer->endCOFFSymbolDef();
1543
1544 OutStreamer->beginCOFFSymbolDef(Symbol: Sym);
1545 OutStreamer->emitCOFFSymbolStorageClass(StorageClass: COFF::IMAGE_SYM_CLASS_EXTERNAL);
1546 OutStreamer->emitCOFFSymbolType(Type: COFF::IMAGE_SYM_DTYPE_FUNCTION
1547 << COFF::SCT_COMPLEX_TYPE_SHIFT);
1548 OutStreamer->endCOFFSymbolDef();
1549 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Weak);
1550 OutStreamer->emitAssignment(
1551 Symbol: Sym, Value: MCSymbolRefExpr::create(Symbol: ExpSym, Ctx&: MMI->getContext()));
1552 return;
1553 }
1554 }
1555 AsmPrinter::emitGlobalAlias(M, GA);
1556}
1557
1558/// Small jump tables contain an unsigned byte or half, representing the offset
1559/// from the lowest-addressed possible destination to the desired basic
1560/// block. Since all instructions are 4-byte aligned, this is further compressed
1561/// by counting in instructions rather than bytes (i.e. divided by 4). So, to
1562/// materialize the correct destination we need:
1563///
1564/// adr xDest, .LBB0_0
1565/// ldrb wScratch, [xTable, xEntry] (with "lsl #1" for ldrh).
1566/// add xDest, xDest, xScratch (with "lsl #2" for smaller entries)
1567void AArch64AsmPrinter::LowerJumpTableDest(llvm::MCStreamer &OutStreamer,
1568 const llvm::MachineInstr &MI) {
1569 Register DestReg = MI.getOperand(i: 0).getReg();
1570 Register ScratchReg = MI.getOperand(i: 1).getReg();
1571 Register ScratchRegW =
1572 STI->getRegisterInfo()->getSubReg(Reg: ScratchReg, Idx: AArch64::sub_32);
1573 Register TableReg = MI.getOperand(i: 2).getReg();
1574 Register EntryReg = MI.getOperand(i: 3).getReg();
1575 int JTIdx = MI.getOperand(i: 4).getIndex();
1576 int Size = AArch64FI->getJumpTableEntrySize(Idx: JTIdx);
1577
1578 // This has to be first because the compression pass based its reachability
1579 // calculations on the start of the JumpTableDest instruction.
1580 auto Label =
1581 MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(Idx: JTIdx);
1582
1583 // If we don't already have a symbol to use as the base, use the ADR
1584 // instruction itself.
1585 if (!Label) {
1586 Label = MF->getContext().createTempSymbol();
1587 AArch64FI->setJumpTableEntryInfo(Idx: JTIdx, Size, PCRelSym: Label);
1588 OutStreamer.emitLabel(Symbol: Label);
1589 }
1590
1591 auto LabelExpr = MCSymbolRefExpr::create(Symbol: Label, Ctx&: MF->getContext());
1592 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::ADR)
1593 .addReg(Reg: DestReg)
1594 .addExpr(Val: LabelExpr));
1595
1596 // Load the number of instruction-steps to offset from the label.
1597 unsigned LdrOpcode;
1598 switch (Size) {
1599 case 1: LdrOpcode = AArch64::LDRBBroX; break;
1600 case 2: LdrOpcode = AArch64::LDRHHroX; break;
1601 case 4: LdrOpcode = AArch64::LDRSWroX; break;
1602 default:
1603 llvm_unreachable("Unknown jump table size");
1604 }
1605
1606 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(LdrOpcode)
1607 .addReg(Reg: Size == 4 ? ScratchReg : ScratchRegW)
1608 .addReg(Reg: TableReg)
1609 .addReg(Reg: EntryReg)
1610 .addImm(Val: 0)
1611 .addImm(Val: Size == 1 ? 0 : 1));
1612
1613 // Add to the already materialized base label address, multiplying by 4 if
1614 // compressed.
1615 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::ADDXrs)
1616 .addReg(Reg: DestReg)
1617 .addReg(Reg: DestReg)
1618 .addReg(Reg: ScratchReg)
1619 .addImm(Val: Size == 4 ? 0 : 2));
1620}
1621
1622void AArch64AsmPrinter::LowerHardenedBRJumpTable(const MachineInstr &MI) {
1623 const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
1624 assert(MJTI && "Can't lower jump-table dispatch without JTI");
1625
1626 const std::vector<MachineJumpTableEntry> &JTs = MJTI->getJumpTables();
1627 assert(!JTs.empty() && "Invalid JT index for jump-table dispatch");
1628
1629 // Emit:
1630 // mov x17, #<size of table> ; depending on table size, with MOVKs
1631 // cmp x16, x17 ; or #imm if table size fits in 12-bit
1632 // csel x16, x16, xzr, ls ; check for index overflow
1633 //
1634 // adrp x17, Ltable@PAGE ; materialize table address
1635 // add x17, Ltable@PAGEOFF
1636 // ldrsw x16, [x17, x16, lsl #2] ; load table entry
1637 //
1638 // Lanchor:
1639 // adr x17, Lanchor ; compute target address
1640 // add x16, x17, x16
1641 // br x16 ; branch to target
1642
1643 MachineOperand JTOp = MI.getOperand(i: 0);
1644
1645 unsigned JTI = JTOp.getIndex();
1646 assert(!AArch64FI->getJumpTableEntryPCRelSymbol(JTI) &&
1647 "unsupported compressed jump table");
1648
1649 const uint64_t NumTableEntries = JTs[JTI].MBBs.size();
1650
1651 // cmp only supports a 12-bit immediate. If we need more, materialize the
1652 // immediate, using x17 as a scratch register.
1653 uint64_t MaxTableEntry = NumTableEntries - 1;
1654 if (isUInt<12>(x: MaxTableEntry)) {
1655 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::SUBSXri)
1656 .addReg(Reg: AArch64::XZR)
1657 .addReg(Reg: AArch64::X16)
1658 .addImm(Val: MaxTableEntry)
1659 .addImm(Val: 0));
1660 } else {
1661 emitMOVZ(Dest: AArch64::X17, Imm: static_cast<uint16_t>(MaxTableEntry), Shift: 0);
1662 // It's sad that we have to manually materialize instructions, but we can't
1663 // trivially reuse the main pseudo expansion logic.
1664 // A MOVK sequence is easy enough to generate and handles the general case.
1665 for (int Offset = 16; Offset < 64; Offset += 16) {
1666 if ((MaxTableEntry >> Offset) == 0)
1667 break;
1668 emitMOVK(Dest: AArch64::X17, Imm: static_cast<uint16_t>(MaxTableEntry >> Offset),
1669 Shift: Offset);
1670 }
1671 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::SUBSXrs)
1672 .addReg(Reg: AArch64::XZR)
1673 .addReg(Reg: AArch64::X16)
1674 .addReg(Reg: AArch64::X17)
1675 .addImm(Val: 0));
1676 }
1677
1678 // This picks entry #0 on failure.
1679 // We might want to trap instead.
1680 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::CSELXr)
1681 .addReg(Reg: AArch64::X16)
1682 .addReg(Reg: AArch64::X16)
1683 .addReg(Reg: AArch64::XZR)
1684 .addImm(Val: AArch64CC::LS));
1685
1686 // Prepare the @PAGE/@PAGEOFF low/high operands.
1687 MachineOperand JTMOHi(JTOp), JTMOLo(JTOp);
1688 MCOperand JTMCHi, JTMCLo;
1689
1690 JTMOHi.setTargetFlags(AArch64II::MO_PAGE);
1691 JTMOLo.setTargetFlags(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
1692
1693 MCInstLowering.lowerOperand(MO: JTMOHi, MCOp&: JTMCHi);
1694 MCInstLowering.lowerOperand(MO: JTMOLo, MCOp&: JTMCLo);
1695
1696 EmitToStreamer(
1697 S&: *OutStreamer,
1698 Inst: MCInstBuilder(AArch64::ADRP).addReg(Reg: AArch64::X17).addOperand(Op: JTMCHi));
1699
1700 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ADDXri)
1701 .addReg(Reg: AArch64::X17)
1702 .addReg(Reg: AArch64::X17)
1703 .addOperand(Op: JTMCLo)
1704 .addImm(Val: 0));
1705
1706 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::LDRSWroX)
1707 .addReg(Reg: AArch64::X16)
1708 .addReg(Reg: AArch64::X17)
1709 .addReg(Reg: AArch64::X16)
1710 .addImm(Val: 0)
1711 .addImm(Val: 1));
1712
1713 MCSymbol *AdrLabel = MF->getContext().createTempSymbol();
1714 const auto *AdrLabelE = MCSymbolRefExpr::create(Symbol: AdrLabel, Ctx&: MF->getContext());
1715 AArch64FI->setJumpTableEntryInfo(Idx: JTI, Size: 4, PCRelSym: AdrLabel);
1716
1717 OutStreamer->emitLabel(Symbol: AdrLabel);
1718 EmitToStreamer(
1719 S&: *OutStreamer,
1720 Inst: MCInstBuilder(AArch64::ADR).addReg(Reg: AArch64::X17).addExpr(Val: AdrLabelE));
1721
1722 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ADDXrs)
1723 .addReg(Reg: AArch64::X16)
1724 .addReg(Reg: AArch64::X17)
1725 .addReg(Reg: AArch64::X16)
1726 .addImm(Val: 0));
1727
1728 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::BR).addReg(Reg: AArch64::X16));
1729}
1730
1731void AArch64AsmPrinter::LowerMOPS(llvm::MCStreamer &OutStreamer,
1732 const llvm::MachineInstr &MI) {
1733 unsigned Opcode = MI.getOpcode();
1734 assert(STI->hasMOPS());
1735 assert(STI->hasMTE() || Opcode != AArch64::MOPSMemorySetTaggingPseudo);
1736
1737 const auto Ops = [Opcode]() -> std::array<unsigned, 3> {
1738 if (Opcode == AArch64::MOPSMemoryCopyPseudo)
1739 return {AArch64::CPYFP, AArch64::CPYFM, AArch64::CPYFE};
1740 if (Opcode == AArch64::MOPSMemoryMovePseudo)
1741 return {AArch64::CPYP, AArch64::CPYM, AArch64::CPYE};
1742 if (Opcode == AArch64::MOPSMemorySetPseudo)
1743 return {AArch64::SETP, AArch64::SETM, AArch64::SETE};
1744 if (Opcode == AArch64::MOPSMemorySetTaggingPseudo)
1745 return {AArch64::SETGP, AArch64::SETGM, AArch64::MOPSSETGE};
1746 llvm_unreachable("Unhandled memory operation pseudo");
1747 }();
1748 const bool IsSet = Opcode == AArch64::MOPSMemorySetPseudo ||
1749 Opcode == AArch64::MOPSMemorySetTaggingPseudo;
1750
1751 for (auto Op : Ops) {
1752 int i = 0;
1753 auto MCIB = MCInstBuilder(Op);
1754 // Destination registers
1755 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1756 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1757 if (!IsSet)
1758 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1759 // Input registers
1760 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1761 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1762 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1763
1764 EmitToStreamer(S&: OutStreamer, Inst: MCIB);
1765 }
1766}
1767
1768void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
1769 const MachineInstr &MI) {
1770 unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
1771
1772 auto &Ctx = OutStreamer.getContext();
1773 MCSymbol *MILabel = Ctx.createTempSymbol();
1774 OutStreamer.emitLabel(Symbol: MILabel);
1775
1776 SM.recordStackMap(L: *MILabel, MI);
1777 assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1778
1779 // Scan ahead to trim the shadow.
1780 const MachineBasicBlock &MBB = *MI.getParent();
1781 MachineBasicBlock::const_iterator MII(MI);
1782 ++MII;
1783 while (NumNOPBytes > 0) {
1784 if (MII == MBB.end() || MII->isCall() ||
1785 MII->getOpcode() == AArch64::DBG_VALUE ||
1786 MII->getOpcode() == TargetOpcode::PATCHPOINT ||
1787 MII->getOpcode() == TargetOpcode::STACKMAP)
1788 break;
1789 ++MII;
1790 NumNOPBytes -= 4;
1791 }
1792
1793 // Emit nops.
1794 for (unsigned i = 0; i < NumNOPBytes; i += 4)
1795 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::NOP));
1796}
1797
1798// Lower a patchpoint of the form:
1799// [<def>], <id>, <numBytes>, <target>, <numArgs>
1800void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1801 const MachineInstr &MI) {
1802 auto &Ctx = OutStreamer.getContext();
1803 MCSymbol *MILabel = Ctx.createTempSymbol();
1804 OutStreamer.emitLabel(Symbol: MILabel);
1805 SM.recordPatchPoint(L: *MILabel, MI);
1806
1807 PatchPointOpers Opers(&MI);
1808
1809 int64_t CallTarget = Opers.getCallTarget().getImm();
1810 unsigned EncodedBytes = 0;
1811 if (CallTarget) {
1812 assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
1813 "High 16 bits of call target should be zero.");
1814 Register ScratchReg = MI.getOperand(i: Opers.getNextScratchIdx()).getReg();
1815 EncodedBytes = 16;
1816 // Materialize the jump address:
1817 emitMOVZ(Dest: ScratchReg, Imm: (CallTarget >> 32) & 0xFFFF, Shift: 32);
1818 emitMOVK(Dest: ScratchReg, Imm: (CallTarget >> 16) & 0xFFFF, Shift: 16);
1819 emitMOVK(Dest: ScratchReg, Imm: CallTarget & 0xFFFF, Shift: 0);
1820 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::BLR).addReg(Reg: ScratchReg));
1821 }
1822 // Emit padding.
1823 unsigned NumBytes = Opers.getNumPatchBytes();
1824 assert(NumBytes >= EncodedBytes &&
1825 "Patchpoint can't request size less than the length of a call.");
1826 assert((NumBytes - EncodedBytes) % 4 == 0 &&
1827 "Invalid number of NOP bytes requested!");
1828 for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
1829 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::NOP));
1830}
1831
1832void AArch64AsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1833 const MachineInstr &MI) {
1834 StatepointOpers SOpers(&MI);
1835 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
1836 assert(PatchBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1837 for (unsigned i = 0; i < PatchBytes; i += 4)
1838 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::NOP));
1839 } else {
1840 // Lower call target and choose correct opcode
1841 const MachineOperand &CallTarget = SOpers.getCallTarget();
1842 MCOperand CallTargetMCOp;
1843 unsigned CallOpcode;
1844 switch (CallTarget.getType()) {
1845 case MachineOperand::MO_GlobalAddress:
1846 case MachineOperand::MO_ExternalSymbol:
1847 MCInstLowering.lowerOperand(MO: CallTarget, MCOp&: CallTargetMCOp);
1848 CallOpcode = AArch64::BL;
1849 break;
1850 case MachineOperand::MO_Immediate:
1851 CallTargetMCOp = MCOperand::createImm(Val: CallTarget.getImm());
1852 CallOpcode = AArch64::BL;
1853 break;
1854 case MachineOperand::MO_Register:
1855 CallTargetMCOp = MCOperand::createReg(Reg: CallTarget.getReg());
1856 CallOpcode = AArch64::BLR;
1857 break;
1858 default:
1859 llvm_unreachable("Unsupported operand type in statepoint call target");
1860 break;
1861 }
1862
1863 EmitToStreamer(S&: OutStreamer,
1864 Inst: MCInstBuilder(CallOpcode).addOperand(Op: CallTargetMCOp));
1865 }
1866
1867 auto &Ctx = OutStreamer.getContext();
1868 MCSymbol *MILabel = Ctx.createTempSymbol();
1869 OutStreamer.emitLabel(Symbol: MILabel);
1870 SM.recordStatepoint(L: *MILabel, MI);
1871}
1872
1873void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) {
1874 // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
1875 // <opcode>, <operands>
1876
1877 Register DefRegister = FaultingMI.getOperand(i: 0).getReg();
1878 FaultMaps::FaultKind FK =
1879 static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(i: 1).getImm());
1880 MCSymbol *HandlerLabel = FaultingMI.getOperand(i: 2).getMBB()->getSymbol();
1881 unsigned Opcode = FaultingMI.getOperand(i: 3).getImm();
1882 unsigned OperandsBeginIdx = 4;
1883
1884 auto &Ctx = OutStreamer->getContext();
1885 MCSymbol *FaultingLabel = Ctx.createTempSymbol();
1886 OutStreamer->emitLabel(Symbol: FaultingLabel);
1887
1888 assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
1889 FM.recordFaultingOp(FaultTy: FK, FaultingLabel, HandlerLabel);
1890
1891 MCInst MI;
1892 MI.setOpcode(Opcode);
1893
1894 if (DefRegister != (Register)0)
1895 MI.addOperand(Op: MCOperand::createReg(Reg: DefRegister));
1896
1897 for (const MachineOperand &MO :
1898 llvm::drop_begin(RangeOrContainer: FaultingMI.operands(), N: OperandsBeginIdx)) {
1899 MCOperand Dest;
1900 lowerOperand(MO, MCOp&: Dest);
1901 MI.addOperand(Op: Dest);
1902 }
1903
1904 OutStreamer->AddComment(T: "on-fault: " + HandlerLabel->getName());
1905 EmitToStreamer(Inst: MI);
1906}
1907
1908void AArch64AsmPrinter::emitMovXReg(Register Dest, Register Src) {
1909 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ORRXrs)
1910 .addReg(Reg: Dest)
1911 .addReg(Reg: AArch64::XZR)
1912 .addReg(Reg: Src)
1913 .addImm(Val: 0));
1914}
1915
1916void AArch64AsmPrinter::emitMOVZ(Register Dest, uint64_t Imm, unsigned Shift) {
1917 bool Is64Bit = AArch64::GPR64RegClass.contains(Reg: Dest);
1918 EmitToStreamer(S&: *OutStreamer,
1919 Inst: MCInstBuilder(Is64Bit ? AArch64::MOVZXi : AArch64::MOVZWi)
1920 .addReg(Reg: Dest)
1921 .addImm(Val: Imm)
1922 .addImm(Val: Shift));
1923}
1924
1925void AArch64AsmPrinter::emitMOVK(Register Dest, uint64_t Imm, unsigned Shift) {
1926 bool Is64Bit = AArch64::GPR64RegClass.contains(Reg: Dest);
1927 EmitToStreamer(S&: *OutStreamer,
1928 Inst: MCInstBuilder(Is64Bit ? AArch64::MOVKXi : AArch64::MOVKWi)
1929 .addReg(Reg: Dest)
1930 .addReg(Reg: Dest)
1931 .addImm(Val: Imm)
1932 .addImm(Val: Shift));
1933}
1934
1935void AArch64AsmPrinter::emitAUT(AArch64PACKey::ID Key, Register Pointer,
1936 Register Disc) {
1937 bool IsZeroDisc = Disc == AArch64::XZR;
1938 unsigned Opcode = getAUTOpcodeForKey(K: Key, Zero: IsZeroDisc);
1939
1940 // autiza x16 ; if IsZeroDisc
1941 // autia x16, x17 ; if !IsZeroDisc
1942 MCInst AUTInst;
1943 AUTInst.setOpcode(Opcode);
1944 AUTInst.addOperand(Op: MCOperand::createReg(Reg: Pointer));
1945 AUTInst.addOperand(Op: MCOperand::createReg(Reg: Pointer));
1946 if (!IsZeroDisc)
1947 AUTInst.addOperand(Op: MCOperand::createReg(Reg: Disc));
1948
1949 EmitToStreamer(Inst: AUTInst);
1950}
1951
1952void AArch64AsmPrinter::emitPAC(AArch64PACKey::ID Key, Register Pointer,
1953 Register Disc) {
1954 bool IsZeroDisc = Disc == AArch64::XZR;
1955 unsigned Opcode = getPACOpcodeForKey(K: Key, Zero: IsZeroDisc);
1956
1957 // paciza x16 ; if IsZeroDisc
1958 // pacia x16, x17 ; if !IsZeroDisc
1959 MCInst PACInst;
1960 PACInst.setOpcode(Opcode);
1961 PACInst.addOperand(Op: MCOperand::createReg(Reg: Pointer));
1962 PACInst.addOperand(Op: MCOperand::createReg(Reg: Pointer));
1963 if (!IsZeroDisc)
1964 PACInst.addOperand(Op: MCOperand::createReg(Reg: Disc));
1965
1966 EmitToStreamer(Inst: PACInst);
1967}
1968
1969void AArch64AsmPrinter::emitBLRA(bool IsCall, AArch64PACKey::ID Key,
1970 Register Target, Register Disc) {
1971 bool IsZeroDisc = Disc == AArch64::XZR;
1972 unsigned Opcode = getBranchOpcodeForKey(IsCall, K: Key, Zero: IsZeroDisc);
1973
1974 // blraaz x16 ; if IsZeroDisc
1975 // blraa x16, x17 ; if !IsZeroDisc
1976 MCInst Inst;
1977 Inst.setOpcode(Opcode);
1978 Inst.addOperand(Op: MCOperand::createReg(Reg: Target));
1979 if (!IsZeroDisc)
1980 Inst.addOperand(Op: MCOperand::createReg(Reg: Disc));
1981 EmitToStreamer(Inst);
1982}
1983
1984void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) {
1985 Register DestReg = MI.getOperand(i: 0).getReg();
1986 if (!STI->hasZeroCycleZeroingFPWorkaround() && STI->isNeonAvailable()) {
1987 if (STI->hasZeroCycleZeroingFPR64()) {
1988 // Convert H/S register to corresponding D register
1989 const AArch64RegisterInfo *TRI = STI->getRegisterInfo();
1990 if (AArch64::FPR16RegClass.contains(Reg: DestReg))
1991 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
1992 RC: &AArch64::FPR64RegClass);
1993 else if (AArch64::FPR32RegClass.contains(Reg: DestReg))
1994 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::ssub,
1995 RC: &AArch64::FPR64RegClass);
1996 else
1997 assert(AArch64::FPR64RegClass.contains(DestReg));
1998
1999 MCInst MOVI;
2000 MOVI.setOpcode(AArch64::MOVID);
2001 MOVI.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2002 MOVI.addOperand(Op: MCOperand::createImm(Val: 0));
2003 EmitToStreamer(S&: *OutStreamer, Inst: MOVI);
2004 ++NumZCZeroingInstrsFPR;
2005 } else if (STI->hasZeroCycleZeroingFPR128()) {
2006 // Convert H/S/D register to corresponding Q register
2007 const AArch64RegisterInfo *TRI = STI->getRegisterInfo();
2008 if (AArch64::FPR16RegClass.contains(Reg: DestReg)) {
2009 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
2010 RC: &AArch64::FPR128RegClass);
2011 } else if (AArch64::FPR32RegClass.contains(Reg: DestReg)) {
2012 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::ssub,
2013 RC: &AArch64::FPR128RegClass);
2014 } else {
2015 assert(AArch64::FPR64RegClass.contains(DestReg));
2016 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::dsub,
2017 RC: &AArch64::FPR128RegClass);
2018 }
2019
2020 MCInst MOVI;
2021 MOVI.setOpcode(AArch64::MOVIv2d_ns);
2022 MOVI.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2023 MOVI.addOperand(Op: MCOperand::createImm(Val: 0));
2024 EmitToStreamer(S&: *OutStreamer, Inst: MOVI);
2025 ++NumZCZeroingInstrsFPR;
2026 } else {
2027 emitFMov0AsFMov(MI, DestReg);
2028 }
2029 } else {
2030 emitFMov0AsFMov(MI, DestReg);
2031 }
2032}
2033
2034void AArch64AsmPrinter::emitFMov0AsFMov(const MachineInstr &MI,
2035 Register DestReg) {
2036 MCInst FMov;
2037 switch (MI.getOpcode()) {
2038 default:
2039 llvm_unreachable("Unexpected opcode");
2040 case AArch64::FMOVH0:
2041 FMov.setOpcode(STI->hasFullFP16() ? AArch64::FMOVWHr : AArch64::FMOVWSr);
2042 if (!STI->hasFullFP16())
2043 DestReg = (AArch64::S0 + (DestReg - AArch64::H0));
2044 FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2045 FMov.addOperand(Op: MCOperand::createReg(Reg: AArch64::WZR));
2046 break;
2047 case AArch64::FMOVS0:
2048 FMov.setOpcode(AArch64::FMOVWSr);
2049 FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2050 FMov.addOperand(Op: MCOperand::createReg(Reg: AArch64::WZR));
2051 break;
2052 case AArch64::FMOVD0:
2053 FMov.setOpcode(AArch64::FMOVXDr);
2054 FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2055 FMov.addOperand(Op: MCOperand::createReg(Reg: AArch64::XZR));
2056 break;
2057 }
2058 EmitToStreamer(S&: *OutStreamer, Inst: FMov);
2059}
2060
2061Register AArch64AsmPrinter::emitPtrauthDiscriminator(uint64_t Disc,
2062 Register AddrDisc,
2063 Register ScratchReg,
2064 bool MayClobberAddrDisc) {
2065 assert(isPtrauthRegSafe(ScratchReg) &&
2066 "Safe scratch register must be provided by the caller");
2067 assert(isUInt<16>(Disc) && "Constant discriminator is too wide");
2068
2069 // So far we've used NoRegister in pseudos. Now we need real encodings.
2070 if (!AddrDisc.isValid())
2071 AddrDisc = AArch64::XZR;
2072
2073 // If there is no constant discriminator, there's no blend involved:
2074 // just use the address discriminator register as-is (XZR or not).
2075 if (!Disc)
2076 return AddrDisc;
2077
2078 // If there's only a constant discriminator, MOV it into the scratch register.
2079 if (AddrDisc == AArch64::XZR) {
2080 emitMOVZ(Dest: ScratchReg, Imm: Disc, Shift: 0);
2081 return ScratchReg;
2082 }
2083
2084 // If there are both, emit a blend into the scratch register.
2085
2086 // Check if we can save one MOV instruction.
2087 if (MayClobberAddrDisc && isPtrauthRegSafe(Reg: AddrDisc)) {
2088 ScratchReg = AddrDisc;
2089 } else {
2090 emitMovXReg(Dest: ScratchReg, Src: AddrDisc);
2091 assert(ScratchReg != AddrDisc &&
2092 "Forbidden to clobber AddrDisc, but have to");
2093 }
2094
2095 emitMOVK(Dest: ScratchReg, Imm: Disc, Shift: 48);
2096 return ScratchReg;
2097}
2098
2099/// Emit a code sequence to check an authenticated pointer value.
2100///
2101/// This function emits a sequence of instructions that checks if TestedReg was
2102/// authenticated successfully. On success, execution continues at the next
2103/// instruction after the sequence.
2104///
2105/// The action performed on failure depends on the OnFailure argument:
2106/// * if OnFailure is not nullptr, control is transferred to that label after
2107/// clearing the PAC field
2108/// * otherwise, BRK instruction is emitted to generate an error
2109void AArch64AsmPrinter::emitPtrauthCheckAuthenticatedValue(
2110 Register TestedReg, Register ScratchReg, AArch64PACKey::ID Key,
2111 AArch64PAuth::AuthCheckMethod Method, const MCSymbol *OnFailure) {
2112 // Insert a sequence to check if authentication of TestedReg succeeded,
2113 // such as:
2114 //
2115 // - checked and clearing:
2116 // ; x16 is TestedReg, x17 is ScratchReg
2117 // mov x17, x16
2118 // xpaci x17
2119 // cmp x16, x17
2120 // b.eq Lsuccess
2121 // mov x16, x17
2122 // b Lend
2123 // Lsuccess:
2124 // ; skipped if authentication failed
2125 // Lend:
2126 // ...
2127 //
2128 // - checked and trapping:
2129 // mov x17, x16
2130 // xpaci x17
2131 // cmp x16, x17
2132 // b.eq Lsuccess
2133 // brk #<0xc470 + aut key>
2134 // Lsuccess:
2135 // ...
2136 //
2137 // See the documentation on AuthCheckMethod enumeration constants for
2138 // the specific code sequences that can be used to perform the check.
2139 using AArch64PAuth::AuthCheckMethod;
2140
2141 if (Method == AuthCheckMethod::None)
2142 return;
2143 if (Method == AuthCheckMethod::DummyLoad) {
2144 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRWui)
2145 .addReg(Reg: getWRegFromXReg(Reg: ScratchReg))
2146 .addReg(Reg: TestedReg)
2147 .addImm(Val: 0));
2148 assert(!OnFailure && "DummyLoad always traps on error");
2149 return;
2150 }
2151
2152 MCSymbol *SuccessSym = createTempSymbol(Name: "auth_success_");
2153 if (Method == AuthCheckMethod::XPAC || Method == AuthCheckMethod::XPACHint) {
2154 // mov Xscratch, Xtested
2155 emitMovXReg(Dest: ScratchReg, Src: TestedReg);
2156
2157 if (Method == AuthCheckMethod::XPAC) {
2158 // xpac(i|d) Xscratch
2159 unsigned XPACOpc = getXPACOpcodeForKey(K: Key);
2160 EmitToStreamer(
2161 Inst: MCInstBuilder(XPACOpc).addReg(Reg: ScratchReg).addReg(Reg: ScratchReg));
2162 } else {
2163 // xpaclri
2164
2165 // Note that this method applies XPAC to TestedReg instead of ScratchReg.
2166 assert(TestedReg == AArch64::LR &&
2167 "XPACHint mode is only compatible with checking the LR register");
2168 assert((Key == AArch64PACKey::IA || Key == AArch64PACKey::IB) &&
2169 "XPACHint mode is only compatible with I-keys");
2170 EmitToStreamer(Inst: MCInstBuilder(AArch64::XPACLRI));
2171 }
2172
2173 // cmp Xtested, Xscratch
2174 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSXrs)
2175 .addReg(Reg: AArch64::XZR)
2176 .addReg(Reg: TestedReg)
2177 .addReg(Reg: ScratchReg)
2178 .addImm(Val: 0));
2179
2180 // b.eq Lsuccess
2181 EmitToStreamer(
2182 Inst: MCInstBuilder(AArch64::Bcc)
2183 .addImm(Val: AArch64CC::EQ)
2184 .addExpr(Val: MCSymbolRefExpr::create(Symbol: SuccessSym, Ctx&: OutContext)));
2185 } else if (Method == AuthCheckMethod::HighBitsNoTBI) {
2186 // eor Xscratch, Xtested, Xtested, lsl #1
2187 EmitToStreamer(Inst: MCInstBuilder(AArch64::EORXrs)
2188 .addReg(Reg: ScratchReg)
2189 .addReg(Reg: TestedReg)
2190 .addReg(Reg: TestedReg)
2191 .addImm(Val: 1));
2192 // tbz Xscratch, #62, Lsuccess
2193 EmitToStreamer(
2194 Inst: MCInstBuilder(AArch64::TBZX)
2195 .addReg(Reg: ScratchReg)
2196 .addImm(Val: 62)
2197 .addExpr(Val: MCSymbolRefExpr::create(Symbol: SuccessSym, Ctx&: OutContext)));
2198 } else {
2199 llvm_unreachable("Unsupported check method");
2200 }
2201
2202 if (!OnFailure) {
2203 // Trapping sequences do a 'brk'.
2204 // brk #<0xc470 + aut key>
2205 EmitToStreamer(Inst: MCInstBuilder(AArch64::BRK).addImm(Val: 0xc470 | Key));
2206 } else {
2207 // Non-trapping checked sequences return the stripped result in TestedReg,
2208 // skipping over success-only code (such as re-signing the pointer) by
2209 // jumping to OnFailure label.
2210 // Note that this can introduce an authentication oracle (such as based on
2211 // the high bits of the re-signed value).
2212
2213 // FIXME: The XPAC method can be optimized by applying XPAC to TestedReg
2214 // instead of ScratchReg, thus eliminating one `mov` instruction.
2215 // Both XPAC and XPACHint can be further optimized by not using a
2216 // conditional branch jumping over an unconditional one.
2217
2218 switch (Method) {
2219 case AuthCheckMethod::XPACHint:
2220 // LR is already XPAC-ed at this point.
2221 break;
2222 case AuthCheckMethod::XPAC:
2223 // mov Xtested, Xscratch
2224 emitMovXReg(Dest: TestedReg, Src: ScratchReg);
2225 break;
2226 default:
2227 // If Xtested was not XPAC-ed so far, emit XPAC here.
2228 // xpac(i|d) Xtested
2229 unsigned XPACOpc = getXPACOpcodeForKey(K: Key);
2230 EmitToStreamer(
2231 Inst: MCInstBuilder(XPACOpc).addReg(Reg: TestedReg).addReg(Reg: TestedReg));
2232 }
2233
2234 // b Lend
2235 const auto *OnFailureExpr = MCSymbolRefExpr::create(Symbol: OnFailure, Ctx&: OutContext);
2236 EmitToStreamer(Inst: MCInstBuilder(AArch64::B).addExpr(Val: OnFailureExpr));
2237 }
2238
2239 // If the auth check succeeds, we can continue.
2240 // Lsuccess:
2241 OutStreamer->emitLabel(Symbol: SuccessSym);
2242}
2243
2244// With Pointer Authentication, it may be needed to explicitly check the
2245// authenticated value in LR before performing a tail call.
2246// Otherwise, the callee may re-sign the invalid return address,
2247// introducing a signing oracle.
2248void AArch64AsmPrinter::emitPtrauthTailCallHardening(const MachineInstr *TC) {
2249 if (!AArch64FI->shouldSignReturnAddress(MF: *MF))
2250 return;
2251
2252 auto LRCheckMethod = STI->getAuthenticatedLRCheckMethod(MF: *MF);
2253 if (LRCheckMethod == AArch64PAuth::AuthCheckMethod::None)
2254 return;
2255
2256 const AArch64RegisterInfo *TRI = STI->getRegisterInfo();
2257 Register ScratchReg =
2258 TC->readsRegister(Reg: AArch64::X16, TRI) ? AArch64::X17 : AArch64::X16;
2259 assert(!TC->readsRegister(ScratchReg, TRI) &&
2260 "Neither x16 nor x17 is available as a scratch register");
2261 AArch64PACKey::ID Key =
2262 AArch64FI->shouldSignWithBKey() ? AArch64PACKey::IB : AArch64PACKey::IA;
2263 emitPtrauthCheckAuthenticatedValue(TestedReg: AArch64::LR, ScratchReg, Key,
2264 Method: LRCheckMethod);
2265}
2266
2267bool AArch64AsmPrinter::emitDeactivationSymbolRelocation(Value *DS) {
2268 if (!DS)
2269 return false;
2270
2271 if (isa<GlobalAlias>(Val: DS)) {
2272 // Just emit the nop directly.
2273 EmitToStreamer(Inst: MCInstBuilder(AArch64::NOP));
2274 return true;
2275 }
2276 MCSymbol *Dot = OutContext.createTempSymbol();
2277 OutStreamer->emitLabel(Symbol: Dot);
2278 const MCExpr *DeactDotExpr = MCSymbolRefExpr::create(Symbol: Dot, Ctx&: OutContext);
2279
2280 const MCExpr *DSExpr = MCSymbolRefExpr::create(
2281 Symbol: OutContext.getOrCreateSymbol(Name: DS->getName()), Ctx&: OutContext);
2282 OutStreamer->emitRelocDirective(Offset: *DeactDotExpr, Name: "R_AARCH64_PATCHINST", Expr: DSExpr,
2283 Loc: SMLoc());
2284 return false;
2285}
2286
2287AArch64AsmPrinter::PtrAuthSchema AArch64AsmPrinter::PtrAuthSchema::CreateImmReg(
2288 AArch64PACKey::ID Key, uint64_t IntDisc, const MachineOperand &AddrDiscOp) {
2289 PtrAuthSchema Schema;
2290 Schema.Key = Key;
2291 Schema.IntDisc = IntDisc;
2292 Schema.AddrDisc = AddrDiscOp.getReg();
2293 Schema.AddrDiscIsKilled = AddrDiscOp.isKill();
2294 Schema.PCDisc = Register();
2295 return Schema;
2296}
2297
2298AArch64AsmPrinter::PtrAuthSchema AArch64AsmPrinter::PtrAuthSchema::CreateRegReg(
2299 AArch64PACKey::ID Key, Register AddrDisc, Register PCDisc) {
2300 assert(PCDisc.isValid() && "Use CreateImmReg for non-PC schemas");
2301 PtrAuthSchema Schema;
2302 Schema.Key = Key;
2303 Schema.IntDisc = 0;
2304 Schema.AddrDisc = AddrDisc;
2305 Schema.AddrDiscIsKilled = false;
2306 Schema.PCDisc = PCDisc;
2307 return Schema;
2308}
2309
2310void AArch64AsmPrinter::emitPtrauthApplyIndirectAddend(Register Pointer,
2311 Register Scratch,
2312 int64_t Addend) {
2313 if (isInt<9>(x: Addend)) {
2314 // ldrsw Scratch, [Pointer, #Addend]! ; note: Pointer+Addend is used later.
2315 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRSWpre)
2316 .addReg(Reg: Pointer)
2317 .addReg(Reg: Scratch)
2318 .addReg(Reg: Pointer)
2319 .addImm(/*simm9:*/ Val: Addend));
2320 } else {
2321 // Pointer += Addend computation has 2 variants
2322 if (isUInt<24>(x: Addend)) {
2323 // Variant 1: add Pointer, Pointer, (Addend >> shift12) lsl shift12
2324 // This can take up to 2 instructions.
2325 for (int BitPos = 0; BitPos != 24 && (Addend >> BitPos); BitPos += 12) {
2326 EmitToStreamer(
2327 Inst: MCInstBuilder(AArch64::ADDXri)
2328 .addReg(Reg: Pointer)
2329 .addReg(Reg: Pointer)
2330 .addImm(Val: (Addend >> BitPos) & 0xfff)
2331 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: BitPos)));
2332 }
2333 } else {
2334 // Variant 2: accumulate constant in Scratch 16 bits at a time,
2335 // and add it to Pointer. This can take 2-5 instructions.
2336 emitMOVZ(Dest: Scratch, Imm: Addend & 0xffff, Shift: 0);
2337 for (int Offset = 16; Offset < 64; Offset += 16) {
2338 if (unsigned Fragment = (Addend >> Offset) & 0xffff)
2339 emitMOVK(Dest: Scratch, Imm: Fragment, Shift: Offset);
2340 }
2341
2342 // add Pointer, Pointer, Scratch
2343 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXrs)
2344 .addReg(Reg: Pointer)
2345 .addReg(Reg: Pointer)
2346 .addReg(Reg: Scratch)
2347 .addImm(Val: 0));
2348 }
2349 // ldrsw Scratch, [Pointer]
2350 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRSWui)
2351 .addReg(Reg: Scratch)
2352 .addReg(Reg: Pointer)
2353 .addImm(Val: 0));
2354 }
2355 // add Pointer, Pointer, Scratch
2356 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXrs)
2357 .addReg(Reg: Pointer)
2358 .addReg(Reg: Pointer)
2359 .addReg(Reg: Scratch)
2360 .addImm(Val: 0));
2361}
2362
2363static PtrauthCheckMode getCheckMode(const MachineFunction *MF) {
2364 const AArch64Subtarget &STI = MF->getSubtarget<AArch64Subtarget>();
2365
2366 // If an override is passed via command line argument, just use that value.
2367 if (std::optional<PtrauthCheckMode> Mode =
2368 STI.getCLOpts().ptrauth_auth_checks)
2369 return *Mode;
2370
2371 // Otherwise, on an FPAC CPU, you get traps whether you want them or not:
2372 // there's no point in emitting checks or traps.
2373 if (STI.hasFPAC())
2374 return PtrauthCheckMode::Unchecked;
2375
2376 bool ShouldTrap = MF->getFunction().hasFnAttribute(Kind: "ptrauth-auth-traps");
2377 return ShouldTrap ? PtrauthCheckMode::Trap : PtrauthCheckMode::Poison;
2378}
2379
2380// We expand non-signing AUT* pseudo instructions into a sequence of the form
2381//
2382// ; 1. Authenticate Pointer
2383//
2384// or
2385//
2386// ; 1. Authenticate Pointer
2387// ; 2. Check that Pointer is valid, trap otherwise
2388//
2389// We expand AUT*PAC pseudo instructions into a sequence of the form
2390// (with addend only applied if Addend argument is given):
2391//
2392// ; 1. Authenticate Pointer
2393// ; 3. Apply addend and sign Pointer
2394//
2395// or
2396//
2397// ; 1. Authenticate Pointer
2398// ; 2. Check that Pointer is valid, trap otherwise
2399// ; 3. Apply addend and sign Pointer
2400//
2401// or
2402//
2403// ; 1. Authenticate Pointer
2404// ; 2. Check that Pointer is valid, jump to .Lon_failure otherwise
2405// ; 3. Apply addend and sign Pointer
2406// .Lon_failure:
2407//
2408void AArch64AsmPrinter::emitPtrauthAuthResign(
2409 Register Pointer, Register Scratch, PtrAuthSchema AuthSchema,
2410 std::optional<PtrAuthSchema> SignSchema, std::optional<int64_t> Addend,
2411 Value *DS) {
2412 const PtrauthCheckMode CheckMode = getCheckMode(MF);
2413 const bool IsAuthWithPC = AuthSchema.PCDisc.isValid();
2414 assert(!SignSchema || !SignSchema->PCDisc.isValid());
2415
2416 Register SignAddrDiscOrNone = SignSchema ? SignSchema->AddrDisc : Register();
2417
2418 // 1. Authenticate Pointer - this is the only common step.
2419 // It is more complex than signing because AUTI[AB]171615 may be used.
2420
2421 if (IsAuthWithPC) {
2422 assert(Pointer == AArch64::X17 && Scratch == AArch64::X16 &&
2423 "AUTPCPAC must use x17/x16 as Pointer/Scratch");
2424
2425 assert(AuthSchema.AddrDisc == AArch64::X16 &&
2426 "AUTPCPAC requires address discriminator in X16");
2427
2428 assert(AuthSchema.PCDisc == AArch64::X15 &&
2429 "AUTPCPAC requires PC discriminator in X15");
2430
2431 assert(AuthSchema.IntDisc == 0 && "AUTPCPAC does not support IntDisc");
2432
2433 assert((AuthSchema.Key == AArch64PACKey::IB ||
2434 AuthSchema.Key == AArch64PACKey::IA) &&
2435 "AUTPCPAC only supports AUT-ing with IA/IB");
2436
2437 if (!emitDeactivationSymbolRelocation(DS)) {
2438 unsigned AutOpc = (AuthSchema.Key == AArch64PACKey::IB)
2439 ? AArch64::AUTIB171615
2440 : AArch64::AUTIA171615;
2441 EmitToStreamer(Inst: MCInstBuilder(AutOpc));
2442 }
2443 } else {
2444 // emitPtrauthDiscriminator is allowed to clobber AuthSchema.AddrDisc as
2445 // long as it is not used past this point neither externally (the register
2446 // operand is "killed"), nor internally (it does not alias anything being
2447 // used later by this pseudo instruction).
2448 //
2449 // Note that, while rather unlikely, it is technically possible to use the
2450 // Pointer to compute its own discriminator.
2451 Register AUTDiscReg = emitPtrauthDiscriminator(
2452 Disc: AuthSchema.IntDisc, AddrDisc: AuthSchema.AddrDisc, ScratchReg: Scratch,
2453 MayClobberAddrDisc: AuthSchema.addrDiscIsKilledAndNoneOf(Regs: {Pointer, SignAddrDiscOrNone}));
2454 if (!emitDeactivationSymbolRelocation(DS))
2455 emitAUT(Key: AuthSchema.Key, Pointer, Disc: AUTDiscReg);
2456 }
2457
2458 // The other two steps are optional, define lambdas for them:
2459 // 2. Check that Pointer is valid, on failure jump to label or trap.
2460 auto EmitCheck = [&](MCSymbol *OnFailure = nullptr) {
2461 emitPtrauthCheckAuthenticatedValue(TestedReg: Pointer, ScratchReg: Scratch, Key: AuthSchema.Key,
2462 Method: AArch64PAuth::AuthCheckMethod::XPAC,
2463 OnFailure);
2464 };
2465 // 3. Apply addend and sign Pointer.
2466 auto EmitResignOnSuccess = [&]() {
2467 if (Addend.has_value())
2468 emitPtrauthApplyIndirectAddend(Pointer, Scratch, Addend: *Addend);
2469
2470 assert(Pointer != SignSchema->AddrDisc && "Pointer is early-clobbered");
2471 Register PACDiscReg =
2472 emitPtrauthDiscriminator(Disc: SignSchema->IntDisc, AddrDisc: SignSchema->AddrDisc,
2473 ScratchReg: Scratch, MayClobberAddrDisc: SignSchema->AddrDiscIsKilled);
2474 emitPAC(Key: SignSchema->Key, Pointer, Disc: PACDiscReg);
2475 };
2476
2477 // Emit checking and resigning as needed.
2478
2479 if (!SignSchema) {
2480 if (CheckMode == PtrauthCheckMode::Trap)
2481 EmitCheck();
2482 // For authentication-only pseudos, Poison is demoted to Unchecked.
2483 return;
2484 }
2485
2486 switch (CheckMode) {
2487 case PtrauthCheckMode::Unchecked:
2488 EmitResignOnSuccess();
2489 break;
2490 case PtrauthCheckMode::Trap:
2491 EmitCheck();
2492 EmitResignOnSuccess();
2493 break;
2494 case PtrauthCheckMode::Poison:
2495 MCSymbol *OnFailure = createTempSymbol(Name: "resign_end_");
2496 EmitCheck(OnFailure);
2497 EmitResignOnSuccess();
2498 OutStreamer->emitLabel(Symbol: OnFailure);
2499 break;
2500 }
2501}
2502
2503void AArch64AsmPrinter::emitPtrauthSign(const MachineInstr *MI) {
2504 Register Val = MI->getOperand(i: 1).getReg();
2505 auto Key = (AArch64PACKey::ID)MI->getOperand(i: 2).getImm();
2506 uint64_t Disc = MI->getOperand(i: 3).getImm();
2507 Register AddrDisc = MI->getOperand(i: 4).getReg();
2508 bool AddrDiscKilled = MI->getOperand(i: 4).isKill();
2509
2510 // As long as at least one of Val and AddrDisc is in GPR64noip, a scratch
2511 // register is available.
2512 Register ScratchReg = Val == AArch64::X16 ? AArch64::X17 : AArch64::X16;
2513 assert(ScratchReg != AddrDisc &&
2514 "Neither X16 nor X17 is available as a scratch register");
2515
2516 // Compute pac discriminator
2517 Register DiscReg = emitPtrauthDiscriminator(
2518 Disc, AddrDisc, ScratchReg, /*MayClobberAddrDisc=*/AddrDiscKilled);
2519
2520 if (emitDeactivationSymbolRelocation(DS: MI->getDeactivationSymbol()))
2521 return;
2522
2523 emitPAC(Key, Pointer: Val, Disc: DiscReg);
2524}
2525
2526void AArch64AsmPrinter::emitPtrauthBranch(const MachineInstr *MI) {
2527 bool IsCall = MI->getOpcode() == AArch64::BLRA;
2528 unsigned BrTarget = MI->getOperand(i: 0).getReg();
2529
2530 auto Key = (AArch64PACKey::ID)MI->getOperand(i: 1).getImm();
2531 uint64_t Disc = MI->getOperand(i: 2).getImm();
2532
2533 unsigned AddrDisc = MI->getOperand(i: 3).getReg();
2534
2535 // Make sure AddrDisc is solely used to compute the discriminator.
2536 // While hardly meaningful, it is still possible to describe an authentication
2537 // of a pointer against its own value (instead of storage address) with
2538 // intrinsics, so use report_fatal_error instead of assert.
2539 if (BrTarget == AddrDisc)
2540 report_fatal_error(reason: "Branch target is signed with its own value");
2541
2542 // If we are printing BLRA pseudo, try to save one MOV by making use of the
2543 // fact that x16 and x17 are described as clobbered by the MI instruction and
2544 // AddrDisc is not used as any other input.
2545 //
2546 // Back in the day, emitPtrauthDiscriminator was restricted to only returning
2547 // either x16 or x17, meaning the returned register is always among the
2548 // implicit-def'ed registers of BLRA pseudo. Now this property can be violated
2549 // if isX16X17Safer predicate is false, thus manually check if AddrDisc is
2550 // among x16 and x17 to prevent clobbering unexpected registers.
2551 //
2552 // Unlike BLRA, BRA pseudo is used to perform computed goto, and thus not
2553 // declared as clobbering x16/x17.
2554 //
2555 // FIXME: Make use of `killed` flags and register masks instead.
2556 bool AddrDiscIsImplicitDef =
2557 IsCall && (AddrDisc == AArch64::X16 || AddrDisc == AArch64::X17);
2558 Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, ScratchReg: AArch64::X17,
2559 MayClobberAddrDisc: AddrDiscIsImplicitDef);
2560 emitBLRA(IsCall, Key, Target: BrTarget, Disc: DiscReg);
2561}
2562
2563void AArch64AsmPrinter::emitAddImm(MCRegister Reg, int64_t Addend,
2564 MCRegister Tmp) {
2565 if (Addend != 0) {
2566 const uint64_t AbsOffset = (Addend > 0 ? Addend : -((uint64_t)Addend));
2567 const bool IsNeg = Addend < 0;
2568 if (isUInt<24>(x: AbsOffset)) {
2569 for (int BitPos = 0; BitPos != 24 && (AbsOffset >> BitPos);
2570 BitPos += 12) {
2571 EmitToStreamer(
2572 Inst: MCInstBuilder(IsNeg ? AArch64::SUBXri : AArch64::ADDXri)
2573 .addReg(Reg)
2574 .addReg(Reg)
2575 .addImm(Val: (AbsOffset >> BitPos) & 0xfff)
2576 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: BitPos)));
2577 }
2578 } else {
2579 const uint64_t UAddend = Addend;
2580 EmitToStreamer(Inst: MCInstBuilder(IsNeg ? AArch64::MOVNXi : AArch64::MOVZXi)
2581 .addReg(Reg: Tmp)
2582 .addImm(Val: (IsNeg ? ~UAddend : UAddend) & 0xffff)
2583 .addImm(/*shift=*/Val: 0));
2584 auto NeedMovk = [IsNeg, UAddend](int BitPos) -> bool {
2585 assert(BitPos == 16 || BitPos == 32 || BitPos == 48);
2586 uint64_t Shifted = UAddend >> BitPos;
2587 if (!IsNeg)
2588 return Shifted != 0;
2589 for (int I = 0; I != 64 - BitPos; I += 16)
2590 if (((Shifted >> I) & 0xffff) != 0xffff)
2591 return true;
2592 return false;
2593 };
2594 for (int BitPos = 16; BitPos != 64 && NeedMovk(BitPos); BitPos += 16)
2595 emitMOVK(Dest: Tmp, Imm: (UAddend >> BitPos) & 0xffff, Shift: BitPos);
2596
2597 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXrs)
2598 .addReg(Reg)
2599 .addReg(Reg)
2600 .addReg(Reg: Tmp)
2601 .addImm(/*shift=*/Val: 0));
2602 }
2603 }
2604}
2605
2606void AArch64AsmPrinter::emitAddress(MCRegister Reg, const MCExpr *Expr,
2607 MCRegister Tmp, bool DSOLocal,
2608 const MCSubtargetInfo &STI) {
2609 MCValue Val;
2610 if (!Expr->evaluateAsRelocatable(Res&: Val, Asm: nullptr))
2611 report_fatal_error(reason: "emitAddress could not evaluate");
2612 if (DSOLocal) {
2613 EmitToStreamer(
2614 Inst: MCInstBuilder(AArch64::ADRP)
2615 .addReg(Reg)
2616 .addExpr(Val: MCSpecifierExpr::create(Expr, S: AArch64::S_ABS_PAGE,
2617 Ctx&: OutStreamer->getContext())));
2618 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
2619 .addReg(Reg)
2620 .addReg(Reg)
2621 .addExpr(Val: MCSpecifierExpr::create(
2622 Expr, S: AArch64::S_LO12, Ctx&: OutStreamer->getContext()))
2623 .addImm(Val: 0));
2624 } else {
2625 auto *SymRef =
2626 MCSymbolRefExpr::create(Symbol: Val.getAddSym(), Ctx&: OutStreamer->getContext());
2627 EmitToStreamer(
2628 Inst: MCInstBuilder(AArch64::ADRP)
2629 .addReg(Reg)
2630 .addExpr(Val: MCSpecifierExpr::create(Expr: SymRef, S: AArch64::S_GOT_PAGE,
2631 Ctx&: OutStreamer->getContext())));
2632 EmitToStreamer(
2633 Inst: MCInstBuilder(AArch64::LDRXui)
2634 .addReg(Reg)
2635 .addReg(Reg)
2636 .addExpr(Val: MCSpecifierExpr::create(Expr: SymRef, S: AArch64::S_GOT_LO12,
2637 Ctx&: OutStreamer->getContext())));
2638 emitAddImm(Reg, Addend: Val.getConstant(), Tmp);
2639 }
2640}
2641
2642static bool targetSupportsIRelativeRelocation(const Triple &TT) {
2643 // IFUNCs are ELF-only.
2644 if (!TT.isOSBinFormatELF())
2645 return false;
2646
2647 // IFUNCs are supported on glibc, bionic, and some but not all of the BSDs.
2648 return TT.isOSGlibc() || TT.isAndroid() || TT.isOSFreeBSD() ||
2649 TT.isOSDragonFly() || TT.isOSNetBSD();
2650}
2651
2652// Emit an ifunc resolver that returns a signed pointer to the specified target,
2653// and return a FUNCINIT reference to the resolver. In the linked binary, this
2654// function becomes the target of an IRELATIVE relocation. This resolver is used
2655// to relocate signed pointers in global variable initializers in special cases
2656// where the standard R_AARCH64_AUTH_ABS64 relocation would not work.
2657//
2658// Example (signed null pointer, not address discriminated):
2659//
2660// .8byte .Lpauth_ifunc0
2661// .pushsection .text.startup,"ax",@progbits
2662// .Lpauth_ifunc0:
2663// mov x0, #0
2664// mov x1, #12345
2665// b __emupac_pacda
2666//
2667// Example (signed null pointer, address discriminated):
2668//
2669// .Ltmp:
2670// .8byte .Lpauth_ifunc0
2671// .pushsection .text.startup,"ax",@progbits
2672// .Lpauth_ifunc0:
2673// mov x0, #0
2674// adrp x1, .Ltmp
2675// add x1, x1, :lo12:.Ltmp
2676// b __emupac_pacda
2677// .popsection
2678//
2679// Example (signed pointer to symbol, not address discriminated):
2680//
2681// .Ltmp:
2682// .8byte .Lpauth_ifunc0
2683// .pushsection .text.startup,"ax",@progbits
2684// .Lpauth_ifunc0:
2685// adrp x0, symbol
2686// add x0, x0, :lo12:symbol
2687// mov x1, #12345
2688// b __emupac_pacda
2689// .popsection
2690//
2691// Example (signed null pointer, not address discriminated, with deactivation
2692// symbol ds):
2693//
2694// .8byte .Lpauth_ifunc0
2695// .pushsection .text.startup,"ax",@progbits
2696// .Lpauth_ifunc0:
2697// mov x0, #0
2698// mov x1, #12345
2699// .reloc ., R_AARCH64_PATCHINST, ds
2700// b __emupac_pacda
2701// ret
2702// .popsection
2703const MCExpr *AArch64AsmPrinter::emitPAuthRelocationAsIRelative(
2704 const MCExpr *Target, uint64_t Disc, AArch64PACKey::ID KeyID,
2705 bool HasAddressDiversity, bool IsDSOLocal, const MCExpr *DSExpr) {
2706 const Triple &TT = TM.getTargetTriple();
2707
2708 // We only emit an IRELATIVE relocation if the target supports IRELATIVE.
2709 if (!targetSupportsIRelativeRelocation(TT))
2710 return nullptr;
2711
2712 // For now, only the DA key is supported.
2713 if (KeyID != AArch64PACKey::DA)
2714 return nullptr;
2715
2716 // AArch64Subtarget is huge, so heap allocate it so we don't run out of stack
2717 // space.
2718 auto STI = std::make_unique<AArch64Subtarget>(
2719 args: TT, args: TM.getTargetCPU(), args: TM.getTargetCPU(), args: TM.getTargetFeatureString(), args&: TM,
2720 args: true);
2721 this->STI = STI.get();
2722
2723 MCSymbol *Place = OutStreamer->getContext().createTempSymbol();
2724 OutStreamer->emitLabel(Symbol: Place);
2725 OutStreamer->pushSection();
2726
2727 const MCSymbolELF *Group =
2728 static_cast<MCSectionELF *>(OutStreamer->getCurrentSectionOnly())
2729 ->getGroup();
2730 auto Flags = ELF::SHF_ALLOC | ELF::SHF_EXECINSTR;
2731 if (Group)
2732 Flags |= ELF::SHF_GROUP;
2733 OutStreamer->switchSection(Section: OutStreamer->getContext().getELFSection(
2734 Section: ".text.startup", Type: ELF::SHT_PROGBITS, Flags, EntrySize: 0, Group, IsComdat: true,
2735 UniqueID: Group ? MCSection::NonUniqueID : PAuthIFuncNextUniqueID++, LinkedToSym: nullptr));
2736
2737 MCSymbol *IRelativeSym =
2738 OutStreamer->getContext().createLinkerPrivateSymbol(Name: "pauth_ifunc");
2739 OutStreamer->emitLabel(Symbol: IRelativeSym);
2740 if (isa<MCConstantExpr>(Val: Target)) {
2741 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::MOVZXi)
2742 .addReg(Reg: AArch64::X0)
2743 .addExpr(Val: Target)
2744 .addImm(Val: 0),
2745 STI: *STI);
2746 } else {
2747 emitAddress(Reg: AArch64::X0, Expr: Target, Tmp: AArch64::X16, DSOLocal: IsDSOLocal, STI: *STI);
2748 }
2749 if (HasAddressDiversity) {
2750 auto *PlacePlusDisc = MCBinaryExpr::createAdd(
2751 LHS: MCSymbolRefExpr::create(Symbol: Place, Ctx&: OutStreamer->getContext()),
2752 RHS: MCConstantExpr::create(Value: Disc, Ctx&: OutStreamer->getContext()),
2753 Ctx&: OutStreamer->getContext());
2754 emitAddress(Reg: AArch64::X1, Expr: PlacePlusDisc, Tmp: AArch64::X16, /*IsDSOLocal=*/DSOLocal: true,
2755 STI: *STI);
2756 } else {
2757 if (!isUInt<16>(x: Disc)) {
2758 OutContext.reportError(L: SMLoc(), Msg: "AArch64 PAC Discriminator '" +
2759 Twine(Disc) +
2760 "' out of range [0, 0xFFFF]");
2761 }
2762 emitMOVZ(Dest: AArch64::X1, Imm: Disc, Shift: 0);
2763 }
2764
2765 if (DSExpr) {
2766 MCSymbol *PrePACInst = OutStreamer->getContext().createTempSymbol();
2767 OutStreamer->emitLabel(Symbol: PrePACInst);
2768
2769 auto *PrePACInstExpr =
2770 MCSymbolRefExpr::create(Symbol: PrePACInst, Ctx&: OutStreamer->getContext());
2771 OutStreamer->emitRelocDirective(Offset: *PrePACInstExpr, Name: "R_AARCH64_PATCHINST",
2772 Expr: DSExpr, Loc: SMLoc());
2773 }
2774
2775 // We don't know the subtarget because this is being emitted for a global
2776 // initializer. Because the performance of IFUNC resolvers is unimportant, we
2777 // always call the EmuPAC runtime, which will end up using the PAC instruction
2778 // if the target supports PAC.
2779 MCSymbol *EmuPAC =
2780 OutStreamer->getContext().getOrCreateSymbol(Name: "__emupac_pacda");
2781 const MCSymbolRefExpr *EmuPACRef =
2782 MCSymbolRefExpr::create(Symbol: EmuPAC, Ctx&: OutStreamer->getContext());
2783 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::B).addExpr(Val: EmuPACRef),
2784 STI: *STI);
2785
2786 // We need a RET despite the above tail call because the deactivation symbol
2787 // may replace the tail call with a NOP.
2788 if (DSExpr)
2789 OutStreamer->emitInstruction(
2790 Inst: MCInstBuilder(AArch64::RET).addReg(Reg: AArch64::LR), STI: *STI);
2791 OutStreamer->popSection();
2792
2793 return MCSpecifierExpr::create(
2794 Expr: MCSymbolRefExpr::create(Symbol: IRelativeSym, Ctx&: OutStreamer->getContext()),
2795 S: AArch64::S_FUNCINIT, Ctx&: OutStreamer->getContext());
2796}
2797
2798const MCExpr *
2799AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) {
2800 MCContext &Ctx = OutContext;
2801
2802 // Figure out the base symbol and the addend, if any.
2803 APInt Offset(64, 0);
2804 const Value *BaseGV = CPA.getPointer()->stripAndAccumulateConstantOffsets(
2805 DL: getDataLayout(), Offset, /*AllowNonInbounds=*/true);
2806
2807 auto *BaseGVB = dyn_cast<GlobalValue>(Val: BaseGV);
2808
2809 const MCExpr *Sym;
2810 if (BaseGVB) {
2811 // If there is an addend, turn that into the appropriate MCExpr.
2812 Sym = MCSymbolRefExpr::create(Symbol: getSymbol(GV: BaseGVB), Ctx);
2813 if (Offset.sgt(RHS: 0))
2814 Sym = MCBinaryExpr::createAdd(
2815 LHS: Sym, RHS: MCConstantExpr::create(Value: Offset.getSExtValue(), Ctx), Ctx);
2816 else if (Offset.slt(RHS: 0))
2817 Sym = MCBinaryExpr::createSub(
2818 LHS: Sym, RHS: MCConstantExpr::create(Value: (-Offset).getSExtValue(), Ctx), Ctx);
2819 } else if (isa<ConstantPointerNull>(Val: BaseGV)) {
2820 Sym = MCConstantExpr::create(Value: Offset.getSExtValue(), Ctx);
2821 } else {
2822 reportFatalUsageError(reason: "unsupported constant expression in ptrauth pointer");
2823 }
2824
2825 const MCExpr *DSExpr = nullptr;
2826 if (auto *DS = dyn_cast<GlobalValue>(Val: CPA.getDeactivationSymbol())) {
2827 if (isa<GlobalAlias>(Val: DS))
2828 return Sym;
2829 DSExpr = MCSymbolRefExpr::create(Symbol: getSymbol(GV: DS), Ctx);
2830 }
2831
2832 uint64_t KeyID = CPA.getKey()->getZExtValue();
2833 // We later rely on valid KeyID value in AArch64PACKeyIDToString call from
2834 // AArch64AuthMCExpr::printImpl, so fail fast.
2835 if (KeyID > AArch64PACKey::LAST) {
2836 CPA.getContext().emitError(ErrorStr: "AArch64 PAC Key ID '" + Twine(KeyID) +
2837 "' out of range [0, " +
2838 Twine((unsigned)AArch64PACKey::LAST) + "]");
2839 KeyID = 0;
2840 }
2841
2842 uint64_t Disc = CPA.getDiscriminator()->getZExtValue();
2843
2844 // Check if we can represent this with an IRELATIVE and emit it if so.
2845 if (auto *IFuncSym = emitPAuthRelocationAsIRelative(
2846 Target: Sym, Disc, KeyID: AArch64PACKey::ID(KeyID), HasAddressDiversity: CPA.hasAddressDiscriminator(),
2847 IsDSOLocal: BaseGVB && BaseGVB->isDSOLocal(), DSExpr))
2848 return IFuncSym;
2849
2850 if (!isUInt<16>(x: Disc)) {
2851 CPA.getContext().emitError(ErrorStr: "AArch64 PAC Discriminator '" + Twine(Disc) +
2852 "' out of range [0, 0xFFFF]");
2853 Disc = 0;
2854 }
2855
2856 if (DSExpr)
2857 report_fatal_error(reason: "deactivation symbols unsupported in constant "
2858 "expressions on this target");
2859
2860 // Finally build the complete @AUTH expr.
2861 return AArch64AuthMCExpr::create(Expr: Sym, Discriminator: Disc, Key: AArch64PACKey::ID(KeyID),
2862 HasAddressDiversity: CPA.hasAddressDiscriminator(), Ctx);
2863}
2864
2865void AArch64AsmPrinter::LowerLOADauthptrstatic(const MachineInstr &MI) {
2866 unsigned DstReg = MI.getOperand(i: 0).getReg();
2867 const MachineOperand &GAOp = MI.getOperand(i: 1);
2868 const uint64_t KeyC = MI.getOperand(i: 2).getImm();
2869 assert(KeyC <= AArch64PACKey::LAST &&
2870 "key is out of range [0, AArch64PACKey::LAST]");
2871 const auto Key = (AArch64PACKey::ID)KeyC;
2872 const uint64_t Disc = MI.getOperand(i: 3).getImm();
2873 assert(isUInt<16>(Disc) &&
2874 "constant discriminator is out of range [0, 0xffff]");
2875
2876 // Emit instruction sequence like the following:
2877 // ADRP x16, symbol$auth_ptr$key$disc
2878 // LDR x16, [x16, :lo12:symbol$auth_ptr$key$disc]
2879 //
2880 // Where the $auth_ptr$ symbol is the stub slot containing the signed pointer
2881 // to symbol.
2882 MCSymbol *AuthPtrStubSym;
2883 if (TM.getTargetTriple().isOSBinFormatELF()) {
2884 const auto &TLOF =
2885 static_cast<const AArch64_ELFTargetObjectFile &>(getObjFileLowering());
2886
2887 assert(GAOp.getOffset() == 0 &&
2888 "non-zero offset for $auth_ptr$ stub slots is not supported");
2889 const MCSymbol *GASym = TM.getSymbol(GV: GAOp.getGlobal());
2890 AuthPtrStubSym = TLOF.getAuthPtrSlotSymbol(TM, MMI, RawSym: GASym, Key, Discriminator: Disc);
2891 } else {
2892 assert(TM.getTargetTriple().isOSBinFormatMachO() &&
2893 "LOADauthptrstatic is implemented only for MachO/ELF");
2894
2895 const auto &TLOF = static_cast<const AArch64_MachoTargetObjectFile &>(
2896 getObjFileLowering());
2897
2898 assert(GAOp.getOffset() == 0 &&
2899 "non-zero offset for $auth_ptr$ stub slots is not supported");
2900 const MCSymbol *GASym = TM.getSymbol(GV: GAOp.getGlobal());
2901 AuthPtrStubSym = TLOF.getAuthPtrSlotSymbol(TM, MMI, RawSym: GASym, Key, Discriminator: Disc);
2902 }
2903
2904 MachineOperand StubMOHi =
2905 MachineOperand::CreateMCSymbol(Sym: AuthPtrStubSym, TargetFlags: AArch64II::MO_PAGE);
2906 MachineOperand StubMOLo = MachineOperand::CreateMCSymbol(
2907 Sym: AuthPtrStubSym, TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2908 MCOperand StubMCHi, StubMCLo;
2909
2910 MCInstLowering.lowerOperand(MO: StubMOHi, MCOp&: StubMCHi);
2911 MCInstLowering.lowerOperand(MO: StubMOLo, MCOp&: StubMCLo);
2912
2913 EmitToStreamer(
2914 S&: *OutStreamer,
2915 Inst: MCInstBuilder(AArch64::ADRP).addReg(Reg: DstReg).addOperand(Op: StubMCHi));
2916
2917 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::LDRXui)
2918 .addReg(Reg: DstReg)
2919 .addReg(Reg: DstReg)
2920 .addOperand(Op: StubMCLo));
2921}
2922
2923void AArch64AsmPrinter::LowerMOVaddrPAC(const MachineInstr &MI) {
2924 const bool IsGOTLoad = MI.getOpcode() == AArch64::LOADgotPAC;
2925 const bool IsELFSignedGOT = MI.getParent()
2926 ->getParent()
2927 ->getInfo<AArch64FunctionInfo>()
2928 ->hasELFSignedGOT();
2929 MachineOperand GAOp = MI.getOperand(i: 0);
2930 const uint64_t KeyC = MI.getOperand(i: 1).getImm();
2931 assert(KeyC <= AArch64PACKey::LAST &&
2932 "key is out of range [0, AArch64PACKey::LAST]");
2933 const auto Key = (AArch64PACKey::ID)KeyC;
2934 const unsigned AddrDisc = MI.getOperand(i: 2).getReg();
2935 const uint64_t Disc = MI.getOperand(i: 3).getImm();
2936
2937 const int64_t Offset = GAOp.getOffset();
2938 GAOp.setOffset(0);
2939
2940 // Emit:
2941 // target materialization:
2942 // - via GOT:
2943 // - unsigned GOT:
2944 // adrp x16, :got:target
2945 // ldr x16, [x16, :got_lo12:target]
2946 // add offset to x16 if offset != 0
2947 // - ELF signed GOT:
2948 // adrp x17, :got:target
2949 // add x17, x17, :got_auth_lo12:target
2950 // ldr x16, [x17]
2951 // aut{i|d}a x16, x17
2952 // check+trap sequence (if no FPAC)
2953 // add offset to x16 if offset != 0
2954 //
2955 // - direct:
2956 // adrp x16, target
2957 // add x16, x16, :lo12:target
2958 // add offset to x16 if offset != 0
2959 //
2960 // add offset to x16:
2961 // - abs(offset) fits 24 bits:
2962 // add/sub x16, x16, #<offset>[, #lsl 12] (up to 2 instructions)
2963 // - abs(offset) does not fit 24 bits:
2964 // - offset < 0:
2965 // movn+movk sequence filling x17 register with the offset (up to 4
2966 // instructions)
2967 // add x16, x16, x17
2968 // - offset > 0:
2969 // movz+movk sequence filling x17 register with the offset (up to 4
2970 // instructions)
2971 // add x16, x16, x17
2972 //
2973 // signing:
2974 // - 0 discriminator:
2975 // paciza x16
2976 // - Non-0 discriminator, no address discriminator:
2977 // mov x17, #Disc
2978 // pacia x16, x17
2979 // - address discriminator (with potentially folded immediate discriminator):
2980 // pacia x16, xAddrDisc
2981
2982 MachineOperand GAMOHi(GAOp), GAMOLo(GAOp);
2983 MCOperand GAMCHi, GAMCLo;
2984
2985 GAMOHi.setTargetFlags(AArch64II::MO_PAGE);
2986 GAMOLo.setTargetFlags(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2987 if (IsGOTLoad) {
2988 GAMOHi.addTargetFlag(F: AArch64II::MO_GOT);
2989 GAMOLo.addTargetFlag(F: AArch64II::MO_GOT);
2990 }
2991
2992 MCInstLowering.lowerOperand(MO: GAMOHi, MCOp&: GAMCHi);
2993 MCInstLowering.lowerOperand(MO: GAMOLo, MCOp&: GAMCLo);
2994
2995 EmitToStreamer(
2996 Inst: MCInstBuilder(AArch64::ADRP)
2997 .addReg(Reg: IsGOTLoad && IsELFSignedGOT ? AArch64::X17 : AArch64::X16)
2998 .addOperand(Op: GAMCHi));
2999
3000 if (IsGOTLoad) {
3001 if (IsELFSignedGOT) {
3002 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
3003 .addReg(Reg: AArch64::X17)
3004 .addReg(Reg: AArch64::X17)
3005 .addOperand(Op: GAMCLo)
3006 .addImm(Val: 0));
3007
3008 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
3009 .addReg(Reg: AArch64::X16)
3010 .addReg(Reg: AArch64::X17)
3011 .addImm(Val: 0));
3012
3013 assert(GAOp.isGlobal());
3014 assert(GAOp.getGlobal()->getValueType() != nullptr);
3015
3016 bool IsFunctionTy = GAOp.getGlobal()->getValueType()->isFunctionTy();
3017 auto AuthKey = IsFunctionTy ? AArch64PACKey::IA : AArch64PACKey::DA;
3018 emitAUT(Key: AuthKey, Pointer: AArch64::X16, Disc: AArch64::X17);
3019
3020 if (!STI->hasFPAC())
3021 emitPtrauthCheckAuthenticatedValue(TestedReg: AArch64::X16, ScratchReg: AArch64::X17, Key: AuthKey,
3022 Method: AArch64PAuth::AuthCheckMethod::XPAC);
3023 } else {
3024 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
3025 .addReg(Reg: AArch64::X16)
3026 .addReg(Reg: AArch64::X16)
3027 .addOperand(Op: GAMCLo));
3028 }
3029 } else {
3030 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
3031 .addReg(Reg: AArch64::X16)
3032 .addReg(Reg: AArch64::X16)
3033 .addOperand(Op: GAMCLo)
3034 .addImm(Val: 0));
3035 }
3036
3037 emitAddImm(Reg: AArch64::X16, Addend: Offset, Tmp: AArch64::X17);
3038 Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, ScratchReg: AArch64::X17);
3039
3040 emitPAC(Key, Pointer: AArch64::X16, Disc: DiscReg);
3041}
3042
3043void AArch64AsmPrinter::LowerLOADgotAUTH(const MachineInstr &MI) {
3044 Register DstReg = MI.getOperand(i: 0).getReg();
3045 Register AuthResultReg = STI->hasFPAC() ? DstReg : AArch64::X16;
3046 const MachineOperand &GAMO = MI.getOperand(i: 1);
3047 assert(GAMO.getOffset() == 0);
3048
3049 if (MI.getMF()->getTarget().getCodeModel() == CodeModel::Tiny) {
3050 MCOperand GAMC;
3051 MCInstLowering.lowerOperand(MO: GAMO, MCOp&: GAMC);
3052 EmitToStreamer(
3053 Inst: MCInstBuilder(AArch64::ADR).addReg(Reg: AArch64::X17).addOperand(Op: GAMC));
3054 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
3055 .addReg(Reg: AuthResultReg)
3056 .addReg(Reg: AArch64::X17)
3057 .addImm(Val: 0));
3058 } else {
3059 MachineOperand GAHiOp(GAMO);
3060 MachineOperand GALoOp(GAMO);
3061 GAHiOp.addTargetFlag(F: AArch64II::MO_PAGE);
3062 GALoOp.addTargetFlag(F: AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
3063
3064 MCOperand GAMCHi, GAMCLo;
3065 MCInstLowering.lowerOperand(MO: GAHiOp, MCOp&: GAMCHi);
3066 MCInstLowering.lowerOperand(MO: GALoOp, MCOp&: GAMCLo);
3067
3068 EmitToStreamer(
3069 Inst: MCInstBuilder(AArch64::ADRP).addReg(Reg: AArch64::X17).addOperand(Op: GAMCHi));
3070
3071 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
3072 .addReg(Reg: AArch64::X17)
3073 .addReg(Reg: AArch64::X17)
3074 .addOperand(Op: GAMCLo)
3075 .addImm(Val: 0));
3076
3077 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
3078 .addReg(Reg: AuthResultReg)
3079 .addReg(Reg: AArch64::X17)
3080 .addImm(Val: 0));
3081 }
3082
3083 assert(GAMO.isGlobal());
3084 MCSymbol *UndefWeakSym;
3085 if (GAMO.getGlobal()->hasExternalWeakLinkage()) {
3086 UndefWeakSym = createTempSymbol(Name: "undef_weak");
3087 EmitToStreamer(
3088 Inst: MCInstBuilder(AArch64::CBZX)
3089 .addReg(Reg: AuthResultReg)
3090 .addExpr(Val: MCSymbolRefExpr::create(Symbol: UndefWeakSym, Ctx&: OutContext)));
3091 }
3092
3093 assert(GAMO.getGlobal()->getValueType() != nullptr);
3094
3095 bool IsFunctionTy = GAMO.getGlobal()->getValueType()->isFunctionTy();
3096 auto AuthKey = IsFunctionTy ? AArch64PACKey::IA : AArch64PACKey::DA;
3097 emitAUT(Key: AuthKey, Pointer: AuthResultReg, Disc: AArch64::X17);
3098
3099 if (GAMO.getGlobal()->hasExternalWeakLinkage())
3100 OutStreamer->emitLabel(Symbol: UndefWeakSym);
3101
3102 if (!STI->hasFPAC()) {
3103 emitPtrauthCheckAuthenticatedValue(TestedReg: AuthResultReg, ScratchReg: AArch64::X17, Key: AuthKey,
3104 Method: AArch64PAuth::AuthCheckMethod::XPAC);
3105
3106 emitMovXReg(Dest: DstReg, Src: AuthResultReg);
3107 }
3108}
3109
3110const MCExpr *
3111AArch64AsmPrinter::lowerBlockAddressConstant(const BlockAddress &BA) {
3112 const MCExpr *BAE = AsmPrinter::lowerBlockAddressConstant(BA);
3113 const Function &Fn = *BA.getFunction();
3114
3115 if (std::optional<uint16_t> BADisc =
3116 STI->getPtrAuthBlockAddressDiscriminatorIfEnabled(ParentFn: Fn))
3117 return AArch64AuthMCExpr::create(Expr: BAE, Discriminator: *BADisc, Key: AArch64PACKey::IA,
3118 /*HasAddressDiversity=*/false, Ctx&: OutContext);
3119
3120 return BAE;
3121}
3122
3123void AArch64AsmPrinter::emitCBPseudoExpansion(const MachineInstr *MI) {
3124 bool IsImm = false;
3125 unsigned Width = 0;
3126
3127 switch (MI->getOpcode()) {
3128 default:
3129 llvm_unreachable("This is not a CB pseudo instruction");
3130 case AArch64::CBBAssertExt:
3131 IsImm = false;
3132 Width = 8;
3133 break;
3134 case AArch64::CBHAssertExt:
3135 IsImm = false;
3136 Width = 16;
3137 break;
3138 case AArch64::CBWPrr:
3139 Width = 32;
3140 break;
3141 case AArch64::CBXPrr:
3142 Width = 64;
3143 break;
3144 case AArch64::CBWPri:
3145 IsImm = true;
3146 Width = 32;
3147 break;
3148 case AArch64::CBXPri:
3149 IsImm = true;
3150 Width = 64;
3151 break;
3152 }
3153
3154 AArch64CC::CondCode CC =
3155 static_cast<AArch64CC::CondCode>(MI->getOperand(i: 0).getImm());
3156 bool NeedsRegSwap = false;
3157 bool NeedsImmDec = false;
3158 bool NeedsImmInc = false;
3159
3160#define GET_CB_OPC(IsImm, Width, ImmCond, RegCond) \
3161 (IsImm \
3162 ? (Width == 32 ? AArch64::CB##ImmCond##Wri : AArch64::CB##ImmCond##Xri) \
3163 : (Width == 8 \
3164 ? AArch64::CBB##RegCond##Wrr \
3165 : (Width == 16 ? AArch64::CBH##RegCond##Wrr \
3166 : (Width == 32 ? AArch64::CB##RegCond##Wrr \
3167 : AArch64::CB##RegCond##Xrr))))
3168 unsigned MCOpC;
3169
3170 // Decide if we need to either swap register operands or increment/decrement
3171 // immediate operands
3172 switch (CC) {
3173 default:
3174 llvm_unreachable("Invalid CB condition code");
3175 case AArch64CC::EQ:
3176 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ EQ, /* Reg-Reg */ EQ);
3177 break;
3178 case AArch64CC::NE:
3179 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ NE, /* Reg-Reg */ NE);
3180 break;
3181 case AArch64CC::HS:
3182 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ HI, /* Reg-Reg */ HS);
3183 NeedsImmDec = IsImm;
3184 break;
3185 case AArch64CC::LO:
3186 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ LO, /* Reg-Reg */ HI);
3187 NeedsRegSwap = !IsImm;
3188 break;
3189 case AArch64CC::HI:
3190 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ HI, /* Reg-Reg */ HI);
3191 break;
3192 case AArch64CC::LS:
3193 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ LO, /* Reg-Reg */ HS);
3194 NeedsRegSwap = !IsImm;
3195 NeedsImmInc = IsImm;
3196 break;
3197 case AArch64CC::GE:
3198 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ GT, /* Reg-Reg */ GE);
3199 NeedsImmDec = IsImm;
3200 break;
3201 case AArch64CC::LT:
3202 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ LT, /* Reg-Reg */ GT);
3203 NeedsRegSwap = !IsImm;
3204 break;
3205 case AArch64CC::GT:
3206 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ GT, /* Reg-Reg */ GT);
3207 break;
3208 case AArch64CC::LE:
3209 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ LT, /* Reg-Reg */ GE);
3210 NeedsRegSwap = !IsImm;
3211 NeedsImmInc = IsImm;
3212 break;
3213 }
3214#undef GET_CB_OPC
3215
3216 MCInst Inst;
3217 Inst.setOpcode(MCOpC);
3218
3219 MCOperand Lhs, Rhs, Trgt;
3220 lowerOperand(MO: MI->getOperand(i: 1), MCOp&: Lhs);
3221 lowerOperand(MO: MI->getOperand(i: 2), MCOp&: Rhs);
3222 lowerOperand(MO: MI->getOperand(i: 3), MCOp&: Trgt);
3223
3224 // Now swap, increment or decrement
3225 if (NeedsRegSwap) {
3226 assert(Lhs.isReg() && "Expected register operand for CB");
3227 assert(Rhs.isReg() && "Expected register operand for CB");
3228 Inst.addOperand(Op: Rhs);
3229 Inst.addOperand(Op: Lhs);
3230 } else if (NeedsImmDec) {
3231 Rhs.setImm(Rhs.getImm() - 1);
3232 Inst.addOperand(Op: Lhs);
3233 Inst.addOperand(Op: Rhs);
3234 } else if (NeedsImmInc) {
3235 Rhs.setImm(Rhs.getImm() + 1);
3236 Inst.addOperand(Op: Lhs);
3237 Inst.addOperand(Op: Rhs);
3238 } else {
3239 Inst.addOperand(Op: Lhs);
3240 Inst.addOperand(Op: Rhs);
3241 }
3242
3243 assert((!IsImm || (Rhs.getImm() >= 0 && Rhs.getImm() < 64)) &&
3244 "CB immediate operand out-of-bounds");
3245
3246 Inst.addOperand(Op: Trgt);
3247 EmitToStreamer(S&: *OutStreamer, Inst);
3248}
3249
3250void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI) {
3251
3252 unsigned StOpc;
3253 unsigned Relaxed = MI->getOperand(i: 2).getImm();
3254 assert(Relaxed < 2 && "Atomic hint relaxed immediate out-of-bounds");
3255 switch (MI->getOpcode()) {
3256 case AArch64::ATOMIC_STORE_HINT_B:
3257 StOpc = Relaxed ? AArch64::STRBBui : AArch64::STLRB;
3258 break;
3259 case AArch64::ATOMIC_STORE_HINT_H:
3260 StOpc = Relaxed ? AArch64::STRHHui : AArch64::STLRH;
3261 break;
3262 case AArch64::ATOMIC_STORE_HINT_W:
3263 StOpc = Relaxed ? AArch64::STRWui : AArch64::STLRW;
3264 break;
3265 case AArch64::ATOMIC_STORE_HINT_X:
3266 StOpc = Relaxed ? AArch64::STRXui : AArch64::STLRX;
3267 break;
3268 default:
3269 llvm_unreachable("Unexpected atomic hint size.");
3270 }
3271
3272 EmitToStreamer(
3273 Inst: MCInstBuilder(AArch64::HINT).addImm(Val: MI->getOperand(i: 3).getImm()));
3274
3275 MCInst Store;
3276 Store.setOpcode(StOpc);
3277 Store.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg()));
3278 Store.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 1).getReg()));
3279 Store.setFlags(MI->getFlags());
3280 if (Relaxed)
3281 Store.addOperand(Op: MCOperand::createImm(Val: 0));
3282 EmitToStreamer(S&: *OutStreamer, Inst: Store);
3283}
3284
3285void AArch64AsmPrinter::emitAtomicHintPseudoExpansionRO(
3286 const MachineInstr *MI) {
3287 unsigned StOpc;
3288 assert(MI->getOperand(5).getImm() == 1 &&
3289 "Atomic store addressing mode only supports relaxed stores");
3290
3291 switch (MI->getOpcode()) {
3292 case AArch64::ATOMIC_STORE_HINT_BroW:
3293 StOpc = AArch64::STRBBroW;
3294 break;
3295 case AArch64::ATOMIC_STORE_HINT_HroW:
3296 StOpc = AArch64::STRHHroW;
3297 break;
3298 case AArch64::ATOMIC_STORE_HINT_WroW:
3299 StOpc = AArch64::STRWroW;
3300 break;
3301 case AArch64::ATOMIC_STORE_HINT_XroW:
3302 StOpc = AArch64::STRXroW;
3303 break;
3304 case AArch64::ATOMIC_STORE_HINT_SroW:
3305 StOpc = AArch64::STRSroW;
3306 break;
3307 case AArch64::ATOMIC_STORE_HINT_DroW:
3308 StOpc = AArch64::STRDroW;
3309 break;
3310 case AArch64::ATOMIC_STORE_HINT_BroX:
3311 StOpc = AArch64::STRBBroX;
3312 break;
3313 case AArch64::ATOMIC_STORE_HINT_HroX:
3314 StOpc = AArch64::STRHHroX;
3315 break;
3316 case AArch64::ATOMIC_STORE_HINT_WroX:
3317 StOpc = AArch64::STRWroX;
3318 break;
3319 case AArch64::ATOMIC_STORE_HINT_XroX:
3320 StOpc = AArch64::STRXroX;
3321 break;
3322 case AArch64::ATOMIC_STORE_HINT_SroX:
3323 StOpc = AArch64::STRSroX;
3324 break;
3325 case AArch64::ATOMIC_STORE_HINT_DroX:
3326 StOpc = AArch64::STRDroX;
3327 break;
3328 default:
3329 llvm_unreachable("Unexpected atomic hint opcode.");
3330 }
3331
3332 EmitToStreamer(
3333 Inst: MCInstBuilder(AArch64::HINT).addImm(Val: MI->getOperand(i: 6).getImm()));
3334
3335 MCInst Store;
3336 Store.setOpcode(StOpc);
3337 Store.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg())); // Rn
3338 Store.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 1).getReg())); // Rm
3339 Store.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 2).getReg())); // Data
3340 Store.addOperand(Op: MCOperand::createImm(Val: MI->getOperand(i: 3).getImm())); // Signed
3341 Store.addOperand(Op: MCOperand::createImm(Val: MI->getOperand(i: 4).getImm())); // Shift
3342 Store.setFlags(MI->getFlags());
3343 EmitToStreamer(S&: *OutStreamer, Inst: Store);
3344}
3345
3346void AArch64AsmPrinter::emitAtomicHintPseudoExpansionImm(
3347 const MachineInstr *MI) {
3348 unsigned StOpc;
3349 assert(MI->getOperand(3).getImm() == 1 &&
3350 "Atomic store addressing mode only supports relaxed stores");
3351
3352 switch (MI->getOpcode()) {
3353 case AArch64::ATOMIC_STORE_HINT_Bui:
3354 StOpc = AArch64::STRBBui;
3355 break;
3356 case AArch64::ATOMIC_STORE_HINT_Hui:
3357 StOpc = AArch64::STRHHui;
3358 break;
3359 case AArch64::ATOMIC_STORE_HINT_Wui:
3360 StOpc = AArch64::STRWui;
3361 break;
3362 case AArch64::ATOMIC_STORE_HINT_Xui:
3363 StOpc = AArch64::STRXui;
3364 break;
3365 case AArch64::ATOMIC_STORE_HINT_Sui:
3366 StOpc = AArch64::STRSui;
3367 break;
3368 case AArch64::ATOMIC_STORE_HINT_Dui:
3369 StOpc = AArch64::STRDui;
3370 break;
3371 case AArch64::ATOMIC_STORE_HINT_Bi:
3372 StOpc = AArch64::STURBBi;
3373 break;
3374 case AArch64::ATOMIC_STORE_HINT_Hi:
3375 StOpc = AArch64::STURHHi;
3376 break;
3377 case AArch64::ATOMIC_STORE_HINT_Wi:
3378 StOpc = AArch64::STURWi;
3379 break;
3380 case AArch64::ATOMIC_STORE_HINT_Xi:
3381 StOpc = AArch64::STURXi;
3382 break;
3383 case AArch64::ATOMIC_STORE_HINT_Si:
3384 StOpc = AArch64::STURSi;
3385 break;
3386 case AArch64::ATOMIC_STORE_HINT_Di:
3387 StOpc = AArch64::STURDi;
3388 break;
3389 default:
3390 llvm_unreachable("Unexpected atomic hint opcode.");
3391 }
3392
3393 EmitToStreamer(
3394 Inst: MCInstBuilder(AArch64::HINT).addImm(Val: MI->getOperand(i: 4).getImm()));
3395
3396 MCInst Store;
3397 Store.setOpcode(StOpc);
3398 Store.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg())); // Rn
3399 Store.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 1).getReg())); // Data
3400 Store.addOperand(Op: MCOperand::createImm(Val: MI->getOperand(i: 2).getImm())); // Imm
3401 Store.setFlags(MI->getFlags());
3402 EmitToStreamer(S&: *OutStreamer, Inst: Store);
3403}
3404
3405// Simple pseudo-instructions have their lowering (with expansion to real
3406// instructions) auto-generated.
3407#include "AArch64GenMCPseudoLowering.inc"
3408
3409void AArch64AsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) {
3410 S.emitInstruction(Inst, STI: *STI);
3411#ifndef NDEBUG
3412 ++InstsEmitted;
3413#endif
3414}
3415
3416void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
3417 AArch64_MC::verifyInstructionPredicates(Opcode: MI->getOpcode(), Features: STI->getFeatureBits());
3418
3419#ifndef NDEBUG
3420 InstsEmitted = 0;
3421 llvm::scope_exit CheckMISize([&]() {
3422 assert(STI->getInstrInfo()->getInstSizeInBytes(*MI) >= InstsEmitted * 4);
3423 });
3424#endif
3425
3426 // Do any auto-generated pseudo lowerings.
3427 if (MCInst OutInst; lowerPseudoInstExpansion(MI, Inst&: OutInst)) {
3428 EmitToStreamer(S&: *OutStreamer, Inst: OutInst);
3429 return;
3430 }
3431
3432 if (MI->getOpcode() == AArch64::ADRP) {
3433 for (auto &Opd : MI->operands()) {
3434 if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) ==
3435 "swift_async_extendedFramePointerFlags") {
3436 ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true;
3437 }
3438 }
3439 }
3440
3441 if (AArch64FI->getLOHRelated().count(Ptr: MI)) {
3442 // Generate a label for LOH related instruction
3443 MCSymbol *LOHLabel = createTempSymbol(Name: "loh");
3444 // Associate the instruction with the label
3445 LOHInstToLabel[MI] = LOHLabel;
3446 OutStreamer->emitLabel(Symbol: LOHLabel);
3447 }
3448
3449 AArch64TargetStreamer *TS =
3450 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
3451 // Do any manual lowerings.
3452 switch (MI->getOpcode()) {
3453 default:
3454 assert(!AArch64InstrInfo::isTailCallReturnInst(*MI) &&
3455 "Unhandled tail call instruction");
3456 break;
3457 case AArch64::READ_REGISTER_GPR64:
3458 // Read of a named GPR: emit "mov Xt, Xn" (ORR Xt, XZR, Xn). The source
3459 // register is encoded as an immediate operand so that earlier passes do not
3460 // see a use of an undefined physical register.
3461 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ORRXrs)
3462 .addReg(Reg: MI->getOperand(i: 0).getReg())
3463 .addReg(Reg: AArch64::XZR)
3464 .addReg(Reg: MI->getOperand(i: 1).getImm())
3465 .addImm(Val: 0));
3466 return;
3467 case AArch64::READ_REGISTER_FPR64:
3468 // Read of a named FP/SIMD d-register: emit "fmov Dt, Dn".
3469 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::FMOVDr)
3470 .addReg(Reg: MI->getOperand(i: 0).getReg())
3471 .addReg(Reg: MI->getOperand(i: 1).getImm()));
3472 return;
3473 case AArch64::HINT: {
3474 // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
3475 // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
3476 // non-empty. If MI is the initial BTI, place the
3477 // __patchable_function_entries label after BTI.
3478 if (CurrentPatchableFunctionEntrySym &&
3479 CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
3480 MI == &MF->front().front()) {
3481 int64_t Imm = MI->getOperand(i: 0).getImm();
3482 if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38) {
3483 MCInst Inst;
3484 MCInstLowering.Lower(MI, OutMI&: Inst);
3485 EmitToStreamer(S&: *OutStreamer, Inst);
3486 CurrentPatchableFunctionEntrySym = createTempSymbol(Name: "patch");
3487 OutStreamer->emitLabel(Symbol: CurrentPatchableFunctionEntrySym);
3488 return;
3489 }
3490 }
3491 break;
3492 }
3493 case AArch64::MOVMCSym: {
3494 Register DestReg = MI->getOperand(i: 0).getReg();
3495 const MachineOperand &MO_Sym = MI->getOperand(i: 1);
3496 MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
3497 MCOperand Hi_MCSym, Lo_MCSym;
3498
3499 Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
3500 Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
3501
3502 MCInstLowering.lowerOperand(MO: Hi_MOSym, MCOp&: Hi_MCSym);
3503 MCInstLowering.lowerOperand(MO: Lo_MOSym, MCOp&: Lo_MCSym);
3504
3505 MCInst MovZ;
3506 MovZ.setOpcode(AArch64::MOVZXi);
3507 MovZ.addOperand(Op: MCOperand::createReg(Reg: DestReg));
3508 MovZ.addOperand(Op: Hi_MCSym);
3509 MovZ.addOperand(Op: MCOperand::createImm(Val: 16));
3510 EmitToStreamer(S&: *OutStreamer, Inst: MovZ);
3511
3512 MCInst MovK;
3513 MovK.setOpcode(AArch64::MOVKXi);
3514 MovK.addOperand(Op: MCOperand::createReg(Reg: DestReg));
3515 MovK.addOperand(Op: MCOperand::createReg(Reg: DestReg));
3516 MovK.addOperand(Op: Lo_MCSym);
3517 MovK.addOperand(Op: MCOperand::createImm(Val: 0));
3518 EmitToStreamer(S&: *OutStreamer, Inst: MovK);
3519 return;
3520 }
3521 case AArch64::MOVIv2d_ns:
3522 // It is generally beneficial to rewrite "fmov s0, wzr" to "movi d0, #0".
3523 // as movi is more efficient across all cores. Newer cores can eliminate
3524 // fmovs early and there is no difference with movi, but this not true for
3525 // all implementations.
3526 //
3527 // The floating-point version doesn't quite work in rare cases on older
3528 // CPUs, so on those targets we lower this instruction to movi.16b instead.
3529 if (STI->hasZeroCycleZeroingFPWorkaround() &&
3530 MI->getOperand(i: 1).getImm() == 0) {
3531 MCInst TmpInst;
3532 TmpInst.setOpcode(AArch64::MOVIv16b_ns);
3533 TmpInst.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg()));
3534 TmpInst.addOperand(Op: MCOperand::createImm(Val: 0));
3535 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
3536 return;
3537 }
3538 break;
3539
3540 case AArch64::DBG_VALUE:
3541 case AArch64::DBG_VALUE_LIST:
3542 if (isVerbose() && OutStreamer->hasRawTextSupport()) {
3543 SmallString<128> TmpStr;
3544 raw_svector_ostream OS(TmpStr);
3545 PrintDebugValueComment(MI, OS);
3546 OutStreamer->emitRawText(String: StringRef(OS.str()));
3547 }
3548 return;
3549
3550 case AArch64::EMITBKEY: {
3551 ExceptionHandling ExceptionHandlingType = MAI.getExceptionHandlingType();
3552 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
3553 ExceptionHandlingType != ExceptionHandling::ARM)
3554 return;
3555
3556 if (getFunctionCFISectionType(MF: *MF) == CFISection::None)
3557 return;
3558
3559 OutStreamer->emitCFIBKeyFrame();
3560 return;
3561 }
3562
3563 case AArch64::EMITMTETAGGED: {
3564 ExceptionHandling ExceptionHandlingType = MAI.getExceptionHandlingType();
3565 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
3566 ExceptionHandlingType != ExceptionHandling::ARM)
3567 return;
3568
3569 if (getFunctionCFISectionType(MF: *MF) != CFISection::None)
3570 OutStreamer->emitCFIMTETaggedFrame();
3571 return;
3572 }
3573
3574 case AArch64::AUTx16x17: {
3575 const Register Pointer = AArch64::X16;
3576 const Register Scratch = AArch64::X17;
3577
3578 auto AuthSchema = PtrAuthSchema::CreateImmReg(
3579 Key: (AArch64PACKey::ID)MI->getOperand(i: 0).getImm(),
3580 IntDisc: MI->getOperand(i: 1).getImm(), AddrDiscOp: MI->getOperand(i: 2));
3581
3582 emitPtrauthAuthResign(Pointer, Scratch, AuthSchema, SignSchema: std::nullopt,
3583 Addend: std::nullopt, DS: MI->getDeactivationSymbol());
3584 return;
3585 }
3586
3587 case AArch64::AUTxMxN: {
3588 const Register Pointer = MI->getOperand(i: 0).getReg();
3589 const Register Scratch = MI->getOperand(i: 1).getReg();
3590
3591 auto AuthSchema = PtrAuthSchema::CreateImmReg(
3592 Key: (AArch64PACKey::ID)MI->getOperand(i: 3).getImm(),
3593 IntDisc: MI->getOperand(i: 4).getImm(), AddrDiscOp: MI->getOperand(i: 5));
3594
3595 emitPtrauthAuthResign(Pointer, Scratch, AuthSchema, SignSchema: std::nullopt,
3596 Addend: std::nullopt, DS: MI->getDeactivationSymbol());
3597 return;
3598 }
3599
3600 case AArch64::AUTPAC: {
3601 const Register Pointer = AArch64::X16;
3602 const Register Scratch = AArch64::X17;
3603
3604 auto AuthSchema = PtrAuthSchema::CreateImmReg(
3605 Key: (AArch64PACKey::ID)MI->getOperand(i: 0).getImm(),
3606 IntDisc: MI->getOperand(i: 1).getImm(), AddrDiscOp: MI->getOperand(i: 2));
3607
3608 auto SignSchema = PtrAuthSchema::CreateImmReg(
3609 Key: (AArch64PACKey::ID)MI->getOperand(i: 3).getImm(),
3610 IntDisc: MI->getOperand(i: 4).getImm(), AddrDiscOp: MI->getOperand(i: 5));
3611
3612 emitPtrauthAuthResign(Pointer, Scratch, AuthSchema, SignSchema,
3613 Addend: std::nullopt, DS: MI->getDeactivationSymbol());
3614 return;
3615 }
3616
3617 case AArch64::AUTPCPAC: {
3618 auto AuthSchema = PtrAuthSchema::CreateRegReg(
3619 Key: (AArch64PACKey::ID)MI->getOperand(i: 0).getImm(), AddrDisc: AArch64::X16,
3620 PCDisc: AArch64::X15);
3621
3622 auto SignSchema = PtrAuthSchema::CreateImmReg(
3623 Key: (AArch64PACKey::ID)MI->getOperand(i: 1).getImm(),
3624 IntDisc: MI->getOperand(i: 2).getImm(), AddrDiscOp: MI->getOperand(i: 3));
3625
3626 emitPtrauthAuthResign(/*Pointer=*/AArch64::X17, /*Scratch=*/AArch64::X16,
3627 AuthSchema, SignSchema, Addend: std::nullopt,
3628 DS: MI->getDeactivationSymbol());
3629 return;
3630 }
3631
3632 case AArch64::AUTRELLOADPAC: {
3633 const Register Pointer = AArch64::X16;
3634 const Register Scratch = AArch64::X17;
3635
3636 auto AuthSchema = PtrAuthSchema::CreateImmReg(
3637 Key: (AArch64PACKey::ID)MI->getOperand(i: 0).getImm(),
3638 IntDisc: MI->getOperand(i: 1).getImm(), AddrDiscOp: MI->getOperand(i: 2));
3639
3640 auto SignSchema = PtrAuthSchema::CreateImmReg(
3641 Key: (AArch64PACKey::ID)MI->getOperand(i: 3).getImm(),
3642 IntDisc: MI->getOperand(i: 4).getImm(), AddrDiscOp: MI->getOperand(i: 5));
3643
3644 emitPtrauthAuthResign(Pointer, Scratch, AuthSchema, SignSchema,
3645 Addend: MI->getOperand(i: 6).getImm(),
3646 DS: MI->getDeactivationSymbol());
3647
3648 return;
3649 }
3650
3651 case AArch64::PAC:
3652 emitPtrauthSign(MI);
3653 return;
3654
3655 case AArch64::LOADauthptrstatic:
3656 LowerLOADauthptrstatic(MI: *MI);
3657 return;
3658
3659 case AArch64::LOADgotPAC:
3660 case AArch64::MOVaddrPAC:
3661 LowerMOVaddrPAC(MI: *MI);
3662 return;
3663
3664 case AArch64::LOADgotAUTH:
3665 LowerLOADgotAUTH(MI: *MI);
3666 return;
3667
3668 case AArch64::BRA:
3669 case AArch64::BLRA:
3670 emitPtrauthBranch(MI);
3671 return;
3672
3673 // Tail calls use pseudo instructions so they have the proper code-gen
3674 // attributes (isCall, isReturn, etc.). We lower them to the real
3675 // instruction here.
3676 case AArch64::AUTH_TCRETURN:
3677 case AArch64::AUTH_TCRETURN_BTI: {
3678 Register Callee = MI->getOperand(i: 0).getReg();
3679 const auto Key = (AArch64PACKey::ID)MI->getOperand(i: 2).getImm();
3680 const uint64_t Disc = MI->getOperand(i: 3).getImm();
3681
3682 Register AddrDisc = MI->getOperand(i: 4).getReg();
3683
3684 Register ScratchReg = Callee == AArch64::X16 ? AArch64::X17 : AArch64::X16;
3685
3686 emitPtrauthTailCallHardening(TC: MI);
3687
3688 // See the comments in emitPtrauthBranch.
3689 if (Callee == AddrDisc)
3690 report_fatal_error(reason: "Call target is signed with its own value");
3691
3692 // After isX16X17Safer predicate was introduced, emitPtrauthDiscriminator is
3693 // no longer restricted to only reusing AddrDisc when it is X16 or X17
3694 // (which are implicit-def'ed by AUTH_TCRETURN pseudos), thus impose this
3695 // restriction manually not to clobber an unexpected register.
3696 bool AddrDiscIsImplicitDef =
3697 AddrDisc == AArch64::X16 || AddrDisc == AArch64::X17;
3698 Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, ScratchReg,
3699 MayClobberAddrDisc: AddrDiscIsImplicitDef);
3700 emitBLRA(/*IsCall*/ false, Key, Target: Callee, Disc: DiscReg);
3701 return;
3702 }
3703
3704 case AArch64::TCRETURNri:
3705 case AArch64::TCRETURNrix16x17:
3706 case AArch64::TCRETURNrix17:
3707 case AArch64::TCRETURNrinotx16:
3708 case AArch64::TCRETURNriALL: {
3709 emitPtrauthTailCallHardening(TC: MI);
3710
3711 recordIfImportCall(BranchInst: MI);
3712 MCInst TmpInst;
3713 TmpInst.setOpcode(AArch64::BR);
3714 TmpInst.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg()));
3715 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
3716 return;
3717 }
3718 case AArch64::TCRETURNdi: {
3719 emitPtrauthTailCallHardening(TC: MI);
3720
3721 MCOperand Dest;
3722 MCInstLowering.lowerOperand(MO: MI->getOperand(i: 0), MCOp&: Dest);
3723 recordIfImportCall(BranchInst: MI);
3724 MCInst TmpInst;
3725 TmpInst.setOpcode(AArch64::B);
3726 TmpInst.addOperand(Op: Dest);
3727 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
3728 return;
3729 }
3730 case AArch64::SpeculationBarrierISBDSBEndBB: {
3731 // Print DSB SYS + ISB
3732 MCInst TmpInstDSB;
3733 TmpInstDSB.setOpcode(AArch64::DSB);
3734 TmpInstDSB.addOperand(Op: MCOperand::createImm(Val: 0xf));
3735 EmitToStreamer(S&: *OutStreamer, Inst: TmpInstDSB);
3736 MCInst TmpInstISB;
3737 TmpInstISB.setOpcode(AArch64::ISB);
3738 TmpInstISB.addOperand(Op: MCOperand::createImm(Val: 0xf));
3739 EmitToStreamer(S&: *OutStreamer, Inst: TmpInstISB);
3740 return;
3741 }
3742 case AArch64::SpeculationBarrierSBEndBB: {
3743 // Print SB
3744 MCInst TmpInstSB;
3745 TmpInstSB.setOpcode(AArch64::SB);
3746 EmitToStreamer(S&: *OutStreamer, Inst: TmpInstSB);
3747 return;
3748 }
3749 case AArch64::TLSDESC_AUTH_CALLSEQ: {
3750 /// lower this to:
3751 /// adrp x0, :tlsdesc_auth:var
3752 /// ldr x16, [x0, #:tlsdesc_auth_lo12:var]
3753 /// add x0, x0, #:tlsdesc_auth_lo12:var
3754 /// .tlsauthdesccall var
3755 /// blraa x16, x0
3756 /// (TPIDR_EL0 offset now in x0)
3757 const MachineOperand &MO_Sym = MI->getOperand(i: 0);
3758 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
3759 MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
3760 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
3761 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
3762 MCInstLowering.lowerOperand(MO: MO_Sym, MCOp&: Sym);
3763 MCInstLowering.lowerOperand(MO: MO_TLSDESC_LO12, MCOp&: SymTLSDescLo12);
3764 MCInstLowering.lowerOperand(MO: MO_TLSDESC, MCOp&: SymTLSDesc);
3765
3766 MCInst Adrp;
3767 Adrp.setOpcode(AArch64::ADRP);
3768 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3769 Adrp.addOperand(Op: SymTLSDesc);
3770 EmitToStreamer(S&: *OutStreamer, Inst: Adrp);
3771
3772 MCInst Ldr;
3773 Ldr.setOpcode(AArch64::LDRXui);
3774 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
3775 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3776 Ldr.addOperand(Op: SymTLSDescLo12);
3777 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
3778 EmitToStreamer(S&: *OutStreamer, Inst: Ldr);
3779
3780 MCInst Add;
3781 Add.setOpcode(AArch64::ADDXri);
3782 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3783 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3784 Add.addOperand(Op: SymTLSDescLo12);
3785 Add.addOperand(Op: MCOperand::createImm(Val: AArch64_AM::getShiftValue(Imm: 0)));
3786 EmitToStreamer(S&: *OutStreamer, Inst: Add);
3787
3788 // Emit a relocation-annotation. This expands to no code, but requests
3789 // the following instruction gets an R_AARCH64_AUTH_TLSDESC_CALL.
3790 MCInst TLSAuthDescCall;
3791 TLSAuthDescCall.setOpcode(AArch64::TLSAUTHDESCCALL);
3792 TLSAuthDescCall.addOperand(Op: Sym);
3793 EmitToStreamer(S&: *OutStreamer, Inst: TLSAuthDescCall);
3794#ifndef NDEBUG
3795 --InstsEmitted; // no code emitted
3796#endif
3797
3798 MCInst Blraa;
3799 Blraa.setOpcode(AArch64::BLRAA);
3800 Blraa.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
3801 Blraa.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3802 EmitToStreamer(S&: *OutStreamer, Inst: Blraa);
3803
3804 return;
3805 }
3806 case AArch64::TLSDESC_CALLSEQ: {
3807 /// lower this to:
3808 /// adrp x0, :tlsdesc:var
3809 /// ldr x1, [x0, #:tlsdesc_lo12:var]
3810 /// add x0, x0, #:tlsdesc_lo12:var
3811 /// .tlsdesccall var
3812 /// blr x1
3813 /// (TPIDR_EL0 offset now in x0)
3814 const MachineOperand &MO_Sym = MI->getOperand(i: 0);
3815 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
3816 MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
3817 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
3818 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
3819 MCInstLowering.lowerOperand(MO: MO_Sym, MCOp&: Sym);
3820 MCInstLowering.lowerOperand(MO: MO_TLSDESC_LO12, MCOp&: SymTLSDescLo12);
3821 MCInstLowering.lowerOperand(MO: MO_TLSDESC, MCOp&: SymTLSDesc);
3822
3823 MCInst Adrp;
3824 Adrp.setOpcode(AArch64::ADRP);
3825 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3826 Adrp.addOperand(Op: SymTLSDesc);
3827 EmitToStreamer(S&: *OutStreamer, Inst: Adrp);
3828
3829 MCInst Ldr;
3830 if (STI->isTargetILP32()) {
3831 Ldr.setOpcode(AArch64::LDRWui);
3832 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::W1));
3833 } else {
3834 Ldr.setOpcode(AArch64::LDRXui);
3835 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X1));
3836 }
3837 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3838 Ldr.addOperand(Op: SymTLSDescLo12);
3839 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
3840 EmitToStreamer(S&: *OutStreamer, Inst: Ldr);
3841
3842 MCInst Add;
3843 if (STI->isTargetILP32()) {
3844 Add.setOpcode(AArch64::ADDWri);
3845 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::W0));
3846 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::W0));
3847 } else {
3848 Add.setOpcode(AArch64::ADDXri);
3849 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3850 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3851 }
3852 Add.addOperand(Op: SymTLSDescLo12);
3853 Add.addOperand(Op: MCOperand::createImm(Val: AArch64_AM::getShiftValue(Imm: 0)));
3854 EmitToStreamer(S&: *OutStreamer, Inst: Add);
3855
3856 // Emit a relocation-annotation. This expands to no code, but requests
3857 // the following instruction gets an R_AARCH64_TLSDESC_CALL.
3858 MCInst TLSDescCall;
3859 TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
3860 TLSDescCall.addOperand(Op: Sym);
3861 EmitToStreamer(S&: *OutStreamer, Inst: TLSDescCall);
3862#ifndef NDEBUG
3863 --InstsEmitted; // no code emitted
3864#endif
3865
3866 MCInst Blr;
3867 Blr.setOpcode(AArch64::BLR);
3868 Blr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X1));
3869 EmitToStreamer(S&: *OutStreamer, Inst: Blr);
3870
3871 return;
3872 }
3873
3874 case AArch64::JumpTableDest32:
3875 case AArch64::JumpTableDest16:
3876 case AArch64::JumpTableDest8:
3877 LowerJumpTableDest(OutStreamer&: *OutStreamer, MI: *MI);
3878 return;
3879
3880 case AArch64::BR_JumpTable:
3881 LowerHardenedBRJumpTable(MI: *MI);
3882 return;
3883
3884 case AArch64::FMOVH0:
3885 case AArch64::FMOVS0:
3886 case AArch64::FMOVD0:
3887 emitFMov0(MI: *MI);
3888 return;
3889
3890 case AArch64::MOPSMemoryCopyPseudo:
3891 case AArch64::MOPSMemoryMovePseudo:
3892 case AArch64::MOPSMemorySetPseudo:
3893 case AArch64::MOPSMemorySetTaggingPseudo:
3894 LowerMOPS(OutStreamer&: *OutStreamer, MI: *MI);
3895 return;
3896
3897 case TargetOpcode::STACKMAP:
3898 return LowerSTACKMAP(OutStreamer&: *OutStreamer, SM, MI: *MI);
3899
3900 case TargetOpcode::PATCHPOINT:
3901 return LowerPATCHPOINT(OutStreamer&: *OutStreamer, SM, MI: *MI);
3902
3903 case TargetOpcode::STATEPOINT:
3904 return LowerSTATEPOINT(OutStreamer&: *OutStreamer, SM, MI: *MI);
3905
3906 case TargetOpcode::FAULTING_OP:
3907 return LowerFAULTING_OP(FaultingMI: *MI);
3908
3909 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
3910 LowerPATCHABLE_FUNCTION_ENTER(MI: *MI);
3911 return;
3912
3913 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
3914 LowerPATCHABLE_FUNCTION_EXIT(MI: *MI);
3915 return;
3916
3917 case TargetOpcode::PATCHABLE_TAIL_CALL:
3918 LowerPATCHABLE_TAIL_CALL(MI: *MI);
3919 return;
3920 case TargetOpcode::PATCHABLE_EVENT_CALL:
3921 return LowerPATCHABLE_EVENT_CALL(MI: *MI, Typed: false);
3922 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
3923 return LowerPATCHABLE_EVENT_CALL(MI: *MI, Typed: true);
3924
3925 case AArch64::KCFI_CHECK:
3926 LowerKCFI_CHECK(MI: *MI);
3927 return;
3928
3929 case AArch64::HWASAN_CHECK_MEMACCESS:
3930 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
3931 case AArch64::HWASAN_CHECK_MEMACCESS_FIXEDSHADOW:
3932 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW:
3933 LowerHWASAN_CHECK_MEMACCESS(MI: *MI);
3934 return;
3935
3936 case AArch64::SEH_StackAlloc:
3937 TS->emitARM64WinCFIAllocStack(Size: MI->getOperand(i: 0).getImm());
3938 return;
3939
3940 case AArch64::SEH_SaveFPLR:
3941 TS->emitARM64WinCFISaveFPLR(Offset: MI->getOperand(i: 0).getImm());
3942 return;
3943
3944 case AArch64::SEH_SaveFPLR_X:
3945 assert(MI->getOperand(0).getImm() < 0 &&
3946 "Pre increment SEH opcode must have a negative offset");
3947 TS->emitARM64WinCFISaveFPLRX(Offset: -MI->getOperand(i: 0).getImm());
3948 return;
3949
3950 case AArch64::SEH_SaveReg:
3951 TS->emitARM64WinCFISaveReg(Reg: MI->getOperand(i: 0).getImm(),
3952 Offset: MI->getOperand(i: 1).getImm());
3953 return;
3954
3955 case AArch64::SEH_SaveReg_X:
3956 assert(MI->getOperand(1).getImm() < 0 &&
3957 "Pre increment SEH opcode must have a negative offset");
3958 TS->emitARM64WinCFISaveRegX(Reg: MI->getOperand(i: 0).getImm(),
3959 Offset: -MI->getOperand(i: 1).getImm());
3960 return;
3961
3962 case AArch64::SEH_SaveRegP:
3963 if (MI->getOperand(i: 1).getImm() == 30 && MI->getOperand(i: 0).getImm() >= 19 &&
3964 MI->getOperand(i: 0).getImm() <= 28) {
3965 assert((MI->getOperand(0).getImm() - 19) % 2 == 0 &&
3966 "Register paired with LR must be odd");
3967 TS->emitARM64WinCFISaveLRPair(Reg: MI->getOperand(i: 0).getImm(),
3968 Offset: MI->getOperand(i: 2).getImm());
3969 return;
3970 }
3971 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
3972 "Non-consecutive registers not allowed for save_regp");
3973 TS->emitARM64WinCFISaveRegP(Reg: MI->getOperand(i: 0).getImm(),
3974 Offset: MI->getOperand(i: 2).getImm());
3975 return;
3976
3977 case AArch64::SEH_SaveRegP_X:
3978 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
3979 "Non-consecutive registers not allowed for save_regp_x");
3980 assert(MI->getOperand(2).getImm() < 0 &&
3981 "Pre increment SEH opcode must have a negative offset");
3982 TS->emitARM64WinCFISaveRegPX(Reg: MI->getOperand(i: 0).getImm(),
3983 Offset: -MI->getOperand(i: 2).getImm());
3984 return;
3985
3986 case AArch64::SEH_SaveFReg:
3987 TS->emitARM64WinCFISaveFReg(Reg: MI->getOperand(i: 0).getImm(),
3988 Offset: MI->getOperand(i: 1).getImm());
3989 return;
3990
3991 case AArch64::SEH_SaveFReg_X:
3992 assert(MI->getOperand(1).getImm() < 0 &&
3993 "Pre increment SEH opcode must have a negative offset");
3994 TS->emitARM64WinCFISaveFRegX(Reg: MI->getOperand(i: 0).getImm(),
3995 Offset: -MI->getOperand(i: 1).getImm());
3996 return;
3997
3998 case AArch64::SEH_SaveFRegP:
3999 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
4000 "Non-consecutive registers not allowed for save_regp");
4001 TS->emitARM64WinCFISaveFRegP(Reg: MI->getOperand(i: 0).getImm(),
4002 Offset: MI->getOperand(i: 2).getImm());
4003 return;
4004
4005 case AArch64::SEH_SaveFRegP_X:
4006 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
4007 "Non-consecutive registers not allowed for save_regp_x");
4008 assert(MI->getOperand(2).getImm() < 0 &&
4009 "Pre increment SEH opcode must have a negative offset");
4010 TS->emitARM64WinCFISaveFRegPX(Reg: MI->getOperand(i: 0).getImm(),
4011 Offset: -MI->getOperand(i: 2).getImm());
4012 return;
4013
4014 case AArch64::SEH_SetFP:
4015 TS->emitARM64WinCFISetFP();
4016 return;
4017
4018 case AArch64::SEH_AddFP:
4019 TS->emitARM64WinCFIAddFP(Size: MI->getOperand(i: 0).getImm());
4020 return;
4021
4022 case AArch64::SEH_Nop:
4023 TS->emitARM64WinCFINop();
4024 return;
4025
4026 case AArch64::SEH_PrologEnd:
4027 TS->emitARM64WinCFIPrologEnd();
4028 return;
4029
4030 case AArch64::SEH_EpilogStart:
4031 TS->emitARM64WinCFIEpilogStart();
4032 return;
4033
4034 case AArch64::SEH_EpilogEnd:
4035 TS->emitARM64WinCFIEpilogEnd();
4036 return;
4037
4038 case AArch64::SEH_PACSignLR:
4039 TS->emitARM64WinCFIPACSignLR();
4040 return;
4041
4042 case AArch64::SEH_SaveAnyRegI:
4043 assert(MI->getOperand(1).getImm() <= 1008 &&
4044 "SaveAnyRegQP SEH opcode offset must fit into 6 bits");
4045 TS->emitARM64WinCFISaveAnyRegI(Reg: MI->getOperand(i: 0).getImm(),
4046 Offset: MI->getOperand(i: 1).getImm());
4047 return;
4048
4049 case AArch64::SEH_SaveAnyRegIP:
4050 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
4051 "Non-consecutive registers not allowed for save_any_reg");
4052 assert(MI->getOperand(2).getImm() <= 1008 &&
4053 "SaveAnyRegQP SEH opcode offset must fit into 6 bits");
4054 TS->emitARM64WinCFISaveAnyRegIP(Reg: MI->getOperand(i: 0).getImm(),
4055 Offset: MI->getOperand(i: 2).getImm());
4056 return;
4057
4058 case AArch64::SEH_SaveAnyRegQP:
4059 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
4060 "Non-consecutive registers not allowed for save_any_reg");
4061 assert(MI->getOperand(2).getImm() >= 0 &&
4062 "SaveAnyRegQP SEH opcode offset must be non-negative");
4063 assert(MI->getOperand(2).getImm() <= 1008 &&
4064 "SaveAnyRegQP SEH opcode offset must fit into 6 bits");
4065 TS->emitARM64WinCFISaveAnyRegQP(Reg: MI->getOperand(i: 0).getImm(),
4066 Offset: MI->getOperand(i: 2).getImm());
4067 return;
4068
4069 case AArch64::SEH_SaveAnyRegQPX:
4070 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
4071 "Non-consecutive registers not allowed for save_any_reg");
4072 assert(MI->getOperand(2).getImm() < 0 &&
4073 "SaveAnyRegQPX SEH opcode offset must be negative");
4074 assert(MI->getOperand(2).getImm() >= -1008 &&
4075 "SaveAnyRegQPX SEH opcode offset must fit into 6 bits");
4076 TS->emitARM64WinCFISaveAnyRegQPX(Reg: MI->getOperand(i: 0).getImm(),
4077 Offset: -MI->getOperand(i: 2).getImm());
4078 return;
4079
4080 case AArch64::SEH_AllocZ:
4081 assert(MI->getOperand(0).getImm() >= 0 &&
4082 "AllocZ SEH opcode offset must be non-negative");
4083 assert(MI->getOperand(0).getImm() <= 255 &&
4084 "AllocZ SEH opcode offset must fit into 8 bits");
4085 TS->emitARM64WinCFIAllocZ(Offset: MI->getOperand(i: 0).getImm());
4086 return;
4087
4088 case AArch64::SEH_SaveZReg:
4089 assert(MI->getOperand(1).getImm() >= 0 &&
4090 "SaveZReg SEH opcode offset must be non-negative");
4091 assert(MI->getOperand(1).getImm() <= 255 &&
4092 "SaveZReg SEH opcode offset must fit into 8 bits");
4093 TS->emitARM64WinCFISaveZReg(Reg: MI->getOperand(i: 0).getImm(),
4094 Offset: MI->getOperand(i: 1).getImm());
4095 return;
4096
4097 case AArch64::SEH_SavePReg:
4098 assert(MI->getOperand(1).getImm() >= 0 &&
4099 "SavePReg SEH opcode offset must be non-negative");
4100 assert(MI->getOperand(1).getImm() <= 255 &&
4101 "SavePReg SEH opcode offset must fit into 8 bits");
4102 TS->emitARM64WinCFISavePReg(Reg: MI->getOperand(i: 0).getImm(),
4103 Offset: MI->getOperand(i: 1).getImm());
4104 return;
4105
4106 case AArch64::BLR:
4107 case AArch64::BR: {
4108 recordIfImportCall(BranchInst: MI);
4109 MCInst TmpInst;
4110 MCInstLowering.Lower(MI, OutMI&: TmpInst);
4111 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
4112 return;
4113 }
4114 case AArch64::CBWPri:
4115 case AArch64::CBXPri:
4116 case AArch64::CBBAssertExt:
4117 case AArch64::CBHAssertExt:
4118 case AArch64::CBWPrr:
4119 case AArch64::CBXPrr:
4120 emitCBPseudoExpansion(MI);
4121 return;
4122 case AArch64::ATOMIC_STORE_HINT_B:
4123 case AArch64::ATOMIC_STORE_HINT_H:
4124 case AArch64::ATOMIC_STORE_HINT_W:
4125 case AArch64::ATOMIC_STORE_HINT_X:
4126 emitAtomicHintPseudoExpansion(MI);
4127 return;
4128 case AArch64::ATOMIC_STORE_HINT_BroW:
4129 case AArch64::ATOMIC_STORE_HINT_HroW:
4130 case AArch64::ATOMIC_STORE_HINT_WroW:
4131 case AArch64::ATOMIC_STORE_HINT_XroW:
4132 case AArch64::ATOMIC_STORE_HINT_SroW:
4133 case AArch64::ATOMIC_STORE_HINT_DroW:
4134 case AArch64::ATOMIC_STORE_HINT_BroX:
4135 case AArch64::ATOMIC_STORE_HINT_HroX:
4136 case AArch64::ATOMIC_STORE_HINT_WroX:
4137 case AArch64::ATOMIC_STORE_HINT_XroX:
4138 case AArch64::ATOMIC_STORE_HINT_SroX:
4139 case AArch64::ATOMIC_STORE_HINT_DroX:
4140 emitAtomicHintPseudoExpansionRO(MI);
4141 return;
4142 case AArch64::ATOMIC_STORE_HINT_Bui:
4143 case AArch64::ATOMIC_STORE_HINT_Hui:
4144 case AArch64::ATOMIC_STORE_HINT_Wui:
4145 case AArch64::ATOMIC_STORE_HINT_Xui:
4146 case AArch64::ATOMIC_STORE_HINT_Sui:
4147 case AArch64::ATOMIC_STORE_HINT_Dui:
4148 case AArch64::ATOMIC_STORE_HINT_Bi:
4149 case AArch64::ATOMIC_STORE_HINT_Hi:
4150 case AArch64::ATOMIC_STORE_HINT_Wi:
4151 case AArch64::ATOMIC_STORE_HINT_Xi:
4152 case AArch64::ATOMIC_STORE_HINT_Si:
4153 case AArch64::ATOMIC_STORE_HINT_Di:
4154 emitAtomicHintPseudoExpansionImm(MI);
4155 return;
4156 }
4157
4158 if (emitDeactivationSymbolRelocation(DS: MI->getDeactivationSymbol()))
4159 return;
4160
4161 // Finally, do the automated lowerings for everything else.
4162 MCInst TmpInst;
4163 MCInstLowering.Lower(MI, OutMI&: TmpInst);
4164 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
4165}
4166
4167void AArch64AsmPrinter::recordIfImportCall(
4168 const llvm::MachineInstr *BranchInst) {
4169 if (!EnableImportCallOptimization)
4170 return;
4171
4172 auto [GV, OpFlags] = BranchInst->getMF()->tryGetCalledGlobal(MI: BranchInst);
4173 if (GV && GV->hasDLLImportStorageClass()) {
4174 auto *CallSiteSymbol = MMI->getContext().createNamedTempSymbol(Name: "impcall");
4175 OutStreamer->emitLabel(Symbol: CallSiteSymbol);
4176
4177 auto *CalledSymbol = MCInstLowering.GetGlobalValueSymbol(GV, TargetFlags: OpFlags);
4178 SectionToImportedFunctionCalls[OutStreamer->getCurrentSectionOnly()]
4179 .push_back(x: {CallSiteSymbol, CalledSymbol});
4180 }
4181}
4182
4183void AArch64AsmPrinter::emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI,
4184 MCSymbol *LazyPointer) {
4185 // _ifunc:
4186 // adrp x16, lazy_pointer@GOTPAGE
4187 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF]
4188 // ldr x16, [x16]
4189 // br x16
4190
4191 {
4192 MCInst Adrp;
4193 Adrp.setOpcode(AArch64::ADRP);
4194 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
4195 MCOperand SymPage;
4196 MCInstLowering.lowerOperand(
4197 MO: MachineOperand::CreateMCSymbol(Sym: LazyPointer,
4198 TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGE),
4199 MCOp&: SymPage);
4200 Adrp.addOperand(Op: SymPage);
4201 EmitToStreamer(Inst: Adrp);
4202 }
4203
4204 {
4205 MCInst Ldr;
4206 Ldr.setOpcode(AArch64::LDRXui);
4207 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
4208 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
4209 MCOperand SymPageOff;
4210 MCInstLowering.lowerOperand(
4211 MO: MachineOperand::CreateMCSymbol(Sym: LazyPointer, TargetFlags: AArch64II::MO_GOT |
4212 AArch64II::MO_PAGEOFF),
4213 MCOp&: SymPageOff);
4214 Ldr.addOperand(Op: SymPageOff);
4215 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
4216 EmitToStreamer(Inst: Ldr);
4217 }
4218
4219 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
4220 .addReg(Reg: AArch64::X16)
4221 .addReg(Reg: AArch64::X16)
4222 .addImm(Val: 0));
4223
4224 EmitToStreamer(Inst: MCInstBuilder(TM.getTargetTriple().isArm64e() ? AArch64::BRAAZ
4225 : AArch64::BR)
4226 .addReg(Reg: AArch64::X16));
4227}
4228
4229void AArch64AsmPrinter::emitMachOIFuncStubHelperBody(Module &M,
4230 const GlobalIFunc &GI,
4231 MCSymbol *LazyPointer) {
4232 // These stub helpers are only ever called once, so here we're optimizing for
4233 // minimum size by using the pre-indexed store variants, which saves a few
4234 // bytes of instructions to bump & restore sp.
4235
4236 // _ifunc.stub_helper:
4237 // stp fp, lr, [sp, #-16]!
4238 // mov fp, sp
4239 // stp x1, x0, [sp, #-16]!
4240 // stp x3, x2, [sp, #-16]!
4241 // stp x5, x4, [sp, #-16]!
4242 // stp x7, x6, [sp, #-16]!
4243 // stp d1, d0, [sp, #-16]!
4244 // stp d3, d2, [sp, #-16]!
4245 // stp d5, d4, [sp, #-16]!
4246 // stp d7, d6, [sp, #-16]!
4247 // bl _resolver
4248 // adrp x16, lazy_pointer@GOTPAGE
4249 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF]
4250 // str x0, [x16]
4251 // mov x16, x0
4252 // ldp d7, d6, [sp], #16
4253 // ldp d5, d4, [sp], #16
4254 // ldp d3, d2, [sp], #16
4255 // ldp d1, d0, [sp], #16
4256 // ldp x7, x6, [sp], #16
4257 // ldp x5, x4, [sp], #16
4258 // ldp x3, x2, [sp], #16
4259 // ldp x1, x0, [sp], #16
4260 // ldp fp, lr, [sp], #16
4261 // br x16
4262
4263 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPXpre)
4264 .addReg(Reg: AArch64::SP)
4265 .addReg(Reg: AArch64::FP)
4266 .addReg(Reg: AArch64::LR)
4267 .addReg(Reg: AArch64::SP)
4268 .addImm(Val: -2));
4269
4270 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
4271 .addReg(Reg: AArch64::FP)
4272 .addReg(Reg: AArch64::SP)
4273 .addImm(Val: 0)
4274 .addImm(Val: 0));
4275
4276 for (int I = 0; I != 4; ++I)
4277 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPXpre)
4278 .addReg(Reg: AArch64::SP)
4279 .addReg(Reg: AArch64::X1 + 2 * I)
4280 .addReg(Reg: AArch64::X0 + 2 * I)
4281 .addReg(Reg: AArch64::SP)
4282 .addImm(Val: -2));
4283
4284 for (int I = 0; I != 4; ++I)
4285 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPDpre)
4286 .addReg(Reg: AArch64::SP)
4287 .addReg(Reg: AArch64::D1 + 2 * I)
4288 .addReg(Reg: AArch64::D0 + 2 * I)
4289 .addReg(Reg: AArch64::SP)
4290 .addImm(Val: -2));
4291
4292 EmitToStreamer(
4293 Inst: MCInstBuilder(AArch64::BL)
4294 .addOperand(Op: MCOperand::createExpr(Val: lowerConstant(CV: GI.getResolver()))));
4295
4296 {
4297 MCInst Adrp;
4298 Adrp.setOpcode(AArch64::ADRP);
4299 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
4300 MCOperand SymPage;
4301 MCInstLowering.lowerOperand(
4302 MO: MachineOperand::CreateES(SymName: LazyPointer->getName().data() + 1,
4303 TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGE),
4304 MCOp&: SymPage);
4305 Adrp.addOperand(Op: SymPage);
4306 EmitToStreamer(Inst: Adrp);
4307 }
4308
4309 {
4310 MCInst Ldr;
4311 Ldr.setOpcode(AArch64::LDRXui);
4312 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
4313 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
4314 MCOperand SymPageOff;
4315 MCInstLowering.lowerOperand(
4316 MO: MachineOperand::CreateES(SymName: LazyPointer->getName().data() + 1,
4317 TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGEOFF),
4318 MCOp&: SymPageOff);
4319 Ldr.addOperand(Op: SymPageOff);
4320 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
4321 EmitToStreamer(Inst: Ldr);
4322 }
4323
4324 EmitToStreamer(Inst: MCInstBuilder(AArch64::STRXui)
4325 .addReg(Reg: AArch64::X0)
4326 .addReg(Reg: AArch64::X16)
4327 .addImm(Val: 0));
4328
4329 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
4330 .addReg(Reg: AArch64::X16)
4331 .addReg(Reg: AArch64::X0)
4332 .addImm(Val: 0)
4333 .addImm(Val: 0));
4334
4335 for (int I = 3; I != -1; --I)
4336 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDPDpost)
4337 .addReg(Reg: AArch64::SP)
4338 .addReg(Reg: AArch64::D1 + 2 * I)
4339 .addReg(Reg: AArch64::D0 + 2 * I)
4340 .addReg(Reg: AArch64::SP)
4341 .addImm(Val: 2));
4342
4343 for (int I = 3; I != -1; --I)
4344 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDPXpost)
4345 .addReg(Reg: AArch64::SP)
4346 .addReg(Reg: AArch64::X1 + 2 * I)
4347 .addReg(Reg: AArch64::X0 + 2 * I)
4348 .addReg(Reg: AArch64::SP)
4349 .addImm(Val: 2));
4350
4351 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDPXpost)
4352 .addReg(Reg: AArch64::SP)
4353 .addReg(Reg: AArch64::FP)
4354 .addReg(Reg: AArch64::LR)
4355 .addReg(Reg: AArch64::SP)
4356 .addImm(Val: 2));
4357
4358 EmitToStreamer(Inst: MCInstBuilder(TM.getTargetTriple().isArm64e() ? AArch64::BRAAZ
4359 : AArch64::BR)
4360 .addReg(Reg: AArch64::X16));
4361}
4362
4363const MCExpr *AArch64AsmPrinter::lowerConstant(const Constant *CV,
4364 const Constant *BaseCV,
4365 uint64_t Offset) {
4366 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: CV)) {
4367 return MCSymbolRefExpr::create(Symbol: MCInstLowering.GetGlobalValueSymbol(GV, TargetFlags: 0),
4368 Ctx&: OutContext);
4369 }
4370
4371 return AsmPrinter::lowerConstant(CV, BaseCV, Offset);
4372}
4373
4374char AArch64AsmPrinter::ID = 0;
4375
4376INITIALIZE_PASS(AArch64AsmPrinter, "aarch64-asm-printer",
4377 "AArch64 Assembly Printer", false, false)
4378
4379// Force static initialization.
4380extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
4381LLVMInitializeAArch64AsmPrinter() {
4382 RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
4383 RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
4384 RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
4385 RegisterAsmPrinter<AArch64AsmPrinter> W(getTheARM64_32Target());
4386 RegisterAsmPrinter<AArch64AsmPrinter> V(getTheAArch64_32Target());
4387}
4388
4389PreservedAnalyses AArch64AsmPrinterBeginPass::run(Module &M,
4390 ModuleAnalysisManager &MAM) {
4391 AArch64AsmPrinter &AsmPrinter = static_cast<AArch64AsmPrinter &>(
4392 MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter());
4393 setupModuleAsmPrinter(M, MAM, AsmPrinter);
4394 AsmPrinter.doInitialization(M);
4395 return PreservedAnalyses::all();
4396}
4397
4398PreservedAnalyses
4399AArch64AsmPrinterPass::run(MachineFunction &MF,
4400 MachineFunctionAnalysisManager &MFAM) {
4401 AArch64AsmPrinter &AsmPrinter = static_cast<AArch64AsmPrinter &>(
4402 MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(IR&: MF)
4403 .getCachedResult<AsmPrinterAnalysis>(IR&: *MF.getFunction().getParent())
4404 ->getPrinter());
4405 setupMachineFunctionAsmPrinter(MFAM, MF, AsmPrinter);
4406 AsmPrinter.runOnMachineFunction(MF);
4407 return PreservedAnalyses::all();
4408}
4409
4410PreservedAnalyses AArch64AsmPrinterEndPass::run(Module &M,
4411 ModuleAnalysisManager &MAM) {
4412 AArch64AsmPrinter &AsmPrinter = static_cast<AArch64AsmPrinter &>(
4413 MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter());
4414 setupModuleAsmPrinter(M, MAM, AsmPrinter);
4415 AsmPrinter.doFinalization(M);
4416 return PreservedAnalyses::all();
4417}
4418