1//===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a printer that converts from our internal representation
10// of machine-dependent LLVM code to the AArch64 assembly language.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AArch64AsmPrinter.h"
15#include "AArch64.h"
16#include "AArch64MCInstLower.h"
17#include "AArch64MachineFunctionInfo.h"
18#include "AArch64RegisterInfo.h"
19#include "AArch64Subtarget.h"
20#include "AArch64TargetObjectFile.h"
21#include "MCTargetDesc/AArch64AddressingModes.h"
22#include "MCTargetDesc/AArch64InstPrinter.h"
23#include "MCTargetDesc/AArch64MCAsmInfo.h"
24#include "MCTargetDesc/AArch64MCTargetDesc.h"
25#include "MCTargetDesc/AArch64TargetStreamer.h"
26#include "TargetInfo/AArch64TargetInfo.h"
27#include "Utils/AArch64BaseInfo.h"
28#include "llvm/ADT/ScopeExit.h"
29#include "llvm/ADT/SmallString.h"
30#include "llvm/ADT/SmallVector.h"
31#include "llvm/ADT/Statistic.h"
32#include "llvm/ADT/StringRef.h"
33#include "llvm/ADT/Twine.h"
34#include "llvm/BinaryFormat/COFF.h"
35#include "llvm/BinaryFormat/ELF.h"
36#include "llvm/BinaryFormat/MachO.h"
37#include "llvm/CodeGen/AsmPrinter.h"
38#include "llvm/CodeGen/AsmPrinterAnalysis.h"
39#include "llvm/CodeGen/FaultMaps.h"
40#include "llvm/CodeGen/MachineBasicBlock.h"
41#include "llvm/CodeGen/MachineFunction.h"
42#include "llvm/CodeGen/MachineInstr.h"
43#include "llvm/CodeGen/MachineJumpTableInfo.h"
44#include "llvm/CodeGen/MachineModuleInfoImpls.h"
45#include "llvm/CodeGen/MachineOperand.h"
46#include "llvm/CodeGen/StackMaps.h"
47#include "llvm/CodeGen/TargetRegisterInfo.h"
48#include "llvm/IR/Analysis.h"
49#include "llvm/IR/DataLayout.h"
50#include "llvm/IR/DebugInfoMetadata.h"
51#include "llvm/IR/Mangler.h"
52#include "llvm/IR/Module.h"
53#include "llvm/IR/PassManager.h"
54#include "llvm/MC/MCAsmInfo.h"
55#include "llvm/MC/MCContext.h"
56#include "llvm/MC/MCExpr.h"
57#include "llvm/MC/MCInst.h"
58#include "llvm/MC/MCInstBuilder.h"
59#include "llvm/MC/MCSectionELF.h"
60#include "llvm/MC/MCSectionMachO.h"
61#include "llvm/MC/MCStreamer.h"
62#include "llvm/MC/MCSymbol.h"
63#include "llvm/MC/MCValue.h"
64#include "llvm/MC/TargetRegistry.h"
65#include "llvm/Support/Casting.h"
66#include "llvm/Support/Compiler.h"
67#include "llvm/Support/ErrorHandling.h"
68#include "llvm/Support/raw_ostream.h"
69#include "llvm/Target/TargetMachine.h"
70#include "llvm/TargetParser/Triple.h"
71#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
72#include <cassert>
73#include <cstdint>
74#include <map>
75#include <memory>
76
77using namespace llvm;
78using AArch64PAuth::PtrauthCheckMode;
79
80#define DEBUG_TYPE "AArch64AsmPrinter"
81
82// Doesn't count FPR128 ZCZ instructions which are handled
83// by TableGen pattern matching
84STATISTIC(NumZCZeroingInstrsFPR,
85 "Number of zero-cycle FPR zeroing instructions expanded from "
86 "canonical pseudo instructions");
87
88namespace {
89
90class AArch64AsmPrinter : public AsmPrinter {
91 AArch64MCInstLower MCInstLowering;
92 FaultMaps FM;
93 const AArch64Subtarget *STI;
94 bool ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = false;
95 bool PtrauthInitFini = false;
96 bool PtrauthInitFiniAddressDisc = false;
97#ifndef NDEBUG
98 unsigned InstsEmitted;
99#endif
100 bool EnableImportCallOptimization = false;
101 MapVector<MCSection *, std::vector<std::pair<MCSymbol *, MCSymbol *>>>
102 SectionToImportedFunctionCalls;
103 unsigned PAuthIFuncNextUniqueID = 1;
104
105public:
106 static char ID;
107
108 AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
109 : AsmPrinter(TM, std::move(Streamer), ID),
110 MCInstLowering(OutContext, *this), FM(*this) {}
111
112 StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
113
114 /// Wrapper for MCInstLowering.lowerOperand() for the
115 /// tblgen'erated pseudo lowering.
116 bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
117 return MCInstLowering.lowerOperand(MO, MCOp);
118 }
119
120 const MCExpr *lowerConstantPtrAuth(const ConstantPtrAuth &CPA) override;
121
122 const MCExpr *lowerBlockAddressConstant(const BlockAddress &BA) override;
123
124 void emitStartOfAsmFile(Module &M) override;
125 void emitJumpTableImpl(const MachineJumpTableInfo &MJTI,
126 ArrayRef<unsigned> JumpTableIndices) override;
127 std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
128 codeview::JumpTableEntrySize>
129 getCodeViewJumpTableInfo(int JTI, const MachineInstr *BranchInstr,
130 const MCSymbol *BranchLabel) const override;
131
132 void emitFunctionEntryLabel() override;
133
134 void emitXXStructor(const DataLayout &DL, const Constant *CV) override;
135
136 void LowerJumpTableDest(MCStreamer &OutStreamer, const MachineInstr &MI);
137
138 void LowerHardenedBRJumpTable(const MachineInstr &MI);
139
140 void LowerMOPS(MCStreamer &OutStreamer, const MachineInstr &MI);
141
142 void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
143 const MachineInstr &MI);
144 void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
145 const MachineInstr &MI);
146 void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
147 const MachineInstr &MI);
148 void LowerFAULTING_OP(const MachineInstr &MI);
149
150 void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
151 void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
152 void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
153 void LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI, bool Typed);
154
155 typedef std::tuple<unsigned, bool, uint32_t, bool, uint64_t>
156 HwasanMemaccessTuple;
157 std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
158 void LowerKCFI_CHECK(const MachineInstr &MI);
159 void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
160 void emitHwasanMemaccessSymbols(Module &M);
161
162 void emitSled(const MachineInstr &MI, SledKind Kind);
163
164 // Returns whether Reg may be used to store sensitive temporary values when
165 // expanding PtrAuth pseudos. Some OSes may take extra care to protect a
166 // small subset of GPRs on context switches - use these registers then.
167 //
168 // If there are no preferred registers, returns true for any Reg.
169 bool isPtrauthRegSafe(Register Reg) const {
170 if (STI->isX16X17Safer())
171 return Reg == AArch64::X16 || Reg == AArch64::X17;
172
173 return true;
174 }
175
176 // Emit the sequence for BRA/BLRA (authenticate + branch/call).
177 void emitPtrauthBranch(const MachineInstr *MI);
178
179 void emitPtrauthCheckAuthenticatedValue(Register TestedReg,
180 Register ScratchReg,
181 AArch64PACKey::ID Key,
182 AArch64PAuth::AuthCheckMethod Method,
183 const MCSymbol *OnFailure = nullptr);
184
185 // Check authenticated LR before tail calling.
186 void emitPtrauthTailCallHardening(const MachineInstr *TC);
187
188 struct PtrAuthSchema {
189 static PtrAuthSchema CreateImmReg(AArch64PACKey::ID Key, uint64_t IntDisc,
190 const MachineOperand &AddrDiscOp);
191 static PtrAuthSchema CreateRegReg(AArch64PACKey::ID Key, Register AddrDisc,
192 Register PCDisc);
193
194 AArch64PACKey::ID Key;
195 uint64_t IntDisc;
196 Register AddrDisc;
197 bool AddrDiscIsKilled;
198 Register PCDisc;
199
200 bool addrDiscIsKilledAndNoneOf(std::initializer_list<Register> Regs) {
201 return AddrDiscIsKilled && !llvm::is_contained(Set: Regs, Element: AddrDisc);
202 }
203 };
204
205 // Helper for emitting AUTRELLOADPAC: increment Pointer by Addend and then by
206 // a 32-bit signed value loaded from memory. The instructions emitted are
207 //
208 // ldrsw Scratch, [Pointer, #Addend]!
209 // add Pointer, Pointer, Scratch
210 //
211 // for small Addend value, with longer sequences required for wider Addend.
212 void emitPtrauthApplyIndirectAddend(Register Pointer, Register Scratch,
213 int64_t Addend);
214
215 // Emit the sequence for AUT or AUTPAC (or their PC-blending variants).
216 // Addend is only used for AUTRELLOADPAC.
217 void emitPtrauthAuthResign(Register Pointer, Register Scratch,
218 PtrAuthSchema AuthSchema,
219 std::optional<PtrAuthSchema> SignSchema,
220 std::optional<int64_t> Addend, Value *DS);
221
222 // Emit R_AARCH64_PATCHINST, the deactivation symbol relocation. Returns true
223 // if no instruction should be emitted because the deactivation symbol is
224 // defined in the current module so this function emitted a NOP instead.
225 bool emitDeactivationSymbolRelocation(Value *DS);
226
227 // Emit the sequence for PAC.
228 void emitPtrauthSign(const MachineInstr *MI);
229
230 // Emit the sequence to compute the discriminator.
231 //
232 // The Scratch register passed to this function must be safe, as returned by
233 // isPtrauthRegSafe(ScratchReg).
234 //
235 // The returned register is either ScratchReg, AddrDisc, or XZR. Furthermore,
236 // it is guaranteed to be safe (or XZR), with the only exception of
237 // passing-through an *unmodified* unsafe AddrDisc register.
238 //
239 // If the expanded pseudo is allowed to clobber AddrDisc register, setting
240 // MayClobberAddrDisc may save one MOV instruction, provided
241 // isPtrauthRegSafe(AddrDisc) is true:
242 //
243 // mov x17, x16
244 // movk x17, #1234, lsl #48
245 // ; x16 is not used anymore
246 //
247 // can be replaced by
248 //
249 // movk x16, #1234, lsl #48
250 Register emitPtrauthDiscriminator(uint64_t Disc, Register AddrDisc,
251 Register ScratchReg,
252 bool MayClobberAddrDisc = false);
253
254 // Emit the sequence for LOADauthptrstatic
255 void LowerLOADauthptrstatic(const MachineInstr &MI);
256
257 // Emit the sequence for LOADgotPAC/MOVaddrPAC (either GOT adrp-ldr or
258 // adrp-add followed by PAC sign)
259 void LowerMOVaddrPAC(const MachineInstr &MI);
260
261 // Emit the sequence for LOADgotAUTH (load signed pointer from signed ELF GOT
262 // and authenticate it with, if FPAC bit is not set, check+trap sequence after
263 // authenticating)
264 void LowerLOADgotAUTH(const MachineInstr &MI);
265
266 void emitAddImm(MCRegister Val, int64_t Addend, MCRegister Tmp);
267 void emitAddress(MCRegister Reg, const MCExpr *Expr, MCRegister Tmp,
268 bool DSOLocal, const MCSubtargetInfo &STI);
269
270 const MCExpr *emitPAuthRelocationAsIRelative(
271 const MCExpr *Target, uint64_t Disc, AArch64PACKey::ID KeyID,
272 bool HasAddressDiversity, bool IsDSOLocal, const MCExpr *DSExpr);
273
274 /// tblgen'erated driver function for lowering simple MI->MC
275 /// pseudo instructions.
276 bool lowerPseudoInstExpansion(const MachineInstr *MI, MCInst &Inst);
277
278 // Emit Build Attributes
279 void emitAttributes(unsigned Flags, uint64_t PAuthABIPlatform,
280 uint64_t PAuthABIVersion, AArch64TargetStreamer *TS);
281
282 // Emit expansion of Compare-and-branch pseudo instructions
283 void emitCBPseudoExpansion(const MachineInstr *MI);
284
285 // Emit expansion of atomic store with hint pseudo instructions
286 void emitAtomicHintPseudoExpansion(const MachineInstr *MI);
287 void emitAtomicHintPseudoExpansionRO(const MachineInstr *MI);
288 void emitAtomicHintPseudoExpansionImm(const MachineInstr *MI);
289
290 // Emit expansion of atomic fetch with hint pseudo instructions
291 void emitAtomicFetchHintPseudoExpansion(const MachineInstr *MI);
292
293 void EmitToStreamer(MCStreamer &S, const MCInst &Inst);
294 void EmitToStreamer(const MCInst &Inst) {
295 EmitToStreamer(S&: *OutStreamer, Inst);
296 }
297
298 void emitInstruction(const MachineInstr *MI) override;
299
300 void emitFunctionHeaderComment() override;
301
302 void getAnalysisUsage(AnalysisUsage &AU) const override {
303 AsmPrinter::getAnalysisUsage(AU);
304 AU.setPreservesAll();
305 }
306
307 bool runOnMachineFunction(MachineFunction &MF) override {
308 if (auto *PSIW = getAnalysisIfAvailable<ProfileSummaryInfoWrapperPass>())
309 PSI = &PSIW->getPSI();
310 if (auto *SDPIW =
311 getAnalysisIfAvailable<StaticDataProfileInfoWrapperPass>())
312 SDPI = &SDPIW->getStaticDataProfileInfo();
313
314 AArch64FI = MF.getInfo<AArch64FunctionInfo>();
315 STI = &MF.getSubtarget<AArch64Subtarget>();
316
317 SetupMachineFunction(MF);
318
319 if (STI->isTargetCOFF()) {
320 bool Local = MF.getFunction().hasLocalLinkage();
321 COFF::SymbolStorageClass Scl =
322 Local ? COFF::IMAGE_SYM_CLASS_STATIC : COFF::IMAGE_SYM_CLASS_EXTERNAL;
323 int Type =
324 COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
325
326 OutStreamer->beginCOFFSymbolDef(Symbol: CurrentFnSym);
327 OutStreamer->emitCOFFSymbolStorageClass(StorageClass: Scl);
328 OutStreamer->emitCOFFSymbolType(Type);
329 OutStreamer->endCOFFSymbolDef();
330 }
331
332 // Emit the rest of the function body.
333 emitFunctionBody();
334
335 // Emit the XRay table for this function.
336 emitXRayTable();
337
338 // We didn't modify anything.
339 return false;
340 }
341
342 const MCExpr *lowerConstant(const Constant *CV,
343 const Constant *BaseCV = nullptr,
344 uint64_t Offset = 0) override;
345
346private:
347 void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
348 bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
349 bool printAsmRegInClass(const MachineOperand &MO,
350 const TargetRegisterClass *RC, unsigned AltName,
351 raw_ostream &O);
352
353 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
354 const char *ExtraCode, raw_ostream &O) override;
355 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
356 const char *ExtraCode, raw_ostream &O) override;
357
358 void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
359
360 void emitFunctionBodyEnd() override;
361 void emitBasicBlockEnd(const MachineBasicBlock &MBB) override;
362 void emitGlobalAlias(const Module &M, const GlobalAlias &GA) override;
363
364 MCSymbol *GetCPISymbol(unsigned CPID) const override;
365 void emitEndOfAsmFile(Module &M) override;
366
367 AArch64FunctionInfo *AArch64FI = nullptr;
368
369 /// Emit the LOHs contained in AArch64FI.
370 void emitLOHs();
371
372 void emitMovXReg(Register Dest, Register Src);
373 void emitMOVZ(Register Dest, uint64_t Imm, unsigned Shift);
374 void emitMOVK(Register Dest, uint64_t Imm, unsigned Shift);
375
376 void emitAUT(AArch64PACKey::ID Key, Register Pointer, Register Disc);
377 void emitPAC(AArch64PACKey::ID Key, Register Pointer, Register Disc);
378 void emitBLRA(bool IsCall, AArch64PACKey::ID Key, Register Target,
379 Register Disc);
380
381 /// Emit instruction to set float register to zero.
382 void emitFMov0(const MachineInstr &MI);
383 void emitFMov0AsFMov(const MachineInstr &MI, Register DestReg);
384
385 using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
386
387 MInstToMCSymbol LOHInstToLabel;
388
389 bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const override {
390 return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags;
391 }
392
393 const MCSubtargetInfo *getIFuncMCSubtargetInfo() const override {
394 assert(STI);
395 return STI;
396 }
397 void emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI,
398 MCSymbol *LazyPointer) override;
399 void emitMachOIFuncStubHelperBody(Module &M, const GlobalIFunc &GI,
400 MCSymbol *LazyPointer) override;
401
402 /// Checks if this instruction is part of a sequence that is eligle for import
403 /// call optimization and, if so, records it to be emitted in the import call
404 /// section.
405 void recordIfImportCall(const MachineInstr *BranchInst);
406};
407
408} // end anonymous namespace
409
410// Get boolean module flag (0 or 1), treating absent flag as having value 0.
411static bool getOptionalBooleanModuleFlag(Module &M, StringRef Name) {
412 Metadata *Flag = M.getModuleFlag(Key: Name);
413 if (!Flag)
414 return false;
415
416 uint64_t Value = mdconst::extract<ConstantInt>(MD&: Flag)->getZExtValue();
417 assert((Value == 0 || Value == 1) && "Boolean flag is expected, if present");
418 return Value;
419}
420
421void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
422 const Triple &TT = M.getTargetTriple();
423
424 if (TT.isOSBinFormatCOFF()) {
425 emitCOFFFeatureSymbol(M);
426 emitCOFFReplaceableFunctionData(M);
427
428 if (M.getModuleFlag(Key: "import-call-optimization"))
429 EnableImportCallOptimization = true;
430 }
431
432 PtrauthInitFini = getOptionalBooleanModuleFlag(M, Name: "ptrauth-init-fini");
433 PtrauthInitFiniAddressDisc = getOptionalBooleanModuleFlag(
434 M, Name: "ptrauth-init-fini-address-discrimination");
435
436 if (!TT.isOSBinFormatELF())
437 return;
438
439 // For emitting build attributes and .note.gnu.property section
440 auto *TS =
441 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
442 // Assemble feature flags that may require creation of build attributes and a
443 // note section.
444 unsigned BAFlags = 0;
445 unsigned GNUFlags = 0;
446 if (const auto *BTE = mdconst::extract_or_null<ConstantInt>(
447 MD: M.getModuleFlag(Key: "branch-target-enforcement"))) {
448 if (!BTE->isZero()) {
449 BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_BTI_Flag;
450 GNUFlags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
451 }
452 }
453
454 if (const auto *GCS = mdconst::extract_or_null<ConstantInt>(
455 MD: M.getModuleFlag(Key: "guarded-control-stack"))) {
456 if (!GCS->isZero()) {
457 BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_GCS_Flag;
458 GNUFlags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_GCS;
459 }
460 }
461
462 if (const auto *Sign = mdconst::extract_or_null<ConstantInt>(
463 MD: M.getModuleFlag(Key: "sign-return-address"))) {
464 if (!Sign->isZero()) {
465 BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_PAC_Flag;
466 GNUFlags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
467 }
468 }
469
470 uint64_t PAuthABIPlatform = -1;
471 if (const auto *PAP = mdconst::extract_or_null<ConstantInt>(
472 MD: M.getModuleFlag(Key: "aarch64-elf-pauthabi-platform"))) {
473 PAuthABIPlatform = PAP->getZExtValue();
474 }
475
476 uint64_t PAuthABIVersion = -1;
477 if (const auto *PAV = mdconst::extract_or_null<ConstantInt>(
478 MD: M.getModuleFlag(Key: "aarch64-elf-pauthabi-version"))) {
479 PAuthABIVersion = PAV->getZExtValue();
480 }
481
482 // For LLVM_LINUX experimental platform, version value of 0 means no PAuth
483 // support. Do not emit corresponding PAuthABI GNU property note and AArch64
484 // build attributes for this case to keep Linux binaries not using PAuth
485 // unaffected.
486 if (PAuthABIPlatform == ELF::AARCH64_PAUTH_PLATFORM_LLVM_LINUX &&
487 PAuthABIVersion == 0) {
488 PAuthABIPlatform = uint64_t(-1);
489 PAuthABIVersion = uint64_t(-1);
490 }
491
492 // Emit AArch64 Build Attributes
493 emitAttributes(Flags: BAFlags, PAuthABIPlatform, PAuthABIVersion, TS);
494 // Emit a .note.gnu.property section with the flags.
495 TS->emitNoteSection(Flags: GNUFlags, PAuthABIPlatform, PAuthABIVersion);
496}
497
498void AArch64AsmPrinter::emitFunctionHeaderComment() {
499 const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>();
500 std::optional<std::string> OutlinerString = FI->getOutliningStyle();
501 if (OutlinerString != std::nullopt)
502 OutStreamer->getCommentOS() << ' ' << OutlinerString;
503}
504
505void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
506{
507 const Function &F = MF->getFunction();
508 if (F.hasFnAttribute(Kind: "patchable-function-entry")) {
509 unsigned Num;
510 if (F.getFnAttribute(Kind: "patchable-function-entry")
511 .getValueAsString()
512 .getAsInteger(Radix: 10, Result&: Num))
513 return;
514 emitNops(N: Num);
515 return;
516 }
517
518 emitSled(MI, Kind: SledKind::FUNCTION_ENTER);
519}
520
521void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI) {
522 emitSled(MI, Kind: SledKind::FUNCTION_EXIT);
523}
524
525void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) {
526 emitSled(MI, Kind: SledKind::TAIL_CALL);
527}
528
529void AArch64AsmPrinter::emitSled(const MachineInstr &MI, SledKind Kind) {
530 static const int8_t NoopsInSledCount = 7;
531 // We want to emit the following pattern:
532 //
533 // .Lxray_sled_N:
534 // ALIGN
535 // B #32
536 // ; 7 NOP instructions (28 bytes)
537 // .tmpN
538 //
539 // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
540 // over the full 32 bytes (8 instructions) with the following pattern:
541 //
542 // STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
543 // LDR W17, #12 ; W17 := function ID
544 // LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
545 // BLR X16 ; call the tracing trampoline
546 // ;DATA: 32 bits of function ID
547 // ;DATA: lower 32 bits of the address of the trampoline
548 // ;DATA: higher 32 bits of the address of the trampoline
549 // LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
550 //
551 OutStreamer->emitCodeAlignment(Alignment: Align(4), STI: getSubtargetInfo());
552 auto CurSled = OutContext.createTempSymbol(Name: "xray_sled_", AlwaysAddSuffix: true);
553 OutStreamer->emitLabel(Symbol: CurSled);
554 auto Target = OutContext.createTempSymbol();
555
556 // Emit "B #32" instruction, which jumps over the next 28 bytes.
557 // The operand has to be the number of 4-byte instructions to jump over,
558 // including the current instruction.
559 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::B).addImm(Val: 8));
560
561 for (int8_t I = 0; I < NoopsInSledCount; I++)
562 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::NOP));
563
564 OutStreamer->emitLabel(Symbol: Target);
565 recordSled(Sled: CurSled, MI, Kind, Version: 2);
566}
567
568void AArch64AsmPrinter::emitAttributes(unsigned Flags,
569 uint64_t PAuthABIPlatform,
570 uint64_t PAuthABIVersion,
571 AArch64TargetStreamer *TS) {
572
573 PAuthABIPlatform = (uint64_t(-1) == PAuthABIPlatform) ? 0 : PAuthABIPlatform;
574 PAuthABIVersion = (uint64_t(-1) == PAuthABIVersion) ? 0 : PAuthABIVersion;
575
576 if (PAuthABIPlatform || PAuthABIVersion) {
577 TS->emitAttributesSubsection(
578 VendorName: AArch64BuildAttributes::getVendorName(
579 Vendor: AArch64BuildAttributes::AEABI_PAUTHABI),
580 IsOptional: AArch64BuildAttributes::SubsectionOptional::REQUIRED,
581 ParameterType: AArch64BuildAttributes::SubsectionType::ULEB128);
582 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
583 Vendor: AArch64BuildAttributes::AEABI_PAUTHABI),
584 Tag: AArch64BuildAttributes::TAG_PAUTH_PLATFORM,
585 Value: PAuthABIPlatform, String: "");
586 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
587 Vendor: AArch64BuildAttributes::AEABI_PAUTHABI),
588 Tag: AArch64BuildAttributes::TAG_PAUTH_SCHEMA, Value: PAuthABIVersion,
589 String: "");
590 }
591
592 unsigned BTIValue =
593 (Flags & AArch64BuildAttributes::Feature_BTI_Flag) ? 1 : 0;
594 unsigned PACValue =
595 (Flags & AArch64BuildAttributes::Feature_PAC_Flag) ? 1 : 0;
596 unsigned GCSValue =
597 (Flags & AArch64BuildAttributes::Feature_GCS_Flag) ? 1 : 0;
598
599 if (BTIValue || PACValue || GCSValue) {
600 TS->emitAttributesSubsection(
601 VendorName: AArch64BuildAttributes::getVendorName(
602 Vendor: AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
603 IsOptional: AArch64BuildAttributes::SubsectionOptional::OPTIONAL,
604 ParameterType: AArch64BuildAttributes::SubsectionType::ULEB128);
605 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
606 Vendor: AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
607 Tag: AArch64BuildAttributes::TAG_FEATURE_BTI, Value: BTIValue, String: "");
608 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
609 Vendor: AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
610 Tag: AArch64BuildAttributes::TAG_FEATURE_PAC, Value: PACValue, String: "");
611 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
612 Vendor: AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
613 Tag: AArch64BuildAttributes::TAG_FEATURE_GCS, Value: GCSValue, String: "");
614 }
615}
616
617// Emit the following code for Intrinsic::{xray_customevent,xray_typedevent}
618// (built-in functions __xray_customevent/__xray_typedevent).
619//
620// .Lxray_event_sled_N:
621// b 1f
622// save x0 and x1 (and also x2 for TYPED_EVENT_CALL)
623// set up x0 and x1 (and also x2 for TYPED_EVENT_CALL)
624// bl __xray_CustomEvent or __xray_TypedEvent
625// restore x0 and x1 (and also x2 for TYPED_EVENT_CALL)
626// 1:
627//
628// There are 6 instructions for EVENT_CALL and 9 for TYPED_EVENT_CALL.
629//
630// Then record a sled of kind CUSTOM_EVENT or TYPED_EVENT.
631// After patching, b .+N will become a nop.
632void AArch64AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI,
633 bool Typed) {
634 auto &O = *OutStreamer;
635 MCSymbol *CurSled = OutContext.createTempSymbol(Name: "xray_sled_", AlwaysAddSuffix: true);
636 O.emitLabel(Symbol: CurSled);
637 bool MachO = MMI->getModule()->getTargetTriple().isOSBinFormatMachO();
638 auto *Sym = MCSymbolRefExpr::create(
639 Symbol: OutContext.getOrCreateSymbol(
640 Name: Twine(MachO ? "_" : "") +
641 (Typed ? "__xray_TypedEvent" : "__xray_CustomEvent")),
642 Ctx&: OutContext);
643 if (Typed) {
644 O.AddComment(T: "Begin XRay typed event");
645 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::B).addImm(Val: 9));
646 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::STPXpre)
647 .addReg(Reg: AArch64::SP)
648 .addReg(Reg: AArch64::X0)
649 .addReg(Reg: AArch64::X1)
650 .addReg(Reg: AArch64::SP)
651 .addImm(Val: -4));
652 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::STRXui)
653 .addReg(Reg: AArch64::X2)
654 .addReg(Reg: AArch64::SP)
655 .addImm(Val: 2));
656 emitMovXReg(Dest: AArch64::X0, Src: MI.getOperand(i: 0).getReg());
657 emitMovXReg(Dest: AArch64::X1, Src: MI.getOperand(i: 1).getReg());
658 emitMovXReg(Dest: AArch64::X2, Src: MI.getOperand(i: 2).getReg());
659 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::BL).addExpr(Val: Sym));
660 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::LDRXui)
661 .addReg(Reg: AArch64::X2)
662 .addReg(Reg: AArch64::SP)
663 .addImm(Val: 2));
664 O.AddComment(T: "End XRay typed event");
665 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::LDPXpost)
666 .addReg(Reg: AArch64::SP)
667 .addReg(Reg: AArch64::X0)
668 .addReg(Reg: AArch64::X1)
669 .addReg(Reg: AArch64::SP)
670 .addImm(Val: 4));
671
672 recordSled(Sled: CurSled, MI, Kind: SledKind::TYPED_EVENT, Version: 2);
673 } else {
674 O.AddComment(T: "Begin XRay custom event");
675 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::B).addImm(Val: 6));
676 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::STPXpre)
677 .addReg(Reg: AArch64::SP)
678 .addReg(Reg: AArch64::X0)
679 .addReg(Reg: AArch64::X1)
680 .addReg(Reg: AArch64::SP)
681 .addImm(Val: -2));
682 emitMovXReg(Dest: AArch64::X0, Src: MI.getOperand(i: 0).getReg());
683 emitMovXReg(Dest: AArch64::X1, Src: MI.getOperand(i: 1).getReg());
684 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::BL).addExpr(Val: Sym));
685 O.AddComment(T: "End XRay custom event");
686 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::LDPXpost)
687 .addReg(Reg: AArch64::SP)
688 .addReg(Reg: AArch64::X0)
689 .addReg(Reg: AArch64::X1)
690 .addReg(Reg: AArch64::SP)
691 .addImm(Val: 2));
692
693 recordSled(Sled: CurSled, MI, Kind: SledKind::CUSTOM_EVENT, Version: 2);
694 }
695}
696
697void AArch64AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
698 Register AddrReg = MI.getOperand(i: 0).getReg();
699 assert(std::next(MI.getIterator())->isCall() &&
700 "KCFI_CHECK not followed by a call instruction");
701 assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg &&
702 "KCFI_CHECK call target doesn't match call operand");
703
704 // Default to using the intra-procedure-call temporary registers for
705 // comparing the hashes.
706 unsigned ScratchRegs[] = {AArch64::W16, AArch64::W17};
707 if (AddrReg == AArch64::XZR) {
708 // Checking XZR makes no sense. Instead of emitting a load, zero
709 // ScratchRegs[0] and use it for the ESR AddrIndex below.
710 AddrReg = getXRegFromWReg(Reg: ScratchRegs[0]);
711 emitMovXReg(Dest: AddrReg, Src: AArch64::XZR);
712 } else {
713 // If one of the scratch registers is used for the call target (e.g.
714 // with AArch64::TCRETURNriBTI), we can clobber another caller-saved
715 // temporary register instead (in this case, AArch64::W9) as the check
716 // is immediately followed by the call instruction.
717 for (auto &Reg : ScratchRegs) {
718 if (Reg == getWRegFromXReg(Reg: AddrReg)) {
719 Reg = AArch64::W9;
720 break;
721 }
722 }
723 assert(ScratchRegs[0] != AddrReg && ScratchRegs[1] != AddrReg &&
724 "Invalid scratch registers for KCFI_CHECK");
725
726 // Adjust the offset for patchable-function-prefix. This assumes that
727 // patchable-function-prefix is the same for all functions.
728 int64_t PrefixNops =
729 MI.getMF()->getFunction().getFnAttributeAsParsedInteger(
730 Kind: "patchable-function-prefix");
731
732 // Load the target function type hash.
733 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::LDURWi)
734 .addReg(Reg: ScratchRegs[0])
735 .addReg(Reg: AddrReg)
736 .addImm(Val: -(PrefixNops * 4 + 4)));
737 }
738
739 // Load the expected type hash.
740 const int64_t Type = MI.getOperand(i: 1).getImm();
741 emitMOVK(Dest: ScratchRegs[1], Imm: Type & 0xFFFF, Shift: 0);
742 emitMOVK(Dest: ScratchRegs[1], Imm: (Type >> 16) & 0xFFFF, Shift: 16);
743
744 // Compare the hashes and trap if there's a mismatch.
745 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::SUBSWrs)
746 .addReg(Reg: AArch64::WZR)
747 .addReg(Reg: ScratchRegs[0])
748 .addReg(Reg: ScratchRegs[1])
749 .addImm(Val: 0));
750
751 MCSymbol *Pass = OutContext.createTempSymbol();
752 EmitToStreamer(S&: *OutStreamer,
753 Inst: MCInstBuilder(AArch64::Bcc)
754 .addImm(Val: AArch64CC::EQ)
755 .addExpr(Val: MCSymbolRefExpr::create(Symbol: Pass, Ctx&: OutContext)));
756
757 // The base ESR is 0x8000 and the register information is encoded in bits
758 // 0-9 as follows:
759 // - 0-4: n, where the register Xn contains the target address
760 // - 5-9: m, where the register Wm contains the expected type hash
761 // Where n, m are in [0, 30].
762 unsigned TypeIndex = ScratchRegs[1] - AArch64::W0;
763 unsigned AddrIndex;
764 switch (AddrReg) {
765 default:
766 AddrIndex = AddrReg - AArch64::X0;
767 break;
768 case AArch64::FP:
769 AddrIndex = 29;
770 break;
771 case AArch64::LR:
772 AddrIndex = 30;
773 break;
774 }
775
776 assert(AddrIndex < 31 && TypeIndex < 31);
777
778 unsigned ESR = 0x8000 | ((TypeIndex & 31) << 5) | (AddrIndex & 31);
779 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::BRK).addImm(Val: ESR));
780 OutStreamer->emitLabel(Symbol: Pass);
781}
782
783void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
784 Register Reg = MI.getOperand(i: 0).getReg();
785
786 // The HWASan pass won't emit a CHECK_MEMACCESS intrinsic with a pointer
787 // statically known to be zero. However, conceivably, the HWASan pass may
788 // encounter a "cannot currently statically prove to be null" pointer (and is
789 // therefore unable to omit the intrinsic) that later optimization passes
790 // convert into a statically known-null pointer.
791 if (Reg == AArch64::XZR)
792 return;
793
794 bool IsShort =
795 ((MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES) ||
796 (MI.getOpcode() ==
797 AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW));
798 uint32_t AccessInfo = MI.getOperand(i: 1).getImm();
799 bool IsFixedShadow =
800 ((MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_FIXEDSHADOW) ||
801 (MI.getOpcode() ==
802 AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW));
803 uint64_t FixedShadowOffset = IsFixedShadow ? MI.getOperand(i: 2).getImm() : 0;
804
805 MCSymbol *&Sym = HwasanMemaccessSymbols[HwasanMemaccessTuple(
806 Reg, IsShort, AccessInfo, IsFixedShadow, FixedShadowOffset)];
807 if (!Sym) {
808 // FIXME: Make this work on non-ELF.
809 if (!TM.getTargetTriple().isOSBinFormatELF())
810 report_fatal_error(reason: "llvm.hwasan.check.memaccess only supported on ELF");
811
812 std::string SymName = "__hwasan_check_x" + utostr(X: Reg - AArch64::X0) + "_" +
813 utostr(X: AccessInfo);
814 if (IsFixedShadow)
815 SymName += "_fixed_" + utostr(X: FixedShadowOffset);
816 if (IsShort)
817 SymName += "_short_v2";
818 Sym = OutContext.getOrCreateSymbol(Name: SymName);
819 }
820
821 EmitToStreamer(S&: *OutStreamer,
822 Inst: MCInstBuilder(AArch64::BL)
823 .addExpr(Val: MCSymbolRefExpr::create(Symbol: Sym, Ctx&: OutContext)));
824}
825
826void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) {
827 if (HwasanMemaccessSymbols.empty())
828 return;
829
830 const Triple &TT = M.getTargetTriple();
831 assert(TT.isOSBinFormatELF());
832 // AArch64Subtarget is huge, so heap allocate it so we don't run out of stack
833 // space.
834 auto STI = std::make_unique<AArch64Subtarget>(
835 args: TT, args: TM.getTargetCPU(), args: TM.getTargetCPU(), args: TM.getTargetFeatureString(), args&: TM,
836 args: true);
837 this->STI = STI.get();
838
839 MCSymbol *HwasanTagMismatchV1Sym =
840 OutContext.getOrCreateSymbol(Name: "__hwasan_tag_mismatch");
841 MCSymbol *HwasanTagMismatchV2Sym =
842 OutContext.getOrCreateSymbol(Name: "__hwasan_tag_mismatch_v2");
843
844 const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
845 MCSymbolRefExpr::create(Symbol: HwasanTagMismatchV1Sym, Ctx&: OutContext);
846 const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
847 MCSymbolRefExpr::create(Symbol: HwasanTagMismatchV2Sym, Ctx&: OutContext);
848
849 for (auto &P : HwasanMemaccessSymbols) {
850 unsigned Reg = std::get<0>(t: P.first);
851 bool IsShort = std::get<1>(t: P.first);
852 uint32_t AccessInfo = std::get<2>(t: P.first);
853 bool IsFixedShadow = std::get<3>(t: P.first);
854 uint64_t FixedShadowOffset = std::get<4>(t: P.first);
855 const MCSymbolRefExpr *HwasanTagMismatchRef =
856 IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
857 MCSymbol *Sym = P.second;
858
859 bool HasMatchAllTag =
860 (AccessInfo >> HWASanAccessInfo::HasMatchAllShift) & 1;
861 uint8_t MatchAllTag =
862 (AccessInfo >> HWASanAccessInfo::MatchAllShift) & 0xff;
863 unsigned Size =
864 1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf);
865 bool CompileKernel =
866 (AccessInfo >> HWASanAccessInfo::CompileKernelShift) & 1;
867
868 OutStreamer->switchSection(Section: OutContext.getELFSection(
869 Section: ".text.hot", Type: ELF::SHT_PROGBITS,
870 Flags: ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, EntrySize: 0, Group: Sym->getName(),
871 /*IsComdat=*/true));
872
873 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_ELF_TypeFunction);
874 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Weak);
875 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Hidden);
876 OutStreamer->emitLabel(Symbol: Sym);
877
878 EmitToStreamer(Inst: MCInstBuilder(AArch64::SBFMXri)
879 .addReg(Reg: AArch64::X16)
880 .addReg(Reg)
881 .addImm(Val: 4)
882 .addImm(Val: 55));
883
884 if (IsFixedShadow) {
885 // Aarch64 makes it difficult to embed large constants in the code.
886 // Fortuitously, kShadowBaseAlignment == 32, so we use the 32-bit
887 // left-shift option in the MOV instruction. Combined with the 16-bit
888 // immediate, this is enough to represent any offset up to 2**48.
889 emitMOVZ(Dest: AArch64::X17, Imm: FixedShadowOffset >> 32, Shift: 32);
890 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRBBroX)
891 .addReg(Reg: AArch64::W16)
892 .addReg(Reg: AArch64::X17)
893 .addReg(Reg: AArch64::X16)
894 .addImm(Val: 0)
895 .addImm(Val: 0));
896 } else {
897 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRBBroX)
898 .addReg(Reg: AArch64::W16)
899 .addReg(Reg: IsShort ? AArch64::X20 : AArch64::X9)
900 .addReg(Reg: AArch64::X16)
901 .addImm(Val: 0)
902 .addImm(Val: 0));
903 }
904
905 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSXrs)
906 .addReg(Reg: AArch64::XZR)
907 .addReg(Reg: AArch64::X16)
908 .addReg(Reg)
909 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSR, Imm: 56)));
910 MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
911 EmitToStreamer(Inst: MCInstBuilder(AArch64::Bcc)
912 .addImm(Val: AArch64CC::NE)
913 .addExpr(Val: MCSymbolRefExpr::create(
914 Symbol: HandleMismatchOrPartialSym, Ctx&: OutContext)));
915 MCSymbol *ReturnSym = OutContext.createTempSymbol();
916 OutStreamer->emitLabel(Symbol: ReturnSym);
917 EmitToStreamer(Inst: MCInstBuilder(AArch64::RET).addReg(Reg: AArch64::LR));
918 OutStreamer->emitLabel(Symbol: HandleMismatchOrPartialSym);
919
920 if (HasMatchAllTag) {
921 EmitToStreamer(Inst: MCInstBuilder(AArch64::UBFMXri)
922 .addReg(Reg: AArch64::X17)
923 .addReg(Reg)
924 .addImm(Val: 56)
925 .addImm(Val: 63));
926 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSXri)
927 .addReg(Reg: AArch64::XZR)
928 .addReg(Reg: AArch64::X17)
929 .addImm(Val: MatchAllTag)
930 .addImm(Val: 0));
931 EmitToStreamer(
932 Inst: MCInstBuilder(AArch64::Bcc)
933 .addImm(Val: AArch64CC::EQ)
934 .addExpr(Val: MCSymbolRefExpr::create(Symbol: ReturnSym, Ctx&: OutContext)));
935 }
936
937 if (IsShort) {
938 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSWri)
939 .addReg(Reg: AArch64::WZR)
940 .addReg(Reg: AArch64::W16)
941 .addImm(Val: 15)
942 .addImm(Val: 0));
943 MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
944 EmitToStreamer(
945 Inst: MCInstBuilder(AArch64::Bcc)
946 .addImm(Val: AArch64CC::HI)
947 .addExpr(Val: MCSymbolRefExpr::create(Symbol: HandleMismatchSym, Ctx&: OutContext)));
948
949 EmitToStreamer(Inst: MCInstBuilder(AArch64::ANDXri)
950 .addReg(Reg: AArch64::X17)
951 .addReg(Reg)
952 .addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: 0xf, regSize: 64)));
953 if (Size != 1)
954 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
955 .addReg(Reg: AArch64::X17)
956 .addReg(Reg: AArch64::X17)
957 .addImm(Val: Size - 1)
958 .addImm(Val: 0));
959 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSWrs)
960 .addReg(Reg: AArch64::WZR)
961 .addReg(Reg: AArch64::W16)
962 .addReg(Reg: AArch64::W17)
963 .addImm(Val: 0));
964 EmitToStreamer(
965 Inst: MCInstBuilder(AArch64::Bcc)
966 .addImm(Val: AArch64CC::LS)
967 .addExpr(Val: MCSymbolRefExpr::create(Symbol: HandleMismatchSym, Ctx&: OutContext)));
968
969 EmitToStreamer(Inst: MCInstBuilder(AArch64::ORRXri)
970 .addReg(Reg: AArch64::X16)
971 .addReg(Reg)
972 .addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: 0xf, regSize: 64)));
973 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRBBui)
974 .addReg(Reg: AArch64::W16)
975 .addReg(Reg: AArch64::X16)
976 .addImm(Val: 0));
977 EmitToStreamer(
978 Inst: MCInstBuilder(AArch64::SUBSXrs)
979 .addReg(Reg: AArch64::XZR)
980 .addReg(Reg: AArch64::X16)
981 .addReg(Reg)
982 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSR, Imm: 56)));
983 EmitToStreamer(
984 Inst: MCInstBuilder(AArch64::Bcc)
985 .addImm(Val: AArch64CC::EQ)
986 .addExpr(Val: MCSymbolRefExpr::create(Symbol: ReturnSym, Ctx&: OutContext)));
987
988 OutStreamer->emitLabel(Symbol: HandleMismatchSym);
989 }
990
991 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPXpre)
992 .addReg(Reg: AArch64::SP)
993 .addReg(Reg: AArch64::X0)
994 .addReg(Reg: AArch64::X1)
995 .addReg(Reg: AArch64::SP)
996 .addImm(Val: -32));
997 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPXi)
998 .addReg(Reg: AArch64::FP)
999 .addReg(Reg: AArch64::LR)
1000 .addReg(Reg: AArch64::SP)
1001 .addImm(Val: 29));
1002
1003 if (Reg != AArch64::X0)
1004 emitMovXReg(Dest: AArch64::X0, Src: Reg);
1005 emitMOVZ(Dest: AArch64::X1, Imm: AccessInfo & HWASanAccessInfo::RuntimeMask, Shift: 0);
1006
1007 if (CompileKernel) {
1008 // The Linux kernel's dynamic loader doesn't support GOT relative
1009 // relocations, but it doesn't support late binding either, so just call
1010 // the function directly.
1011 EmitToStreamer(Inst: MCInstBuilder(AArch64::B).addExpr(Val: HwasanTagMismatchRef));
1012 } else {
1013 // Intentionally load the GOT entry and branch to it, rather than possibly
1014 // late binding the function, which may clobber the registers before we
1015 // have a chance to save them.
1016 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADRP)
1017 .addReg(Reg: AArch64::X16)
1018 .addExpr(Val: MCSpecifierExpr::create(Expr: HwasanTagMismatchRef,
1019 S: AArch64::S_GOT_PAGE,
1020 Ctx&: OutContext)));
1021 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
1022 .addReg(Reg: AArch64::X16)
1023 .addReg(Reg: AArch64::X16)
1024 .addExpr(Val: MCSpecifierExpr::create(Expr: HwasanTagMismatchRef,
1025 S: AArch64::S_GOT_LO12,
1026 Ctx&: OutContext)));
1027 EmitToStreamer(Inst: MCInstBuilder(AArch64::BR).addReg(Reg: AArch64::X16));
1028 }
1029 }
1030 this->STI = nullptr;
1031}
1032
1033static void emitAuthenticatedPointer(MCStreamer &OutStreamer,
1034 MCSymbol *StubLabel,
1035 const MCExpr *StubAuthPtrRef) {
1036 // sym$auth_ptr$key$disc:
1037 OutStreamer.emitLabel(Symbol: StubLabel);
1038 OutStreamer.emitValue(Value: StubAuthPtrRef, /*size=*/Size: 8);
1039}
1040
1041void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
1042 emitHwasanMemaccessSymbols(M);
1043
1044 const Triple &TT = M.getTargetTriple();
1045 if (TT.isOSBinFormatMachO()) {
1046 // Output authenticated pointers as indirect symbols, if we have any.
1047 MachineModuleInfoMachO &MMIMacho =
1048 MMI->getObjFileInfo<MachineModuleInfoMachO>();
1049
1050 auto Stubs = MMIMacho.getAuthGVStubList();
1051
1052 if (!Stubs.empty()) {
1053 // Switch to the "__auth_ptr" section.
1054 OutStreamer->switchSection(
1055 Section: OutContext.getMachOSection(Segment: "__DATA", Section: "__auth_ptr", TypeAndAttributes: MachO::S_REGULAR,
1056 K: SectionKind::getMetadata()));
1057 emitAlignment(Alignment: Align(8));
1058
1059 for (const auto &Stub : Stubs)
1060 emitAuthenticatedPointer(OutStreamer&: *OutStreamer, StubLabel: Stub.first, StubAuthPtrRef: Stub.second);
1061
1062 OutStreamer->addBlankLine();
1063 }
1064
1065 // Funny Darwin hack: This flag tells the linker that no global symbols
1066 // contain code that falls through to other global symbols (e.g. the obvious
1067 // implementation of multiple entry points). If this doesn't occur, the
1068 // linker can safely perform dead code stripping. Since LLVM never
1069 // generates code that does this, it is always safe to set.
1070 OutStreamer->emitSubsectionsViaSymbols();
1071 }
1072
1073 if (TT.isOSBinFormatELF()) {
1074 // Output authenticated pointers as indirect symbols, if we have any.
1075 MachineModuleInfoELF &MMIELF = MMI->getObjFileInfo<MachineModuleInfoELF>();
1076
1077 auto Stubs = MMIELF.getAuthGVStubList();
1078
1079 if (!Stubs.empty()) {
1080 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
1081 OutStreamer->switchSection(Section: TLOF.getDataSection());
1082 emitAlignment(Alignment: Align(8));
1083
1084 for (const auto &Stub : Stubs)
1085 emitAuthenticatedPointer(OutStreamer&: *OutStreamer, StubLabel: Stub.first, StubAuthPtrRef: Stub.second);
1086
1087 OutStreamer->addBlankLine();
1088 }
1089
1090 // With signed ELF GOT enabled, the linker looks at the symbol type to
1091 // choose between keys IA (for STT_FUNC) and DA (for other types). Symbols
1092 // for functions not defined in the module have STT_NOTYPE type by default.
1093 // This makes linker to emit signing schema with DA key (instead of IA) for
1094 // corresponding R_AARCH64_AUTH_GLOB_DAT dynamic reloc. To avoid that, force
1095 // all function symbols used in the module to have STT_FUNC type. See
1096 // https://github.com/ARM-software/abi-aa/blob/main/pauthabielf64/pauthabielf64.rst#default-signing-schema
1097 const auto *PtrAuthELFGOTFlag = mdconst::extract_or_null<ConstantInt>(
1098 MD: M.getModuleFlag(Key: "ptrauth-elf-got"));
1099 if (PtrAuthELFGOTFlag && PtrAuthELFGOTFlag->getZExtValue() == 1)
1100 for (const GlobalValue &GV : M.global_values())
1101 if (!GV.use_empty() && isa<Function>(Val: GV) &&
1102 !GV.getName().starts_with(Prefix: "llvm."))
1103 OutStreamer->emitSymbolAttribute(Symbol: getSymbol(GV: &GV),
1104 Attribute: MCSA_ELF_TypeFunction);
1105 }
1106
1107 // Emit stack and fault map information.
1108 FM.serializeToFaultMapSection();
1109
1110 // If import call optimization is enabled, emit the appropriate section.
1111 // We do this whether or not we recorded any import calls.
1112 if (EnableImportCallOptimization && TT.isOSBinFormatCOFF()) {
1113 OutStreamer->switchSection(Section: getObjFileLowering().getImportCallSection());
1114
1115 // Section always starts with some magic.
1116 constexpr char ImpCallMagic[12] = "Imp_Call_V1";
1117 OutStreamer->emitBytes(Data: StringRef{ImpCallMagic, sizeof(ImpCallMagic)});
1118
1119 // Layout of this section is:
1120 // Per section that contains calls to imported functions:
1121 // uint32_t SectionSize: Size in bytes for information in this section.
1122 // uint32_t Section Number
1123 // Per call to imported function in section:
1124 // uint32_t Kind: the kind of imported function.
1125 // uint32_t BranchOffset: the offset of the branch instruction in its
1126 // parent section.
1127 // uint32_t TargetSymbolId: the symbol id of the called function.
1128 for (auto &[Section, CallsToImportedFuncs] :
1129 SectionToImportedFunctionCalls) {
1130 unsigned SectionSize =
1131 sizeof(uint32_t) * (2 + 3 * CallsToImportedFuncs.size());
1132 OutStreamer->emitInt32(Value: SectionSize);
1133 OutStreamer->emitCOFFSecNumber(Symbol: Section->getBeginSymbol());
1134 for (auto &[CallsiteSymbol, CalledSymbol] : CallsToImportedFuncs) {
1135 // Kind is always IMAGE_REL_ARM64_DYNAMIC_IMPORT_CALL (0x13).
1136 OutStreamer->emitInt32(Value: 0x13);
1137 OutStreamer->emitCOFFSecOffset(Symbol: CallsiteSymbol);
1138 OutStreamer->emitCOFFSymbolIndex(Symbol: CalledSymbol);
1139 }
1140 }
1141 }
1142}
1143
1144void AArch64AsmPrinter::emitLOHs() {
1145 SmallVector<MCSymbol *, 3> MCArgs;
1146
1147 for (const auto &D : AArch64FI->getLOHContainer()) {
1148 for (const MachineInstr *MI : D.getArgs()) {
1149 MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(x: MI);
1150 assert(LabelIt != LOHInstToLabel.end() &&
1151 "Label hasn't been inserted for LOH related instruction");
1152 MCArgs.push_back(Elt: LabelIt->second);
1153 }
1154 OutStreamer->emitLOHDirective(Kind: D.getKind(), Args: MCArgs);
1155 MCArgs.clear();
1156 }
1157}
1158
1159// Unwinding can get confused if the last instruction in an exception-handling
1160// region (function, funclet, try block, etc.) is a call, because the return
1161// address then points to the *next* region (see
1162// X86AsmPrinter::maybeEmitNopAfterCallForWindowsEH for the X86 equivalent).
1163// Trap after a noreturn call that would end a function or funclet, and elide
1164// the trap when the next instruction is in the same function.
1165void AArch64AsmPrinter::emitBasicBlockEnd(const MachineBasicBlock &MBB) {
1166 if (MBB.succ_empty() && MF->usesWindowsUnwindInfo()) {
1167 auto Next = std::next(x: MBB.getIterator());
1168 if (Next == MF->end() || Next->isEHFuncletEntry()) {
1169 for (const MachineInstr &MI : reverse(C: MBB)) {
1170 if (MI.isMetaInstruction() || AArch64InstrInfo::isSEHInstruction(MI))
1171 continue;
1172 if (MI.isCall() && !MI.isReturn())
1173 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::BRK).addImm(Val: 1));
1174 break;
1175 }
1176 }
1177 }
1178
1179 AsmPrinter::emitBasicBlockEnd(MBB);
1180}
1181
1182void AArch64AsmPrinter::emitFunctionBodyEnd() {
1183 if (!AArch64FI->getLOHRelated().empty())
1184 emitLOHs();
1185}
1186
1187/// GetCPISymbol - Return the symbol for the specified constant pool entry.
1188MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
1189 // Darwin uses a linker-private symbol name for constant-pools (to
1190 // avoid addends on the relocation?), ELF has no such concept and
1191 // uses a normal private symbol.
1192 if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
1193 return OutContext.getOrCreateSymbol(
1194 Name: Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
1195 Twine(getFunctionNumber()) + "_" + Twine(CPID));
1196
1197 return AsmPrinter::GetCPISymbol(CPID);
1198}
1199
1200void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
1201 raw_ostream &O) {
1202 const MachineOperand &MO = MI->getOperand(i: OpNum);
1203 switch (MO.getType()) {
1204 default:
1205 llvm_unreachable("<unknown operand type>");
1206 case MachineOperand::MO_Register: {
1207 Register Reg = MO.getReg();
1208 assert(Reg.isPhysical());
1209 assert(!MO.getSubReg() && "Subregs should be eliminated!");
1210 O << AArch64InstPrinter::getRegisterName(Reg);
1211 break;
1212 }
1213 case MachineOperand::MO_Immediate: {
1214 O << MO.getImm();
1215 break;
1216 }
1217 case MachineOperand::MO_GlobalAddress: {
1218 PrintSymbolOperand(MO, OS&: O);
1219 break;
1220 }
1221 case MachineOperand::MO_BlockAddress: {
1222 MCSymbol *Sym = GetBlockAddressSymbol(BA: MO.getBlockAddress());
1223 Sym->print(OS&: O, MAI);
1224 break;
1225 }
1226 }
1227}
1228
1229bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
1230 raw_ostream &O) {
1231 Register Reg = MO.getReg();
1232 switch (Mode) {
1233 default:
1234 return true; // Unknown mode.
1235 case 'w':
1236 Reg = getWRegFromXReg(Reg);
1237 break;
1238 case 'x':
1239 Reg = getXRegFromWReg(Reg);
1240 break;
1241 case 't':
1242 Reg = getXRegFromXRegTuple(RegTuple: Reg);
1243 break;
1244 }
1245
1246 O << AArch64InstPrinter::getRegisterName(Reg);
1247 return false;
1248}
1249
1250// Prints the register in MO using class RC using the offset in the
1251// new register class. This should not be used for cross class
1252// printing.
1253bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
1254 const TargetRegisterClass *RC,
1255 unsigned AltName, raw_ostream &O) {
1256 assert(MO.isReg() && "Should only get here with a register!");
1257 const TargetRegisterInfo *RI = STI->getRegisterInfo();
1258 Register Reg = MO.getReg();
1259 MCRegister RegToPrint = RC->getRegister(i: RI->getEncodingValue(Reg));
1260 if (!RI->regsOverlap(RegA: RegToPrint, RegB: Reg))
1261 return true;
1262 O << AArch64InstPrinter::getRegisterName(Reg: RegToPrint, AltIdx: AltName);
1263 return false;
1264}
1265
1266bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
1267 const char *ExtraCode, raw_ostream &O) {
1268 const MachineOperand &MO = MI->getOperand(i: OpNum);
1269
1270 // First try the generic code, which knows about modifiers like 'c' and 'n'.
1271 if (!AsmPrinter::PrintAsmOperand(MI, OpNo: OpNum, ExtraCode, OS&: O))
1272 return false;
1273
1274 // Does this asm operand have a single letter operand modifier?
1275 if (ExtraCode && ExtraCode[0]) {
1276 if (ExtraCode[1] != 0)
1277 return true; // Unknown modifier.
1278
1279 switch (ExtraCode[0]) {
1280 default:
1281 return true; // Unknown modifier.
1282 case 'w': // Print W register
1283 case 'x': // Print X register
1284 if (MO.isReg())
1285 return printAsmMRegister(MO, Mode: ExtraCode[0], O);
1286 if (MO.isImm() && MO.getImm() == 0) {
1287 unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
1288 O << AArch64InstPrinter::getRegisterName(Reg);
1289 return false;
1290 }
1291 printOperand(MI, OpNum, O);
1292 return false;
1293 case 'b': // Print B register.
1294 case 'h': // Print H register.
1295 case 's': // Print S register.
1296 case 'd': // Print D register.
1297 case 'q': // Print Q register.
1298 case 'z': // Print Z register.
1299 if (MO.isReg()) {
1300 const TargetRegisterClass *RC;
1301 switch (ExtraCode[0]) {
1302 case 'b':
1303 RC = &AArch64::FPR8RegClass;
1304 break;
1305 case 'h':
1306 RC = &AArch64::FPR16RegClass;
1307 break;
1308 case 's':
1309 RC = &AArch64::FPR32RegClass;
1310 break;
1311 case 'd':
1312 RC = &AArch64::FPR64RegClass;
1313 break;
1314 case 'q':
1315 RC = &AArch64::FPR128RegClass;
1316 break;
1317 case 'z':
1318 RC = &AArch64::ZPRRegClass;
1319 break;
1320 default:
1321 return true;
1322 }
1323 return printAsmRegInClass(MO, RC, AltName: AArch64::NoRegAltName, O);
1324 }
1325 printOperand(MI, OpNum, O);
1326 return false;
1327 }
1328 }
1329
1330 // According to ARM, we should emit x and v registers unless we have a
1331 // modifier.
1332 if (MO.isReg()) {
1333 Register Reg = MO.getReg();
1334
1335 // If this is a w or x register, print an x register.
1336 if (AArch64::GPR32allRegClass.contains(Reg) ||
1337 AArch64::GPR64allRegClass.contains(Reg))
1338 return printAsmMRegister(MO, Mode: 'x', O);
1339
1340 // If this is an x register tuple, print an x register.
1341 if (AArch64::GPR64x8ClassRegClass.contains(Reg))
1342 return printAsmMRegister(MO, Mode: 't', O);
1343
1344 unsigned AltName = AArch64::NoRegAltName;
1345 const TargetRegisterClass *RegClass;
1346 if (AArch64::ZPRRegClass.contains(Reg)) {
1347 RegClass = &AArch64::ZPRRegClass;
1348 } else if (AArch64::PPRRegClass.contains(Reg)) {
1349 RegClass = &AArch64::PPRRegClass;
1350 } else if (AArch64::PNRRegClass.contains(Reg)) {
1351 RegClass = &AArch64::PNRRegClass;
1352 } else {
1353 RegClass = &AArch64::FPR128RegClass;
1354 AltName = AArch64::vreg;
1355 }
1356
1357 // If this is a b, h, s, d, or q register, print it as a v register.
1358 return printAsmRegInClass(MO, RC: RegClass, AltName, O);
1359 }
1360
1361 printOperand(MI, OpNum, O);
1362 return false;
1363}
1364
1365bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
1366 unsigned OpNum,
1367 const char *ExtraCode,
1368 raw_ostream &O) {
1369 if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
1370 return true; // Unknown modifier.
1371
1372 const MachineOperand &MO = MI->getOperand(i: OpNum);
1373 assert(MO.isReg() && "unexpected inline asm memory operand");
1374 O << "[" << AArch64InstPrinter::getRegisterName(Reg: MO.getReg()) << "]";
1375 return false;
1376}
1377
1378void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
1379 raw_ostream &OS) {
1380 unsigned NOps = MI->getNumOperands();
1381 assert(NOps == 4);
1382 OS << '\t' << MAI.getCommentString() << "DEBUG_VALUE: ";
1383 // cast away const; DIetc do not take const operands for some reason.
1384 OS << MI->getDebugVariable()->getName();
1385 OS << " <- ";
1386 // Frame address. Currently handles register +- offset only.
1387 assert(MI->isIndirectDebugValue());
1388 OS << '[';
1389 for (unsigned I = 0, E = llvm::size(Range: MI->debug_operands()); I < E; ++I) {
1390 if (I != 0)
1391 OS << ", ";
1392 printOperand(MI, OpNum: I, O&: OS);
1393 }
1394 OS << ']';
1395 OS << "+";
1396 printOperand(MI, OpNum: NOps - 2, O&: OS);
1397}
1398
1399void AArch64AsmPrinter::emitJumpTableImpl(const MachineJumpTableInfo &MJTI,
1400 ArrayRef<unsigned> JumpTableIndices) {
1401 // Fast return if there is nothing to emit to avoid creating empty sections.
1402 if (JumpTableIndices.empty())
1403 return;
1404 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
1405 const auto &F = MF->getFunction();
1406 ArrayRef<MachineJumpTableEntry> JT = MJTI.getJumpTables();
1407
1408 MCSection *ReadOnlySec = nullptr;
1409 if (TM.Options.EnableStaticDataPartitioning) {
1410 ReadOnlySec =
1411 TLOF.getSectionForJumpTable(F, TM, JTE: &JT[JumpTableIndices.front()]);
1412 } else {
1413 ReadOnlySec = TLOF.getSectionForJumpTable(F, TM);
1414 }
1415 OutStreamer->switchSection(Section: ReadOnlySec);
1416
1417 auto AFI = MF->getInfo<AArch64FunctionInfo>();
1418 for (unsigned JTI : JumpTableIndices) {
1419 const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
1420
1421 // If this jump table was deleted, ignore it.
1422 if (JTBBs.empty()) continue;
1423
1424 unsigned Size = AFI->getJumpTableEntrySize(Idx: JTI);
1425 emitAlignment(Alignment: Align(Size));
1426 OutStreamer->emitLabel(Symbol: GetJTISymbol(JTID: JTI));
1427
1428 const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(Idx: JTI);
1429 const MCExpr *Base = MCSymbolRefExpr::create(Symbol: BaseSym, Ctx&: OutContext);
1430
1431 for (auto *JTBB : JTBBs) {
1432 const MCExpr *Value =
1433 MCSymbolRefExpr::create(Symbol: JTBB->getSymbol(), Ctx&: OutContext);
1434
1435 // Each entry is:
1436 // .byte/.hword (LBB - Lbase)>>2
1437 // or plain:
1438 // .word LBB - Lbase
1439 Value = MCBinaryExpr::createSub(LHS: Value, RHS: Base, Ctx&: OutContext);
1440 if (Size != 4)
1441 Value = MCBinaryExpr::createLShr(
1442 LHS: Value, RHS: MCConstantExpr::create(Value: 2, Ctx&: OutContext), Ctx&: OutContext);
1443
1444 OutStreamer->emitValue(Value, Size);
1445 }
1446 }
1447}
1448
1449std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
1450 codeview::JumpTableEntrySize>
1451AArch64AsmPrinter::getCodeViewJumpTableInfo(int JTI,
1452 const MachineInstr *BranchInstr,
1453 const MCSymbol *BranchLabel) const {
1454 const auto AFI = MF->getInfo<AArch64FunctionInfo>();
1455 const auto Base = AArch64FI->getJumpTableEntryPCRelSymbol(Idx: JTI);
1456 codeview::JumpTableEntrySize EntrySize;
1457 switch (AFI->getJumpTableEntrySize(Idx: JTI)) {
1458 case 1:
1459 EntrySize = codeview::JumpTableEntrySize::UInt8ShiftLeft;
1460 break;
1461 case 2:
1462 EntrySize = codeview::JumpTableEntrySize::UInt16ShiftLeft;
1463 break;
1464 case 4:
1465 EntrySize = codeview::JumpTableEntrySize::Int32;
1466 break;
1467 default:
1468 llvm_unreachable("Unexpected jump table entry size");
1469 }
1470 return std::make_tuple(args: Base, args: 0, args&: BranchLabel, args&: EntrySize);
1471}
1472
1473void AArch64AsmPrinter::emitFunctionEntryLabel() {
1474 const Triple &TT = TM.getTargetTriple();
1475 if (TT.isOSBinFormatELF() &&
1476 (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall ||
1477 MF->getFunction().getCallingConv() ==
1478 CallingConv::AArch64_SVE_VectorCall ||
1479 MF->getInfo<AArch64FunctionInfo>()->isSVECC())) {
1480 auto *TS =
1481 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
1482 TS->emitDirectiveVariantPCS(Symbol: CurrentFnSym);
1483 }
1484
1485 AsmPrinter::emitFunctionEntryLabel();
1486
1487 if (TT.isWindowsArm64EC() && !MF->getFunction().hasLocalLinkage()) {
1488 // For ARM64EC targets, a function definition's name is mangled differently
1489 // from the normal symbol, emit required aliases here.
1490 auto emitFunctionAlias = [&](MCSymbol *Src, MCSymbol *Dst) {
1491 OutStreamer->emitSymbolAttribute(Symbol: Src, Attribute: MCSA_WeakAntiDep);
1492 OutStreamer->emitAssignment(
1493 Symbol: Src, Value: MCSymbolRefExpr::create(Symbol: Dst, Ctx&: MMI->getContext()));
1494 };
1495
1496 auto getSymbolFromMetadata = [&](StringRef Name) {
1497 MCSymbol *Sym = nullptr;
1498 if (MDNode *Node = MF->getFunction().getMetadata(Kind: Name)) {
1499 StringRef NameStr = cast<MDString>(Val: Node->getOperand(I: 0))->getString();
1500 Sym = MMI->getContext().getOrCreateSymbol(Name: NameStr);
1501 }
1502 return Sym;
1503 };
1504
1505 SmallVector<MDNode *> UnmangledNames;
1506 MF->getFunction().getMetadata(Kind: "arm64ec_unmangled_name", MDs&: UnmangledNames);
1507 for (MDNode *Node : UnmangledNames) {
1508 StringRef NameStr = cast<MDString>(Val: Node->getOperand(I: 0))->getString();
1509 MCSymbol *UnmangledSym = MMI->getContext().getOrCreateSymbol(Name: NameStr);
1510 if (std::optional<std::string> MangledName =
1511 getArm64ECMangledFunctionName(Name: UnmangledSym->getName())) {
1512 MCSymbol *ECMangledSym =
1513 MMI->getContext().getOrCreateSymbol(Name: *MangledName);
1514 emitFunctionAlias(UnmangledSym, ECMangledSym);
1515 }
1516 }
1517 if (MCSymbol *ECMangledSym =
1518 getSymbolFromMetadata("arm64ec_ecmangled_name"))
1519 emitFunctionAlias(ECMangledSym, CurrentFnSym);
1520 }
1521}
1522
1523void AArch64AsmPrinter::emitXXStructor(const DataLayout &DL,
1524 const Constant *CV) {
1525 LLVMContext &C = CV->getContext();
1526 assert(!isa<ConstantPtrAuth>(CV) &&
1527 "ctors/dtors are to be signed by asm printer");
1528
1529 if (PtrauthInitFini) {
1530 IntegerType *Int32Ty = IntegerType::get(C, NumBits: 32);
1531 IntegerType *Int64Ty = IntegerType::get(C, NumBits: 64);
1532 PointerType *PtrTy = PointerType::get(C, AddressSpace: 0);
1533
1534 ConstantInt *Key = ConstantInt::get(Ty: Int32Ty, V: AArch64PAuth::InitFiniKey);
1535 ConstantInt *IntDisc = ConstantInt::get(
1536 Ty: Int64Ty, V: AArch64PAuth::InitFiniPointerConstantDiscriminator);
1537 Constant *Null = ConstantPointerNull::get(T: PtrTy);
1538 Constant *AddressDisc = Null;
1539 if (PtrauthInitFiniAddressDisc) {
1540 uint64_t Marker = ConstantPtrAuth::AddrDiscriminator_CtorsDtors;
1541 AddressDisc =
1542 ConstantExpr::getIntToPtr(C: ConstantInt::get(Ty: Int64Ty, V: Marker), Ty: PtrTy);
1543 }
1544
1545 CV = ConstantPtrAuth::get(Ptr: const_cast<Constant *>(CV), Key, Disc: IntDisc,
1546 AddrDisc: AddressDisc, /*DeactivationSymbol=*/Null);
1547 }
1548
1549 // Signed pointers will be lowered by AArch64AsmPrinter::lowerConstantPtrAuth.
1550 AsmPrinter::emitXXStructor(DL, CV);
1551}
1552
1553void AArch64AsmPrinter::emitGlobalAlias(const Module &M,
1554 const GlobalAlias &GA) {
1555 if (auto F = dyn_cast_or_null<Function>(Val: GA.getAliasee())) {
1556 // Global aliases must point to a definition, but unmangled patchable
1557 // symbols are special and need to point to an undefined symbol with "EXP+"
1558 // prefix. Such undefined symbol is resolved by the linker by creating
1559 // x86 thunk that jumps back to the actual EC target.
1560 if (MDNode *Node = F->getMetadata(Kind: "arm64ec_exp_name")) {
1561 StringRef ExpStr = cast<MDString>(Val: Node->getOperand(I: 0))->getString();
1562 MCSymbol *ExpSym = MMI->getContext().getOrCreateSymbol(Name: ExpStr);
1563 MCSymbol *Sym = MMI->getContext().getOrCreateSymbol(Name: GA.getName());
1564
1565 OutStreamer->beginCOFFSymbolDef(Symbol: ExpSym);
1566 OutStreamer->emitCOFFSymbolStorageClass(StorageClass: COFF::IMAGE_SYM_CLASS_EXTERNAL);
1567 OutStreamer->emitCOFFSymbolType(Type: COFF::IMAGE_SYM_DTYPE_FUNCTION
1568 << COFF::SCT_COMPLEX_TYPE_SHIFT);
1569 OutStreamer->endCOFFSymbolDef();
1570
1571 OutStreamer->beginCOFFSymbolDef(Symbol: Sym);
1572 OutStreamer->emitCOFFSymbolStorageClass(StorageClass: COFF::IMAGE_SYM_CLASS_EXTERNAL);
1573 OutStreamer->emitCOFFSymbolType(Type: COFF::IMAGE_SYM_DTYPE_FUNCTION
1574 << COFF::SCT_COMPLEX_TYPE_SHIFT);
1575 OutStreamer->endCOFFSymbolDef();
1576 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Weak);
1577 OutStreamer->emitAssignment(
1578 Symbol: Sym, Value: MCSymbolRefExpr::create(Symbol: ExpSym, Ctx&: MMI->getContext()));
1579 return;
1580 }
1581 }
1582 AsmPrinter::emitGlobalAlias(M, GA);
1583}
1584
1585/// Small jump tables contain an unsigned byte or half, representing the offset
1586/// from the lowest-addressed possible destination to the desired basic
1587/// block. Since all instructions are 4-byte aligned, this is further compressed
1588/// by counting in instructions rather than bytes (i.e. divided by 4). So, to
1589/// materialize the correct destination we need:
1590///
1591/// adr xDest, .LBB0_0
1592/// ldrb wScratch, [xTable, xEntry] (with "lsl #1" for ldrh).
1593/// add xDest, xDest, xScratch (with "lsl #2" for smaller entries)
1594void AArch64AsmPrinter::LowerJumpTableDest(llvm::MCStreamer &OutStreamer,
1595 const llvm::MachineInstr &MI) {
1596 Register DestReg = MI.getOperand(i: 0).getReg();
1597 Register ScratchReg = MI.getOperand(i: 1).getReg();
1598 Register ScratchRegW =
1599 STI->getRegisterInfo()->getSubReg(Reg: ScratchReg, Idx: AArch64::sub_32);
1600 Register TableReg = MI.getOperand(i: 2).getReg();
1601 Register EntryReg = MI.getOperand(i: 3).getReg();
1602 int JTIdx = MI.getOperand(i: 4).getIndex();
1603 int Size = AArch64FI->getJumpTableEntrySize(Idx: JTIdx);
1604
1605 // This has to be first because the compression pass based its reachability
1606 // calculations on the start of the JumpTableDest instruction.
1607 auto Label =
1608 MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(Idx: JTIdx);
1609
1610 // If we don't already have a symbol to use as the base, use the ADR
1611 // instruction itself.
1612 if (!Label) {
1613 Label = MF->getContext().createTempSymbol();
1614 AArch64FI->setJumpTableEntryInfo(Idx: JTIdx, Size, PCRelSym: Label);
1615 OutStreamer.emitLabel(Symbol: Label);
1616 }
1617
1618 auto LabelExpr = MCSymbolRefExpr::create(Symbol: Label, Ctx&: MF->getContext());
1619 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::ADR)
1620 .addReg(Reg: DestReg)
1621 .addExpr(Val: LabelExpr));
1622
1623 // Load the number of instruction-steps to offset from the label.
1624 unsigned LdrOpcode;
1625 switch (Size) {
1626 case 1: LdrOpcode = AArch64::LDRBBroX; break;
1627 case 2: LdrOpcode = AArch64::LDRHHroX; break;
1628 case 4: LdrOpcode = AArch64::LDRSWroX; break;
1629 default:
1630 llvm_unreachable("Unknown jump table size");
1631 }
1632
1633 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(LdrOpcode)
1634 .addReg(Reg: Size == 4 ? ScratchReg : ScratchRegW)
1635 .addReg(Reg: TableReg)
1636 .addReg(Reg: EntryReg)
1637 .addImm(Val: 0)
1638 .addImm(Val: Size == 1 ? 0 : 1));
1639
1640 // Add to the already materialized base label address, multiplying by 4 if
1641 // compressed.
1642 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::ADDXrs)
1643 .addReg(Reg: DestReg)
1644 .addReg(Reg: DestReg)
1645 .addReg(Reg: ScratchReg)
1646 .addImm(Val: Size == 4 ? 0 : 2));
1647}
1648
1649void AArch64AsmPrinter::LowerHardenedBRJumpTable(const MachineInstr &MI) {
1650 const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
1651 assert(MJTI && "Can't lower jump-table dispatch without JTI");
1652
1653 const std::vector<MachineJumpTableEntry> &JTs = MJTI->getJumpTables();
1654 assert(!JTs.empty() && "Invalid JT index for jump-table dispatch");
1655
1656 // Emit:
1657 // mov x17, #<size of table> ; depending on table size, with MOVKs
1658 // cmp x16, x17 ; or #imm if table size fits in 12-bit
1659 // csel x16, x16, xzr, ls ; check for index overflow
1660 //
1661 // adrp x17, Ltable@PAGE ; materialize table address
1662 // add x17, Ltable@PAGEOFF
1663 // ldrsw x16, [x17, x16, lsl #2] ; load table entry
1664 //
1665 // Lanchor:
1666 // adr x17, Lanchor ; compute target address
1667 // add x16, x17, x16
1668 // br x16 ; branch to target
1669
1670 MachineOperand JTOp = MI.getOperand(i: 0);
1671
1672 unsigned JTI = JTOp.getIndex();
1673 assert(!AArch64FI->getJumpTableEntryPCRelSymbol(JTI) &&
1674 "unsupported compressed jump table");
1675
1676 const uint64_t NumTableEntries = JTs[JTI].MBBs.size();
1677
1678 // cmp only supports a 12-bit immediate. If we need more, materialize the
1679 // immediate, using x17 as a scratch register.
1680 uint64_t MaxTableEntry = NumTableEntries - 1;
1681 if (isUInt<12>(x: MaxTableEntry)) {
1682 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::SUBSXri)
1683 .addReg(Reg: AArch64::XZR)
1684 .addReg(Reg: AArch64::X16)
1685 .addImm(Val: MaxTableEntry)
1686 .addImm(Val: 0));
1687 } else {
1688 emitMOVZ(Dest: AArch64::X17, Imm: static_cast<uint16_t>(MaxTableEntry), Shift: 0);
1689 // It's sad that we have to manually materialize instructions, but we can't
1690 // trivially reuse the main pseudo expansion logic.
1691 // A MOVK sequence is easy enough to generate and handles the general case.
1692 for (int Offset = 16; Offset < 64; Offset += 16) {
1693 if ((MaxTableEntry >> Offset) == 0)
1694 break;
1695 emitMOVK(Dest: AArch64::X17, Imm: static_cast<uint16_t>(MaxTableEntry >> Offset),
1696 Shift: Offset);
1697 }
1698 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::SUBSXrs)
1699 .addReg(Reg: AArch64::XZR)
1700 .addReg(Reg: AArch64::X16)
1701 .addReg(Reg: AArch64::X17)
1702 .addImm(Val: 0));
1703 }
1704
1705 // This picks entry #0 on failure.
1706 // We might want to trap instead.
1707 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::CSELXr)
1708 .addReg(Reg: AArch64::X16)
1709 .addReg(Reg: AArch64::X16)
1710 .addReg(Reg: AArch64::XZR)
1711 .addImm(Val: AArch64CC::LS));
1712
1713 // Prepare the @PAGE/@PAGEOFF low/high operands.
1714 MachineOperand JTMOHi(JTOp), JTMOLo(JTOp);
1715 MCOperand JTMCHi, JTMCLo;
1716
1717 JTMOHi.setTargetFlags(AArch64II::MO_PAGE);
1718 JTMOLo.setTargetFlags(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
1719
1720 MCInstLowering.lowerOperand(MO: JTMOHi, MCOp&: JTMCHi);
1721 MCInstLowering.lowerOperand(MO: JTMOLo, MCOp&: JTMCLo);
1722
1723 EmitToStreamer(
1724 S&: *OutStreamer,
1725 Inst: MCInstBuilder(AArch64::ADRP).addReg(Reg: AArch64::X17).addOperand(Op: JTMCHi));
1726
1727 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ADDXri)
1728 .addReg(Reg: AArch64::X17)
1729 .addReg(Reg: AArch64::X17)
1730 .addOperand(Op: JTMCLo)
1731 .addImm(Val: 0));
1732
1733 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::LDRSWroX)
1734 .addReg(Reg: AArch64::X16)
1735 .addReg(Reg: AArch64::X17)
1736 .addReg(Reg: AArch64::X16)
1737 .addImm(Val: 0)
1738 .addImm(Val: 1));
1739
1740 MCSymbol *AdrLabel = MF->getContext().createTempSymbol();
1741 const auto *AdrLabelE = MCSymbolRefExpr::create(Symbol: AdrLabel, Ctx&: MF->getContext());
1742 AArch64FI->setJumpTableEntryInfo(Idx: JTI, Size: 4, PCRelSym: AdrLabel);
1743
1744 OutStreamer->emitLabel(Symbol: AdrLabel);
1745 EmitToStreamer(
1746 S&: *OutStreamer,
1747 Inst: MCInstBuilder(AArch64::ADR).addReg(Reg: AArch64::X17).addExpr(Val: AdrLabelE));
1748
1749 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ADDXrs)
1750 .addReg(Reg: AArch64::X16)
1751 .addReg(Reg: AArch64::X17)
1752 .addReg(Reg: AArch64::X16)
1753 .addImm(Val: 0));
1754
1755 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::BR).addReg(Reg: AArch64::X16));
1756}
1757
1758void AArch64AsmPrinter::LowerMOPS(llvm::MCStreamer &OutStreamer,
1759 const llvm::MachineInstr &MI) {
1760 unsigned Opcode = MI.getOpcode();
1761 assert(STI->hasMOPS());
1762 assert(STI->hasMTE() || Opcode != AArch64::MOPSMemorySetTaggingPseudo);
1763
1764 const auto Ops = [Opcode]() -> std::array<unsigned, 3> {
1765 if (Opcode == AArch64::MOPSMemoryCopyPseudo)
1766 return {AArch64::CPYFP, AArch64::CPYFM, AArch64::CPYFE};
1767 if (Opcode == AArch64::MOPSMemoryMovePseudo)
1768 return {AArch64::CPYP, AArch64::CPYM, AArch64::CPYE};
1769 if (Opcode == AArch64::MOPSMemorySetPseudo)
1770 return {AArch64::SETP, AArch64::SETM, AArch64::SETE};
1771 if (Opcode == AArch64::MOPSMemorySetTaggingPseudo)
1772 return {AArch64::SETGP, AArch64::SETGM, AArch64::MOPSSETGE};
1773 llvm_unreachable("Unhandled memory operation pseudo");
1774 }();
1775 const bool IsSet = Opcode == AArch64::MOPSMemorySetPseudo ||
1776 Opcode == AArch64::MOPSMemorySetTaggingPseudo;
1777
1778 for (auto Op : Ops) {
1779 int i = 0;
1780 auto MCIB = MCInstBuilder(Op);
1781 // Destination registers
1782 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1783 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1784 if (!IsSet)
1785 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1786 // Input registers
1787 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1788 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1789 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1790
1791 EmitToStreamer(S&: OutStreamer, Inst: MCIB);
1792 }
1793}
1794
1795void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
1796 const MachineInstr &MI) {
1797 unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
1798
1799 auto &Ctx = OutStreamer.getContext();
1800 MCSymbol *MILabel = Ctx.createTempSymbol();
1801 OutStreamer.emitLabel(Symbol: MILabel);
1802
1803 SM.recordStackMap(L: *MILabel, MI);
1804 assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1805
1806 // Scan ahead to trim the shadow.
1807 const MachineBasicBlock &MBB = *MI.getParent();
1808 MachineBasicBlock::const_iterator MII(MI);
1809 ++MII;
1810 while (NumNOPBytes > 0) {
1811 if (MII == MBB.end() || MII->isCall() ||
1812 MII->getOpcode() == AArch64::DBG_VALUE ||
1813 MII->getOpcode() == TargetOpcode::PATCHPOINT ||
1814 MII->getOpcode() == TargetOpcode::STACKMAP)
1815 break;
1816 ++MII;
1817 NumNOPBytes -= 4;
1818 }
1819
1820 // Emit nops.
1821 for (unsigned i = 0; i < NumNOPBytes; i += 4)
1822 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::NOP));
1823}
1824
1825// Lower a patchpoint of the form:
1826// [<def>], <id>, <numBytes>, <target>, <numArgs>
1827void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1828 const MachineInstr &MI) {
1829 auto &Ctx = OutStreamer.getContext();
1830 MCSymbol *MILabel = Ctx.createTempSymbol();
1831 OutStreamer.emitLabel(Symbol: MILabel);
1832 SM.recordPatchPoint(L: *MILabel, MI);
1833
1834 PatchPointOpers Opers(&MI);
1835
1836 int64_t CallTarget = Opers.getCallTarget().getImm();
1837 unsigned EncodedBytes = 0;
1838 if (CallTarget) {
1839 assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
1840 "High 16 bits of call target should be zero.");
1841 Register ScratchReg = MI.getOperand(i: Opers.getNextScratchIdx()).getReg();
1842 EncodedBytes = 16;
1843 // Materialize the jump address:
1844 emitMOVZ(Dest: ScratchReg, Imm: (CallTarget >> 32) & 0xFFFF, Shift: 32);
1845 emitMOVK(Dest: ScratchReg, Imm: (CallTarget >> 16) & 0xFFFF, Shift: 16);
1846 emitMOVK(Dest: ScratchReg, Imm: CallTarget & 0xFFFF, Shift: 0);
1847 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::BLR).addReg(Reg: ScratchReg));
1848 }
1849 // Emit padding.
1850 unsigned NumBytes = Opers.getNumPatchBytes();
1851 assert(NumBytes >= EncodedBytes &&
1852 "Patchpoint can't request size less than the length of a call.");
1853 assert((NumBytes - EncodedBytes) % 4 == 0 &&
1854 "Invalid number of NOP bytes requested!");
1855 for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
1856 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::NOP));
1857}
1858
1859void AArch64AsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1860 const MachineInstr &MI) {
1861 StatepointOpers SOpers(&MI);
1862 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
1863 assert(PatchBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1864 for (unsigned i = 0; i < PatchBytes; i += 4)
1865 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::NOP));
1866 } else {
1867 // Lower call target and choose correct opcode
1868 const MachineOperand &CallTarget = SOpers.getCallTarget();
1869 MCOperand CallTargetMCOp;
1870 unsigned CallOpcode;
1871 switch (CallTarget.getType()) {
1872 case MachineOperand::MO_GlobalAddress:
1873 case MachineOperand::MO_ExternalSymbol:
1874 MCInstLowering.lowerOperand(MO: CallTarget, MCOp&: CallTargetMCOp);
1875 CallOpcode = AArch64::BL;
1876 break;
1877 case MachineOperand::MO_Immediate:
1878 CallTargetMCOp = MCOperand::createImm(Val: CallTarget.getImm());
1879 CallOpcode = AArch64::BL;
1880 break;
1881 case MachineOperand::MO_Register:
1882 CallTargetMCOp = MCOperand::createReg(Reg: CallTarget.getReg());
1883 CallOpcode = AArch64::BLR;
1884 break;
1885 default:
1886 llvm_unreachable("Unsupported operand type in statepoint call target");
1887 break;
1888 }
1889
1890 EmitToStreamer(S&: OutStreamer,
1891 Inst: MCInstBuilder(CallOpcode).addOperand(Op: CallTargetMCOp));
1892 }
1893
1894 auto &Ctx = OutStreamer.getContext();
1895 MCSymbol *MILabel = Ctx.createTempSymbol();
1896 OutStreamer.emitLabel(Symbol: MILabel);
1897 SM.recordStatepoint(L: *MILabel, MI);
1898}
1899
1900void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) {
1901 // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
1902 // <opcode>, <operands>
1903
1904 Register DefRegister = FaultingMI.getOperand(i: 0).getReg();
1905 FaultMaps::FaultKind FK =
1906 static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(i: 1).getImm());
1907 MCSymbol *HandlerLabel = FaultingMI.getOperand(i: 2).getMBB()->getSymbol();
1908 unsigned Opcode = FaultingMI.getOperand(i: 3).getImm();
1909 unsigned OperandsBeginIdx = 4;
1910
1911 auto &Ctx = OutStreamer->getContext();
1912 MCSymbol *FaultingLabel = Ctx.createTempSymbol();
1913 OutStreamer->emitLabel(Symbol: FaultingLabel);
1914
1915 assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
1916 FM.recordFaultingOp(FaultTy: FK, FaultingLabel, HandlerLabel);
1917
1918 MCInst MI;
1919 MI.setOpcode(Opcode);
1920
1921 if (DefRegister != (Register)0)
1922 MI.addOperand(Op: MCOperand::createReg(Reg: DefRegister));
1923
1924 for (const MachineOperand &MO :
1925 llvm::drop_begin(RangeOrContainer: FaultingMI.operands(), N: OperandsBeginIdx)) {
1926 MCOperand Dest;
1927 lowerOperand(MO, MCOp&: Dest);
1928 MI.addOperand(Op: Dest);
1929 }
1930
1931 OutStreamer->AddComment(T: "on-fault: " + HandlerLabel->getName());
1932 EmitToStreamer(Inst: MI);
1933}
1934
1935void AArch64AsmPrinter::emitMovXReg(Register Dest, Register Src) {
1936 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ORRXrs)
1937 .addReg(Reg: Dest)
1938 .addReg(Reg: AArch64::XZR)
1939 .addReg(Reg: Src)
1940 .addImm(Val: 0));
1941}
1942
1943void AArch64AsmPrinter::emitMOVZ(Register Dest, uint64_t Imm, unsigned Shift) {
1944 bool Is64Bit = AArch64::GPR64RegClass.contains(Reg: Dest);
1945 EmitToStreamer(S&: *OutStreamer,
1946 Inst: MCInstBuilder(Is64Bit ? AArch64::MOVZXi : AArch64::MOVZWi)
1947 .addReg(Reg: Dest)
1948 .addImm(Val: Imm)
1949 .addImm(Val: Shift));
1950}
1951
1952void AArch64AsmPrinter::emitMOVK(Register Dest, uint64_t Imm, unsigned Shift) {
1953 bool Is64Bit = AArch64::GPR64RegClass.contains(Reg: Dest);
1954 EmitToStreamer(S&: *OutStreamer,
1955 Inst: MCInstBuilder(Is64Bit ? AArch64::MOVKXi : AArch64::MOVKWi)
1956 .addReg(Reg: Dest)
1957 .addReg(Reg: Dest)
1958 .addImm(Val: Imm)
1959 .addImm(Val: Shift));
1960}
1961
1962void AArch64AsmPrinter::emitAUT(AArch64PACKey::ID Key, Register Pointer,
1963 Register Disc) {
1964 bool IsZeroDisc = Disc == AArch64::XZR;
1965 unsigned Opcode = getAUTOpcodeForKey(K: Key, Zero: IsZeroDisc);
1966
1967 // autiza x16 ; if IsZeroDisc
1968 // autia x16, x17 ; if !IsZeroDisc
1969 MCInst AUTInst;
1970 AUTInst.setOpcode(Opcode);
1971 AUTInst.addOperand(Op: MCOperand::createReg(Reg: Pointer));
1972 AUTInst.addOperand(Op: MCOperand::createReg(Reg: Pointer));
1973 if (!IsZeroDisc)
1974 AUTInst.addOperand(Op: MCOperand::createReg(Reg: Disc));
1975
1976 EmitToStreamer(Inst: AUTInst);
1977}
1978
1979void AArch64AsmPrinter::emitPAC(AArch64PACKey::ID Key, Register Pointer,
1980 Register Disc) {
1981 bool IsZeroDisc = Disc == AArch64::XZR;
1982 unsigned Opcode = getPACOpcodeForKey(K: Key, Zero: IsZeroDisc);
1983
1984 // paciza x16 ; if IsZeroDisc
1985 // pacia x16, x17 ; if !IsZeroDisc
1986 MCInst PACInst;
1987 PACInst.setOpcode(Opcode);
1988 PACInst.addOperand(Op: MCOperand::createReg(Reg: Pointer));
1989 PACInst.addOperand(Op: MCOperand::createReg(Reg: Pointer));
1990 if (!IsZeroDisc)
1991 PACInst.addOperand(Op: MCOperand::createReg(Reg: Disc));
1992
1993 EmitToStreamer(Inst: PACInst);
1994}
1995
1996void AArch64AsmPrinter::emitBLRA(bool IsCall, AArch64PACKey::ID Key,
1997 Register Target, Register Disc) {
1998 bool IsZeroDisc = Disc == AArch64::XZR;
1999 unsigned Opcode = getBranchOpcodeForKey(IsCall, K: Key, Zero: IsZeroDisc);
2000
2001 // blraaz x16 ; if IsZeroDisc
2002 // blraa x16, x17 ; if !IsZeroDisc
2003 MCInst Inst;
2004 Inst.setOpcode(Opcode);
2005 Inst.addOperand(Op: MCOperand::createReg(Reg: Target));
2006 if (!IsZeroDisc)
2007 Inst.addOperand(Op: MCOperand::createReg(Reg: Disc));
2008 EmitToStreamer(Inst);
2009}
2010
2011void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) {
2012 Register DestReg = MI.getOperand(i: 0).getReg();
2013 if (!STI->hasZeroCycleZeroingFPWorkaround() && STI->isNeonAvailable()) {
2014 if (STI->hasZeroCycleZeroingFPR64()) {
2015 // Convert H/S register to corresponding D register
2016 const AArch64RegisterInfo *TRI = STI->getRegisterInfo();
2017 if (AArch64::FPR16RegClass.contains(Reg: DestReg))
2018 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
2019 RC: &AArch64::FPR64RegClass);
2020 else if (AArch64::FPR32RegClass.contains(Reg: DestReg))
2021 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::ssub,
2022 RC: &AArch64::FPR64RegClass);
2023 else
2024 assert(AArch64::FPR64RegClass.contains(DestReg));
2025
2026 MCInst MOVI;
2027 MOVI.setOpcode(AArch64::MOVID);
2028 MOVI.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2029 MOVI.addOperand(Op: MCOperand::createImm(Val: 0));
2030 EmitToStreamer(S&: *OutStreamer, Inst: MOVI);
2031 ++NumZCZeroingInstrsFPR;
2032 } else if (STI->hasZeroCycleZeroingFPR128()) {
2033 // Convert H/S/D register to corresponding Q register
2034 const AArch64RegisterInfo *TRI = STI->getRegisterInfo();
2035 if (AArch64::FPR16RegClass.contains(Reg: DestReg)) {
2036 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
2037 RC: &AArch64::FPR128RegClass);
2038 } else if (AArch64::FPR32RegClass.contains(Reg: DestReg)) {
2039 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::ssub,
2040 RC: &AArch64::FPR128RegClass);
2041 } else {
2042 assert(AArch64::FPR64RegClass.contains(DestReg));
2043 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::dsub,
2044 RC: &AArch64::FPR128RegClass);
2045 }
2046
2047 MCInst MOVI;
2048 MOVI.setOpcode(AArch64::MOVIv2d_ns);
2049 MOVI.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2050 MOVI.addOperand(Op: MCOperand::createImm(Val: 0));
2051 EmitToStreamer(S&: *OutStreamer, Inst: MOVI);
2052 ++NumZCZeroingInstrsFPR;
2053 } else {
2054 emitFMov0AsFMov(MI, DestReg);
2055 }
2056 } else {
2057 emitFMov0AsFMov(MI, DestReg);
2058 }
2059}
2060
2061void AArch64AsmPrinter::emitFMov0AsFMov(const MachineInstr &MI,
2062 Register DestReg) {
2063 MCInst FMov;
2064 switch (MI.getOpcode()) {
2065 default:
2066 llvm_unreachable("Unexpected opcode");
2067 case AArch64::FMOVH0:
2068 FMov.setOpcode(STI->hasFullFP16() ? AArch64::FMOVWHr : AArch64::FMOVWSr);
2069 if (!STI->hasFullFP16())
2070 DestReg = (AArch64::S0 + (DestReg - AArch64::H0));
2071 FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2072 FMov.addOperand(Op: MCOperand::createReg(Reg: AArch64::WZR));
2073 break;
2074 case AArch64::FMOVS0:
2075 FMov.setOpcode(AArch64::FMOVWSr);
2076 FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2077 FMov.addOperand(Op: MCOperand::createReg(Reg: AArch64::WZR));
2078 break;
2079 case AArch64::FMOVD0:
2080 FMov.setOpcode(AArch64::FMOVXDr);
2081 FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2082 FMov.addOperand(Op: MCOperand::createReg(Reg: AArch64::XZR));
2083 break;
2084 }
2085 EmitToStreamer(S&: *OutStreamer, Inst: FMov);
2086}
2087
2088Register AArch64AsmPrinter::emitPtrauthDiscriminator(uint64_t Disc,
2089 Register AddrDisc,
2090 Register ScratchReg,
2091 bool MayClobberAddrDisc) {
2092 assert(isPtrauthRegSafe(ScratchReg) &&
2093 "Safe scratch register must be provided by the caller");
2094 assert(isUInt<16>(Disc) && "Constant discriminator is too wide");
2095
2096 // So far we've used NoRegister in pseudos. Now we need real encodings.
2097 if (!AddrDisc.isValid())
2098 AddrDisc = AArch64::XZR;
2099
2100 // If there is no constant discriminator, there's no blend involved:
2101 // just use the address discriminator register as-is (XZR or not).
2102 if (!Disc)
2103 return AddrDisc;
2104
2105 // If there's only a constant discriminator, MOV it into the scratch register.
2106 if (AddrDisc == AArch64::XZR) {
2107 emitMOVZ(Dest: ScratchReg, Imm: Disc, Shift: 0);
2108 return ScratchReg;
2109 }
2110
2111 // If there are both, emit a blend into the scratch register.
2112
2113 // Check if we can save one MOV instruction.
2114 if (MayClobberAddrDisc && isPtrauthRegSafe(Reg: AddrDisc)) {
2115 ScratchReg = AddrDisc;
2116 } else {
2117 emitMovXReg(Dest: ScratchReg, Src: AddrDisc);
2118 assert(ScratchReg != AddrDisc &&
2119 "Forbidden to clobber AddrDisc, but have to");
2120 }
2121
2122 emitMOVK(Dest: ScratchReg, Imm: Disc, Shift: 48);
2123 return ScratchReg;
2124}
2125
2126/// Emit a code sequence to check an authenticated pointer value.
2127///
2128/// This function emits a sequence of instructions that checks if TestedReg was
2129/// authenticated successfully. On success, execution continues at the next
2130/// instruction after the sequence.
2131///
2132/// The action performed on failure depends on the OnFailure argument:
2133/// * if OnFailure is not nullptr, control is transferred to that label after
2134/// clearing the PAC field
2135/// * otherwise, BRK instruction is emitted to generate an error
2136void AArch64AsmPrinter::emitPtrauthCheckAuthenticatedValue(
2137 Register TestedReg, Register ScratchReg, AArch64PACKey::ID Key,
2138 AArch64PAuth::AuthCheckMethod Method, const MCSymbol *OnFailure) {
2139 // Insert a sequence to check if authentication of TestedReg succeeded,
2140 // such as:
2141 //
2142 // - checked and clearing:
2143 // ; x16 is TestedReg, x17 is ScratchReg
2144 // mov x17, x16
2145 // xpaci x17
2146 // cmp x16, x17
2147 // b.eq Lsuccess
2148 // mov x16, x17
2149 // b Lend
2150 // Lsuccess:
2151 // ; skipped if authentication failed
2152 // Lend:
2153 // ...
2154 //
2155 // - checked and trapping:
2156 // mov x17, x16
2157 // xpaci x17
2158 // cmp x16, x17
2159 // b.eq Lsuccess
2160 // brk #<0xc470 + aut key>
2161 // Lsuccess:
2162 // ...
2163 //
2164 // See the documentation on AuthCheckMethod enumeration constants for
2165 // the specific code sequences that can be used to perform the check.
2166 using AArch64PAuth::AuthCheckMethod;
2167
2168 if (Method == AuthCheckMethod::None)
2169 return;
2170 if (Method == AuthCheckMethod::DummyLoad) {
2171 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRWui)
2172 .addReg(Reg: getWRegFromXReg(Reg: ScratchReg))
2173 .addReg(Reg: TestedReg)
2174 .addImm(Val: 0));
2175 assert(!OnFailure && "DummyLoad always traps on error");
2176 return;
2177 }
2178
2179 MCSymbol *SuccessSym = createTempSymbol(Name: "auth_success_");
2180 if (Method == AuthCheckMethod::XPAC || Method == AuthCheckMethod::XPACHint) {
2181 // mov Xscratch, Xtested
2182 emitMovXReg(Dest: ScratchReg, Src: TestedReg);
2183
2184 if (Method == AuthCheckMethod::XPAC) {
2185 // xpac(i|d) Xscratch
2186 unsigned XPACOpc = getXPACOpcodeForKey(K: Key);
2187 EmitToStreamer(
2188 Inst: MCInstBuilder(XPACOpc).addReg(Reg: ScratchReg).addReg(Reg: ScratchReg));
2189 } else {
2190 // xpaclri
2191
2192 // Note that this method applies XPAC to TestedReg instead of ScratchReg.
2193 assert(TestedReg == AArch64::LR &&
2194 "XPACHint mode is only compatible with checking the LR register");
2195 assert((Key == AArch64PACKey::IA || Key == AArch64PACKey::IB) &&
2196 "XPACHint mode is only compatible with I-keys");
2197 EmitToStreamer(Inst: MCInstBuilder(AArch64::XPACLRI));
2198 }
2199
2200 // cmp Xtested, Xscratch
2201 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSXrs)
2202 .addReg(Reg: AArch64::XZR)
2203 .addReg(Reg: TestedReg)
2204 .addReg(Reg: ScratchReg)
2205 .addImm(Val: 0));
2206
2207 // b.eq Lsuccess
2208 EmitToStreamer(
2209 Inst: MCInstBuilder(AArch64::Bcc)
2210 .addImm(Val: AArch64CC::EQ)
2211 .addExpr(Val: MCSymbolRefExpr::create(Symbol: SuccessSym, Ctx&: OutContext)));
2212 } else if (Method == AuthCheckMethod::HighBitsNoTBI) {
2213 // eor Xscratch, Xtested, Xtested, lsl #1
2214 EmitToStreamer(Inst: MCInstBuilder(AArch64::EORXrs)
2215 .addReg(Reg: ScratchReg)
2216 .addReg(Reg: TestedReg)
2217 .addReg(Reg: TestedReg)
2218 .addImm(Val: 1));
2219 // tbz Xscratch, #62, Lsuccess
2220 EmitToStreamer(
2221 Inst: MCInstBuilder(AArch64::TBZX)
2222 .addReg(Reg: ScratchReg)
2223 .addImm(Val: 62)
2224 .addExpr(Val: MCSymbolRefExpr::create(Symbol: SuccessSym, Ctx&: OutContext)));
2225 } else {
2226 llvm_unreachable("Unsupported check method");
2227 }
2228
2229 if (!OnFailure) {
2230 // Trapping sequences do a 'brk'.
2231 // brk #<0xc470 + aut key>
2232 EmitToStreamer(Inst: MCInstBuilder(AArch64::BRK).addImm(Val: 0xc470 | Key));
2233 } else {
2234 // Non-trapping checked sequences return the stripped result in TestedReg,
2235 // skipping over success-only code (such as re-signing the pointer) by
2236 // jumping to OnFailure label.
2237 // Note that this can introduce an authentication oracle (such as based on
2238 // the high bits of the re-signed value).
2239
2240 // FIXME: The XPAC method can be optimized by applying XPAC to TestedReg
2241 // instead of ScratchReg, thus eliminating one `mov` instruction.
2242 // Both XPAC and XPACHint can be further optimized by not using a
2243 // conditional branch jumping over an unconditional one.
2244
2245 switch (Method) {
2246 case AuthCheckMethod::XPACHint:
2247 // LR is already XPAC-ed at this point.
2248 break;
2249 case AuthCheckMethod::XPAC:
2250 // mov Xtested, Xscratch
2251 emitMovXReg(Dest: TestedReg, Src: ScratchReg);
2252 break;
2253 default:
2254 // If Xtested was not XPAC-ed so far, emit XPAC here.
2255 // xpac(i|d) Xtested
2256 unsigned XPACOpc = getXPACOpcodeForKey(K: Key);
2257 EmitToStreamer(
2258 Inst: MCInstBuilder(XPACOpc).addReg(Reg: TestedReg).addReg(Reg: TestedReg));
2259 }
2260
2261 // b Lend
2262 const auto *OnFailureExpr = MCSymbolRefExpr::create(Symbol: OnFailure, Ctx&: OutContext);
2263 EmitToStreamer(Inst: MCInstBuilder(AArch64::B).addExpr(Val: OnFailureExpr));
2264 }
2265
2266 // If the auth check succeeds, we can continue.
2267 // Lsuccess:
2268 OutStreamer->emitLabel(Symbol: SuccessSym);
2269}
2270
2271// With Pointer Authentication, it may be needed to explicitly check the
2272// authenticated value in LR before performing a tail call.
2273// Otherwise, the callee may re-sign the invalid return address,
2274// introducing a signing oracle.
2275void AArch64AsmPrinter::emitPtrauthTailCallHardening(const MachineInstr *TC) {
2276 if (!AArch64FI->shouldSignReturnAddress(MF: *MF))
2277 return;
2278
2279 auto LRCheckMethod = STI->getAuthenticatedLRCheckMethod(MF: *MF);
2280 if (LRCheckMethod == AArch64PAuth::AuthCheckMethod::None)
2281 return;
2282
2283 const AArch64RegisterInfo *TRI = STI->getRegisterInfo();
2284 Register ScratchReg =
2285 TC->readsRegister(Reg: AArch64::X16, TRI) ? AArch64::X17 : AArch64::X16;
2286 assert(!TC->readsRegister(ScratchReg, TRI) &&
2287 "Neither x16 nor x17 is available as a scratch register");
2288 AArch64PACKey::ID Key =
2289 AArch64FI->shouldSignWithBKey() ? AArch64PACKey::IB : AArch64PACKey::IA;
2290 emitPtrauthCheckAuthenticatedValue(TestedReg: AArch64::LR, ScratchReg, Key,
2291 Method: LRCheckMethod);
2292}
2293
2294bool AArch64AsmPrinter::emitDeactivationSymbolRelocation(Value *DS) {
2295 if (!DS)
2296 return false;
2297
2298 if (isa<GlobalAlias>(Val: DS)) {
2299 // Just emit the nop directly.
2300 EmitToStreamer(Inst: MCInstBuilder(AArch64::NOP));
2301 return true;
2302 }
2303 MCSymbol *Dot = OutContext.createTempSymbol();
2304 OutStreamer->emitLabel(Symbol: Dot);
2305 const MCExpr *DeactDotExpr = MCSymbolRefExpr::create(Symbol: Dot, Ctx&: OutContext);
2306
2307 const MCExpr *DSExpr = MCSymbolRefExpr::create(
2308 Symbol: OutContext.getOrCreateSymbol(Name: DS->getName()), Ctx&: OutContext);
2309 OutStreamer->emitRelocDirective(Offset: *DeactDotExpr, Name: "R_AARCH64_PATCHINST", Expr: DSExpr,
2310 Loc: SMLoc());
2311 return false;
2312}
2313
2314AArch64AsmPrinter::PtrAuthSchema AArch64AsmPrinter::PtrAuthSchema::CreateImmReg(
2315 AArch64PACKey::ID Key, uint64_t IntDisc, const MachineOperand &AddrDiscOp) {
2316 PtrAuthSchema Schema;
2317 Schema.Key = Key;
2318 Schema.IntDisc = IntDisc;
2319 Schema.AddrDisc = AddrDiscOp.getReg();
2320 Schema.AddrDiscIsKilled = AddrDiscOp.isKill();
2321 Schema.PCDisc = Register();
2322 return Schema;
2323}
2324
2325AArch64AsmPrinter::PtrAuthSchema AArch64AsmPrinter::PtrAuthSchema::CreateRegReg(
2326 AArch64PACKey::ID Key, Register AddrDisc, Register PCDisc) {
2327 assert(PCDisc.isValid() && "Use CreateImmReg for non-PC schemas");
2328 PtrAuthSchema Schema;
2329 Schema.Key = Key;
2330 Schema.IntDisc = 0;
2331 Schema.AddrDisc = AddrDisc;
2332 Schema.AddrDiscIsKilled = false;
2333 Schema.PCDisc = PCDisc;
2334 return Schema;
2335}
2336
2337void AArch64AsmPrinter::emitPtrauthApplyIndirectAddend(Register Pointer,
2338 Register Scratch,
2339 int64_t Addend) {
2340 if (isInt<9>(x: Addend)) {
2341 // ldrsw Scratch, [Pointer, #Addend]! ; note: Pointer+Addend is used later.
2342 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRSWpre)
2343 .addReg(Reg: Pointer)
2344 .addReg(Reg: Scratch)
2345 .addReg(Reg: Pointer)
2346 .addImm(/*simm9:*/ Val: Addend));
2347 } else {
2348 // Pointer += Addend computation has 2 variants
2349 if (isUInt<24>(x: Addend)) {
2350 // Variant 1: add Pointer, Pointer, (Addend >> shift12) lsl shift12
2351 // This can take up to 2 instructions.
2352 for (int BitPos = 0; BitPos != 24 && (Addend >> BitPos); BitPos += 12) {
2353 EmitToStreamer(
2354 Inst: MCInstBuilder(AArch64::ADDXri)
2355 .addReg(Reg: Pointer)
2356 .addReg(Reg: Pointer)
2357 .addImm(Val: (Addend >> BitPos) & 0xfff)
2358 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: BitPos)));
2359 }
2360 } else {
2361 // Variant 2: accumulate constant in Scratch 16 bits at a time,
2362 // and add it to Pointer. This can take 2-5 instructions.
2363 emitMOVZ(Dest: Scratch, Imm: Addend & 0xffff, Shift: 0);
2364 for (int Offset = 16; Offset < 64; Offset += 16) {
2365 if (unsigned Fragment = (Addend >> Offset) & 0xffff)
2366 emitMOVK(Dest: Scratch, Imm: Fragment, Shift: Offset);
2367 }
2368
2369 // add Pointer, Pointer, Scratch
2370 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXrs)
2371 .addReg(Reg: Pointer)
2372 .addReg(Reg: Pointer)
2373 .addReg(Reg: Scratch)
2374 .addImm(Val: 0));
2375 }
2376 // ldrsw Scratch, [Pointer]
2377 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRSWui)
2378 .addReg(Reg: Scratch)
2379 .addReg(Reg: Pointer)
2380 .addImm(Val: 0));
2381 }
2382 // add Pointer, Pointer, Scratch
2383 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXrs)
2384 .addReg(Reg: Pointer)
2385 .addReg(Reg: Pointer)
2386 .addReg(Reg: Scratch)
2387 .addImm(Val: 0));
2388}
2389
2390static PtrauthCheckMode getCheckMode(const MachineFunction *MF) {
2391 const AArch64Subtarget &STI = MF->getSubtarget<AArch64Subtarget>();
2392
2393 // If an override is passed via command line argument, just use that value.
2394 if (std::optional<PtrauthCheckMode> Mode =
2395 STI.getCLOpts().ptrauth_auth_checks)
2396 return *Mode;
2397
2398 // Otherwise, on an FPAC CPU, you get traps whether you want them or not:
2399 // there's no point in emitting checks or traps.
2400 if (STI.hasFPAC())
2401 return PtrauthCheckMode::Unchecked;
2402
2403 bool ShouldTrap = MF->getFunction().hasFnAttribute(Kind: "ptrauth-auth-traps");
2404 return ShouldTrap ? PtrauthCheckMode::Trap : PtrauthCheckMode::Poison;
2405}
2406
2407// We expand non-signing AUT* pseudo instructions into a sequence of the form
2408//
2409// ; 1. Authenticate Pointer
2410//
2411// or
2412//
2413// ; 1. Authenticate Pointer
2414// ; 2. Check that Pointer is valid, trap otherwise
2415//
2416// We expand AUT*PAC pseudo instructions into a sequence of the form
2417// (with addend only applied if Addend argument is given):
2418//
2419// ; 1. Authenticate Pointer
2420// ; 3. Apply addend and sign Pointer
2421//
2422// or
2423//
2424// ; 1. Authenticate Pointer
2425// ; 2. Check that Pointer is valid, trap otherwise
2426// ; 3. Apply addend and sign Pointer
2427//
2428// or
2429//
2430// ; 1. Authenticate Pointer
2431// ; 2. Check that Pointer is valid, jump to .Lon_failure otherwise
2432// ; 3. Apply addend and sign Pointer
2433// .Lon_failure:
2434//
2435void AArch64AsmPrinter::emitPtrauthAuthResign(
2436 Register Pointer, Register Scratch, PtrAuthSchema AuthSchema,
2437 std::optional<PtrAuthSchema> SignSchema, std::optional<int64_t> Addend,
2438 Value *DS) {
2439 const PtrauthCheckMode CheckMode = getCheckMode(MF);
2440 const bool IsAuthWithPC = AuthSchema.PCDisc.isValid();
2441 assert(!SignSchema || !SignSchema->PCDisc.isValid());
2442
2443 Register SignAddrDiscOrNone = SignSchema ? SignSchema->AddrDisc : Register();
2444
2445 // 1. Authenticate Pointer - this is the only common step.
2446 // It is more complex than signing because AUTI[AB]171615 may be used.
2447
2448 if (IsAuthWithPC) {
2449 assert(Pointer == AArch64::X17 && Scratch == AArch64::X16 &&
2450 "AUTPCPAC must use x17/x16 as Pointer/Scratch");
2451
2452 assert(AuthSchema.AddrDisc == AArch64::X16 &&
2453 "AUTPCPAC requires address discriminator in X16");
2454
2455 assert(AuthSchema.PCDisc == AArch64::X15 &&
2456 "AUTPCPAC requires PC discriminator in X15");
2457
2458 assert(AuthSchema.IntDisc == 0 && "AUTPCPAC does not support IntDisc");
2459
2460 assert((AuthSchema.Key == AArch64PACKey::IB ||
2461 AuthSchema.Key == AArch64PACKey::IA) &&
2462 "AUTPCPAC only supports AUT-ing with IA/IB");
2463
2464 if (!emitDeactivationSymbolRelocation(DS)) {
2465 unsigned AutOpc = (AuthSchema.Key == AArch64PACKey::IB)
2466 ? AArch64::AUTIB171615
2467 : AArch64::AUTIA171615;
2468 EmitToStreamer(Inst: MCInstBuilder(AutOpc));
2469 }
2470 } else {
2471 // emitPtrauthDiscriminator is allowed to clobber AuthSchema.AddrDisc as
2472 // long as it is not used past this point neither externally (the register
2473 // operand is "killed"), nor internally (it does not alias anything being
2474 // used later by this pseudo instruction).
2475 //
2476 // Note that, while rather unlikely, it is technically possible to use the
2477 // Pointer to compute its own discriminator.
2478 Register AUTDiscReg = emitPtrauthDiscriminator(
2479 Disc: AuthSchema.IntDisc, AddrDisc: AuthSchema.AddrDisc, ScratchReg: Scratch,
2480 MayClobberAddrDisc: AuthSchema.addrDiscIsKilledAndNoneOf(Regs: {Pointer, SignAddrDiscOrNone}));
2481 if (!emitDeactivationSymbolRelocation(DS))
2482 emitAUT(Key: AuthSchema.Key, Pointer, Disc: AUTDiscReg);
2483 }
2484
2485 // The other two steps are optional, define lambdas for them:
2486 // 2. Check that Pointer is valid, on failure jump to label or trap.
2487 auto EmitCheck = [&](MCSymbol *OnFailure = nullptr) {
2488 emitPtrauthCheckAuthenticatedValue(TestedReg: Pointer, ScratchReg: Scratch, Key: AuthSchema.Key,
2489 Method: AArch64PAuth::AuthCheckMethod::XPAC,
2490 OnFailure);
2491 };
2492 // 3. Apply addend and sign Pointer.
2493 auto EmitResignOnSuccess = [&]() {
2494 if (Addend.has_value())
2495 emitPtrauthApplyIndirectAddend(Pointer, Scratch, Addend: *Addend);
2496
2497 assert(Pointer != SignSchema->AddrDisc && "Pointer is early-clobbered");
2498 Register PACDiscReg =
2499 emitPtrauthDiscriminator(Disc: SignSchema->IntDisc, AddrDisc: SignSchema->AddrDisc,
2500 ScratchReg: Scratch, MayClobberAddrDisc: SignSchema->AddrDiscIsKilled);
2501 emitPAC(Key: SignSchema->Key, Pointer, Disc: PACDiscReg);
2502 };
2503
2504 // Emit checking and resigning as needed.
2505
2506 if (!SignSchema) {
2507 if (CheckMode == PtrauthCheckMode::Trap)
2508 EmitCheck();
2509 // For authentication-only pseudos, Poison is demoted to Unchecked.
2510 return;
2511 }
2512
2513 switch (CheckMode) {
2514 case PtrauthCheckMode::Unchecked:
2515 EmitResignOnSuccess();
2516 break;
2517 case PtrauthCheckMode::Trap:
2518 EmitCheck();
2519 EmitResignOnSuccess();
2520 break;
2521 case PtrauthCheckMode::Poison:
2522 MCSymbol *OnFailure = createTempSymbol(Name: "resign_end_");
2523 EmitCheck(OnFailure);
2524 EmitResignOnSuccess();
2525 OutStreamer->emitLabel(Symbol: OnFailure);
2526 break;
2527 }
2528}
2529
2530void AArch64AsmPrinter::emitPtrauthSign(const MachineInstr *MI) {
2531 Register Val = MI->getOperand(i: 1).getReg();
2532 auto Key = (AArch64PACKey::ID)MI->getOperand(i: 2).getImm();
2533 uint64_t Disc = MI->getOperand(i: 3).getImm();
2534 Register AddrDisc = MI->getOperand(i: 4).getReg();
2535 bool AddrDiscKilled = MI->getOperand(i: 4).isKill();
2536
2537 // As long as at least one of Val and AddrDisc is in GPR64noip, a scratch
2538 // register is available.
2539 Register ScratchReg = Val == AArch64::X16 ? AArch64::X17 : AArch64::X16;
2540 assert(ScratchReg != AddrDisc &&
2541 "Neither X16 nor X17 is available as a scratch register");
2542
2543 // Compute pac discriminator
2544 Register DiscReg = emitPtrauthDiscriminator(
2545 Disc, AddrDisc, ScratchReg, /*MayClobberAddrDisc=*/AddrDiscKilled);
2546
2547 if (emitDeactivationSymbolRelocation(DS: MI->getDeactivationSymbol()))
2548 return;
2549
2550 emitPAC(Key, Pointer: Val, Disc: DiscReg);
2551}
2552
2553void AArch64AsmPrinter::emitPtrauthBranch(const MachineInstr *MI) {
2554 bool IsCall = MI->getOpcode() == AArch64::BLRA;
2555 unsigned BrTarget = MI->getOperand(i: 0).getReg();
2556
2557 auto Key = (AArch64PACKey::ID)MI->getOperand(i: 1).getImm();
2558 uint64_t Disc = MI->getOperand(i: 2).getImm();
2559
2560 unsigned AddrDisc = MI->getOperand(i: 3).getReg();
2561
2562 // Make sure AddrDisc is solely used to compute the discriminator.
2563 // While hardly meaningful, it is still possible to describe an authentication
2564 // of a pointer against its own value (instead of storage address) with
2565 // intrinsics, so use report_fatal_error instead of assert.
2566 if (BrTarget == AddrDisc)
2567 report_fatal_error(reason: "Branch target is signed with its own value");
2568
2569 // If we are printing BLRA pseudo, try to save one MOV by making use of the
2570 // fact that x16 and x17 are described as clobbered by the MI instruction and
2571 // AddrDisc is not used as any other input.
2572 //
2573 // Back in the day, emitPtrauthDiscriminator was restricted to only returning
2574 // either x16 or x17, meaning the returned register is always among the
2575 // implicit-def'ed registers of BLRA pseudo. Now this property can be violated
2576 // if isX16X17Safer predicate is false, thus manually check if AddrDisc is
2577 // among x16 and x17 to prevent clobbering unexpected registers.
2578 //
2579 // Unlike BLRA, BRA pseudo is used to perform computed goto, and thus not
2580 // declared as clobbering x16/x17.
2581 //
2582 // FIXME: Make use of `killed` flags and register masks instead.
2583 bool AddrDiscIsImplicitDef =
2584 IsCall && (AddrDisc == AArch64::X16 || AddrDisc == AArch64::X17);
2585 Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, ScratchReg: AArch64::X17,
2586 MayClobberAddrDisc: AddrDiscIsImplicitDef);
2587 emitBLRA(IsCall, Key, Target: BrTarget, Disc: DiscReg);
2588}
2589
2590void AArch64AsmPrinter::emitAddImm(MCRegister Reg, int64_t Addend,
2591 MCRegister Tmp) {
2592 if (Addend != 0) {
2593 const uint64_t AbsOffset = (Addend > 0 ? Addend : -((uint64_t)Addend));
2594 const bool IsNeg = Addend < 0;
2595 if (isUInt<24>(x: AbsOffset)) {
2596 for (int BitPos = 0; BitPos != 24 && (AbsOffset >> BitPos);
2597 BitPos += 12) {
2598 EmitToStreamer(
2599 Inst: MCInstBuilder(IsNeg ? AArch64::SUBXri : AArch64::ADDXri)
2600 .addReg(Reg)
2601 .addReg(Reg)
2602 .addImm(Val: (AbsOffset >> BitPos) & 0xfff)
2603 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: BitPos)));
2604 }
2605 } else {
2606 const uint64_t UAddend = Addend;
2607 EmitToStreamer(Inst: MCInstBuilder(IsNeg ? AArch64::MOVNXi : AArch64::MOVZXi)
2608 .addReg(Reg: Tmp)
2609 .addImm(Val: (IsNeg ? ~UAddend : UAddend) & 0xffff)
2610 .addImm(/*shift=*/Val: 0));
2611 auto NeedMovk = [IsNeg, UAddend](int BitPos) -> bool {
2612 assert(BitPos == 16 || BitPos == 32 || BitPos == 48);
2613 uint64_t Shifted = UAddend >> BitPos;
2614 if (!IsNeg)
2615 return Shifted != 0;
2616 for (int I = 0; I != 64 - BitPos; I += 16)
2617 if (((Shifted >> I) & 0xffff) != 0xffff)
2618 return true;
2619 return false;
2620 };
2621 for (int BitPos = 16; BitPos != 64 && NeedMovk(BitPos); BitPos += 16)
2622 emitMOVK(Dest: Tmp, Imm: (UAddend >> BitPos) & 0xffff, Shift: BitPos);
2623
2624 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXrs)
2625 .addReg(Reg)
2626 .addReg(Reg)
2627 .addReg(Reg: Tmp)
2628 .addImm(/*shift=*/Val: 0));
2629 }
2630 }
2631}
2632
2633void AArch64AsmPrinter::emitAddress(MCRegister Reg, const MCExpr *Expr,
2634 MCRegister Tmp, bool DSOLocal,
2635 const MCSubtargetInfo &STI) {
2636 MCValue Val;
2637 if (!Expr->evaluateAsRelocatable(Res&: Val, Asm: nullptr))
2638 report_fatal_error(reason: "emitAddress could not evaluate");
2639 if (DSOLocal) {
2640 EmitToStreamer(
2641 Inst: MCInstBuilder(AArch64::ADRP)
2642 .addReg(Reg)
2643 .addExpr(Val: MCSpecifierExpr::create(Expr, S: AArch64::S_ABS_PAGE,
2644 Ctx&: OutStreamer->getContext())));
2645 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
2646 .addReg(Reg)
2647 .addReg(Reg)
2648 .addExpr(Val: MCSpecifierExpr::create(
2649 Expr, S: AArch64::S_LO12, Ctx&: OutStreamer->getContext()))
2650 .addImm(Val: 0));
2651 } else {
2652 auto *SymRef =
2653 MCSymbolRefExpr::create(Symbol: Val.getAddSym(), Ctx&: OutStreamer->getContext());
2654 EmitToStreamer(
2655 Inst: MCInstBuilder(AArch64::ADRP)
2656 .addReg(Reg)
2657 .addExpr(Val: MCSpecifierExpr::create(Expr: SymRef, S: AArch64::S_GOT_PAGE,
2658 Ctx&: OutStreamer->getContext())));
2659 EmitToStreamer(
2660 Inst: MCInstBuilder(AArch64::LDRXui)
2661 .addReg(Reg)
2662 .addReg(Reg)
2663 .addExpr(Val: MCSpecifierExpr::create(Expr: SymRef, S: AArch64::S_GOT_LO12,
2664 Ctx&: OutStreamer->getContext())));
2665 emitAddImm(Reg, Addend: Val.getConstant(), Tmp);
2666 }
2667}
2668
2669static bool targetSupportsIRelativeRelocation(const Triple &TT) {
2670 // IFUNCs are ELF-only.
2671 if (!TT.isOSBinFormatELF())
2672 return false;
2673
2674 // IFUNCs are supported on glibc, bionic, and some but not all of the BSDs.
2675 return TT.isOSGlibc() || TT.isAndroid() || TT.isOSFreeBSD() ||
2676 TT.isOSDragonFly() || TT.isOSNetBSD();
2677}
2678
2679// Emit an ifunc resolver that returns a signed pointer to the specified target,
2680// and return a FUNCINIT reference to the resolver. In the linked binary, this
2681// function becomes the target of an IRELATIVE relocation. This resolver is used
2682// to relocate signed pointers in global variable initializers in special cases
2683// where the standard R_AARCH64_AUTH_ABS64 relocation would not work.
2684//
2685// Example (signed null pointer, not address discriminated):
2686//
2687// .8byte .Lpauth_ifunc0
2688// .pushsection .text.startup,"ax",@progbits
2689// .Lpauth_ifunc0:
2690// mov x0, #0
2691// mov x1, #12345
2692// b __emupac_pacda
2693//
2694// Example (signed null pointer, address discriminated):
2695//
2696// .Ltmp:
2697// .8byte .Lpauth_ifunc0
2698// .pushsection .text.startup,"ax",@progbits
2699// .Lpauth_ifunc0:
2700// mov x0, #0
2701// adrp x1, .Ltmp
2702// add x1, x1, :lo12:.Ltmp
2703// b __emupac_pacda
2704// .popsection
2705//
2706// Example (signed pointer to symbol, not address discriminated):
2707//
2708// .Ltmp:
2709// .8byte .Lpauth_ifunc0
2710// .pushsection .text.startup,"ax",@progbits
2711// .Lpauth_ifunc0:
2712// adrp x0, symbol
2713// add x0, x0, :lo12:symbol
2714// mov x1, #12345
2715// b __emupac_pacda
2716// .popsection
2717//
2718// Example (signed null pointer, not address discriminated, with deactivation
2719// symbol ds):
2720//
2721// .8byte .Lpauth_ifunc0
2722// .pushsection .text.startup,"ax",@progbits
2723// .Lpauth_ifunc0:
2724// mov x0, #0
2725// mov x1, #12345
2726// .reloc ., R_AARCH64_PATCHINST, ds
2727// b __emupac_pacda
2728// ret
2729// .popsection
2730const MCExpr *AArch64AsmPrinter::emitPAuthRelocationAsIRelative(
2731 const MCExpr *Target, uint64_t Disc, AArch64PACKey::ID KeyID,
2732 bool HasAddressDiversity, bool IsDSOLocal, const MCExpr *DSExpr) {
2733 const Triple &TT = TM.getTargetTriple();
2734
2735 // We only emit an IRELATIVE relocation if the target supports IRELATIVE.
2736 if (!targetSupportsIRelativeRelocation(TT))
2737 return nullptr;
2738
2739 // For now, only the DA key is supported.
2740 if (KeyID != AArch64PACKey::DA)
2741 return nullptr;
2742
2743 // AArch64Subtarget is huge, so heap allocate it so we don't run out of stack
2744 // space.
2745 auto STI = std::make_unique<AArch64Subtarget>(
2746 args: TT, args: TM.getTargetCPU(), args: TM.getTargetCPU(), args: TM.getTargetFeatureString(), args&: TM,
2747 args: true);
2748 this->STI = STI.get();
2749
2750 MCSymbol *Place = OutStreamer->getContext().createTempSymbol();
2751 OutStreamer->emitLabel(Symbol: Place);
2752 OutStreamer->pushSection();
2753
2754 const MCSymbolELF *Group =
2755 static_cast<MCSectionELF *>(OutStreamer->getCurrentSectionOnly())
2756 ->getGroup();
2757 auto Flags = ELF::SHF_ALLOC | ELF::SHF_EXECINSTR;
2758 if (Group)
2759 Flags |= ELF::SHF_GROUP;
2760 OutStreamer->switchSection(Section: OutStreamer->getContext().getELFSection(
2761 Section: ".text.startup", Type: ELF::SHT_PROGBITS, Flags, EntrySize: 0, Group, IsComdat: true,
2762 UniqueID: Group ? MCSection::NonUniqueID : PAuthIFuncNextUniqueID++, LinkedToSym: nullptr));
2763
2764 MCSymbol *IRelativeSym =
2765 OutStreamer->getContext().createLinkerPrivateSymbol(Name: "pauth_ifunc");
2766 OutStreamer->emitLabel(Symbol: IRelativeSym);
2767 if (isa<MCConstantExpr>(Val: Target)) {
2768 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::MOVZXi)
2769 .addReg(Reg: AArch64::X0)
2770 .addExpr(Val: Target)
2771 .addImm(Val: 0),
2772 STI: *STI);
2773 } else {
2774 emitAddress(Reg: AArch64::X0, Expr: Target, Tmp: AArch64::X16, DSOLocal: IsDSOLocal, STI: *STI);
2775 }
2776 if (HasAddressDiversity) {
2777 auto *PlacePlusDisc = MCBinaryExpr::createAdd(
2778 LHS: MCSymbolRefExpr::create(Symbol: Place, Ctx&: OutStreamer->getContext()),
2779 RHS: MCConstantExpr::create(Value: Disc, Ctx&: OutStreamer->getContext()),
2780 Ctx&: OutStreamer->getContext());
2781 emitAddress(Reg: AArch64::X1, Expr: PlacePlusDisc, Tmp: AArch64::X16, /*IsDSOLocal=*/DSOLocal: true,
2782 STI: *STI);
2783 } else {
2784 if (!isUInt<16>(x: Disc)) {
2785 OutContext.reportError(L: SMLoc(), Msg: "AArch64 PAC Discriminator '" +
2786 Twine(Disc) +
2787 "' out of range [0, 0xFFFF]");
2788 }
2789 emitMOVZ(Dest: AArch64::X1, Imm: Disc, Shift: 0);
2790 }
2791
2792 if (DSExpr) {
2793 MCSymbol *PrePACInst = OutStreamer->getContext().createTempSymbol();
2794 OutStreamer->emitLabel(Symbol: PrePACInst);
2795
2796 auto *PrePACInstExpr =
2797 MCSymbolRefExpr::create(Symbol: PrePACInst, Ctx&: OutStreamer->getContext());
2798 OutStreamer->emitRelocDirective(Offset: *PrePACInstExpr, Name: "R_AARCH64_PATCHINST",
2799 Expr: DSExpr, Loc: SMLoc());
2800 }
2801
2802 // We don't know the subtarget because this is being emitted for a global
2803 // initializer. Because the performance of IFUNC resolvers is unimportant, we
2804 // always call the EmuPAC runtime, which will end up using the PAC instruction
2805 // if the target supports PAC.
2806 MCSymbol *EmuPAC =
2807 OutStreamer->getContext().getOrCreateSymbol(Name: "__emupac_pacda");
2808 const MCSymbolRefExpr *EmuPACRef =
2809 MCSymbolRefExpr::create(Symbol: EmuPAC, Ctx&: OutStreamer->getContext());
2810 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::B).addExpr(Val: EmuPACRef),
2811 STI: *STI);
2812
2813 // We need a RET despite the above tail call because the deactivation symbol
2814 // may replace the tail call with a NOP.
2815 if (DSExpr)
2816 OutStreamer->emitInstruction(
2817 Inst: MCInstBuilder(AArch64::RET).addReg(Reg: AArch64::LR), STI: *STI);
2818 OutStreamer->popSection();
2819
2820 return MCSpecifierExpr::create(
2821 Expr: MCSymbolRefExpr::create(Symbol: IRelativeSym, Ctx&: OutStreamer->getContext()),
2822 S: AArch64::S_FUNCINIT, Ctx&: OutStreamer->getContext());
2823}
2824
2825const MCExpr *
2826AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) {
2827 MCContext &Ctx = OutContext;
2828
2829 // Figure out the base symbol and the addend, if any.
2830 APInt Offset(64, 0);
2831 const Value *BaseGV = CPA.getPointer()->stripAndAccumulateConstantOffsets(
2832 DL: getDataLayout(), Offset, /*AllowNonInbounds=*/true);
2833
2834 auto *BaseGVB = dyn_cast<GlobalValue>(Val: BaseGV);
2835
2836 const MCExpr *Sym;
2837 if (BaseGVB) {
2838 // If there is an addend, turn that into the appropriate MCExpr.
2839 Sym = MCSymbolRefExpr::create(Symbol: getSymbol(GV: BaseGVB), Ctx);
2840 if (Offset.sgt(RHS: 0))
2841 Sym = MCBinaryExpr::createAdd(
2842 LHS: Sym, RHS: MCConstantExpr::create(Value: Offset.getSExtValue(), Ctx), Ctx);
2843 else if (Offset.slt(RHS: 0))
2844 Sym = MCBinaryExpr::createSub(
2845 LHS: Sym, RHS: MCConstantExpr::create(Value: (-Offset).getSExtValue(), Ctx), Ctx);
2846 } else if (isa<ConstantPointerNull>(Val: BaseGV)) {
2847 Sym = MCConstantExpr::create(Value: Offset.getSExtValue(), Ctx);
2848 } else {
2849 reportFatalUsageError(reason: "unsupported constant expression in ptrauth pointer");
2850 }
2851
2852 const MCExpr *DSExpr = nullptr;
2853 if (auto *DS = dyn_cast<GlobalValue>(Val: CPA.getDeactivationSymbol())) {
2854 if (isa<GlobalAlias>(Val: DS))
2855 return Sym;
2856 DSExpr = MCSymbolRefExpr::create(Symbol: getSymbol(GV: DS), Ctx);
2857 }
2858
2859 uint64_t KeyID = CPA.getKey()->getZExtValue();
2860 // We later rely on valid KeyID value in AArch64PACKeyIDToString call from
2861 // AArch64AuthMCExpr::printImpl, so fail fast.
2862 if (KeyID > AArch64PACKey::LAST) {
2863 CPA.getContext().emitError(ErrorStr: "AArch64 PAC Key ID '" + Twine(KeyID) +
2864 "' out of range [0, " +
2865 Twine((unsigned)AArch64PACKey::LAST) + "]");
2866 KeyID = 0;
2867 }
2868
2869 uint64_t Disc = CPA.getDiscriminator()->getZExtValue();
2870
2871 // Check if we can represent this with an IRELATIVE and emit it if so.
2872 if (auto *IFuncSym = emitPAuthRelocationAsIRelative(
2873 Target: Sym, Disc, KeyID: AArch64PACKey::ID(KeyID), HasAddressDiversity: CPA.hasAddressDiscriminator(),
2874 IsDSOLocal: BaseGVB && BaseGVB->isDSOLocal(), DSExpr))
2875 return IFuncSym;
2876
2877 if (!isUInt<16>(x: Disc)) {
2878 CPA.getContext().emitError(ErrorStr: "AArch64 PAC Discriminator '" + Twine(Disc) +
2879 "' out of range [0, 0xFFFF]");
2880 Disc = 0;
2881 }
2882
2883 if (DSExpr)
2884 report_fatal_error(reason: "deactivation symbols unsupported in constant "
2885 "expressions on this target");
2886
2887 // Finally build the complete @AUTH expr.
2888 return AArch64AuthMCExpr::create(Expr: Sym, Discriminator: Disc, Key: AArch64PACKey::ID(KeyID),
2889 HasAddressDiversity: CPA.hasAddressDiscriminator(), Ctx);
2890}
2891
2892void AArch64AsmPrinter::LowerLOADauthptrstatic(const MachineInstr &MI) {
2893 unsigned DstReg = MI.getOperand(i: 0).getReg();
2894 const MachineOperand &GAOp = MI.getOperand(i: 1);
2895 const uint64_t KeyC = MI.getOperand(i: 2).getImm();
2896 assert(KeyC <= AArch64PACKey::LAST &&
2897 "key is out of range [0, AArch64PACKey::LAST]");
2898 const auto Key = (AArch64PACKey::ID)KeyC;
2899 const uint64_t Disc = MI.getOperand(i: 3).getImm();
2900 assert(isUInt<16>(Disc) &&
2901 "constant discriminator is out of range [0, 0xffff]");
2902
2903 // Emit instruction sequence like the following:
2904 // ADRP x16, symbol$auth_ptr$key$disc
2905 // LDR x16, [x16, :lo12:symbol$auth_ptr$key$disc]
2906 //
2907 // Where the $auth_ptr$ symbol is the stub slot containing the signed pointer
2908 // to symbol.
2909 MCSymbol *AuthPtrStubSym;
2910 if (TM.getTargetTriple().isOSBinFormatELF()) {
2911 const auto &TLOF =
2912 static_cast<const AArch64_ELFTargetObjectFile &>(getObjFileLowering());
2913
2914 assert(GAOp.getOffset() == 0 &&
2915 "non-zero offset for $auth_ptr$ stub slots is not supported");
2916 const MCSymbol *GASym = TM.getSymbol(GV: GAOp.getGlobal());
2917 AuthPtrStubSym = TLOF.getAuthPtrSlotSymbol(TM, MMI, RawSym: GASym, Key, Discriminator: Disc);
2918 } else {
2919 assert(TM.getTargetTriple().isOSBinFormatMachO() &&
2920 "LOADauthptrstatic is implemented only for MachO/ELF");
2921
2922 const auto &TLOF = static_cast<const AArch64_MachoTargetObjectFile &>(
2923 getObjFileLowering());
2924
2925 assert(GAOp.getOffset() == 0 &&
2926 "non-zero offset for $auth_ptr$ stub slots is not supported");
2927 const MCSymbol *GASym = TM.getSymbol(GV: GAOp.getGlobal());
2928 AuthPtrStubSym = TLOF.getAuthPtrSlotSymbol(TM, MMI, RawSym: GASym, Key, Discriminator: Disc);
2929 }
2930
2931 MachineOperand StubMOHi =
2932 MachineOperand::CreateMCSymbol(Sym: AuthPtrStubSym, TargetFlags: AArch64II::MO_PAGE);
2933 MachineOperand StubMOLo = MachineOperand::CreateMCSymbol(
2934 Sym: AuthPtrStubSym, TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2935 MCOperand StubMCHi, StubMCLo;
2936
2937 MCInstLowering.lowerOperand(MO: StubMOHi, MCOp&: StubMCHi);
2938 MCInstLowering.lowerOperand(MO: StubMOLo, MCOp&: StubMCLo);
2939
2940 EmitToStreamer(
2941 S&: *OutStreamer,
2942 Inst: MCInstBuilder(AArch64::ADRP).addReg(Reg: DstReg).addOperand(Op: StubMCHi));
2943
2944 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::LDRXui)
2945 .addReg(Reg: DstReg)
2946 .addReg(Reg: DstReg)
2947 .addOperand(Op: StubMCLo));
2948}
2949
2950void AArch64AsmPrinter::LowerMOVaddrPAC(const MachineInstr &MI) {
2951 const bool IsGOTLoad = MI.getOpcode() == AArch64::LOADgotPAC;
2952 const bool IsELFSignedGOT = MI.getParent()
2953 ->getParent()
2954 ->getInfo<AArch64FunctionInfo>()
2955 ->hasELFSignedGOT();
2956 MachineOperand GAOp = MI.getOperand(i: 0);
2957 const uint64_t KeyC = MI.getOperand(i: 1).getImm();
2958 assert(KeyC <= AArch64PACKey::LAST &&
2959 "key is out of range [0, AArch64PACKey::LAST]");
2960 const auto Key = (AArch64PACKey::ID)KeyC;
2961 const unsigned AddrDisc = MI.getOperand(i: 2).getReg();
2962 const uint64_t Disc = MI.getOperand(i: 3).getImm();
2963
2964 const int64_t Offset = GAOp.getOffset();
2965 GAOp.setOffset(0);
2966
2967 // Emit:
2968 // target materialization:
2969 // - via GOT:
2970 // - unsigned GOT:
2971 // adrp x16, :got:target
2972 // ldr x16, [x16, :got_lo12:target]
2973 // add offset to x16 if offset != 0
2974 // - ELF signed GOT:
2975 // adrp x17, :got:target
2976 // add x17, x17, :got_auth_lo12:target
2977 // ldr x16, [x17]
2978 // aut{i|d}a x16, x17
2979 // check+trap sequence (if no FPAC)
2980 // add offset to x16 if offset != 0
2981 //
2982 // - direct:
2983 // adrp x16, target
2984 // add x16, x16, :lo12:target
2985 // add offset to x16 if offset != 0
2986 //
2987 // add offset to x16:
2988 // - abs(offset) fits 24 bits:
2989 // add/sub x16, x16, #<offset>[, #lsl 12] (up to 2 instructions)
2990 // - abs(offset) does not fit 24 bits:
2991 // - offset < 0:
2992 // movn+movk sequence filling x17 register with the offset (up to 4
2993 // instructions)
2994 // add x16, x16, x17
2995 // - offset > 0:
2996 // movz+movk sequence filling x17 register with the offset (up to 4
2997 // instructions)
2998 // add x16, x16, x17
2999 //
3000 // signing:
3001 // - 0 discriminator:
3002 // paciza x16
3003 // - Non-0 discriminator, no address discriminator:
3004 // mov x17, #Disc
3005 // pacia x16, x17
3006 // - address discriminator (with potentially folded immediate discriminator):
3007 // pacia x16, xAddrDisc
3008
3009 MachineOperand GAMOHi(GAOp), GAMOLo(GAOp);
3010 MCOperand GAMCHi, GAMCLo;
3011
3012 GAMOHi.setTargetFlags(AArch64II::MO_PAGE);
3013 GAMOLo.setTargetFlags(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
3014 if (IsGOTLoad) {
3015 GAMOHi.addTargetFlag(F: AArch64II::MO_GOT);
3016 GAMOLo.addTargetFlag(F: AArch64II::MO_GOT);
3017 }
3018
3019 MCInstLowering.lowerOperand(MO: GAMOHi, MCOp&: GAMCHi);
3020 MCInstLowering.lowerOperand(MO: GAMOLo, MCOp&: GAMCLo);
3021
3022 EmitToStreamer(
3023 Inst: MCInstBuilder(AArch64::ADRP)
3024 .addReg(Reg: IsGOTLoad && IsELFSignedGOT ? AArch64::X17 : AArch64::X16)
3025 .addOperand(Op: GAMCHi));
3026
3027 if (IsGOTLoad) {
3028 if (IsELFSignedGOT) {
3029 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
3030 .addReg(Reg: AArch64::X17)
3031 .addReg(Reg: AArch64::X17)
3032 .addOperand(Op: GAMCLo)
3033 .addImm(Val: 0));
3034
3035 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
3036 .addReg(Reg: AArch64::X16)
3037 .addReg(Reg: AArch64::X17)
3038 .addImm(Val: 0));
3039
3040 assert(GAOp.isGlobal());
3041 assert(GAOp.getGlobal()->getValueType() != nullptr);
3042
3043 bool IsFunctionTy = GAOp.getGlobal()->getValueType()->isFunctionTy();
3044 auto AuthKey = IsFunctionTy ? AArch64PACKey::IA : AArch64PACKey::DA;
3045 emitAUT(Key: AuthKey, Pointer: AArch64::X16, Disc: AArch64::X17);
3046
3047 if (!STI->hasFPAC())
3048 emitPtrauthCheckAuthenticatedValue(TestedReg: AArch64::X16, ScratchReg: AArch64::X17, Key: AuthKey,
3049 Method: AArch64PAuth::AuthCheckMethod::XPAC);
3050 } else {
3051 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
3052 .addReg(Reg: AArch64::X16)
3053 .addReg(Reg: AArch64::X16)
3054 .addOperand(Op: GAMCLo));
3055 }
3056 } else {
3057 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
3058 .addReg(Reg: AArch64::X16)
3059 .addReg(Reg: AArch64::X16)
3060 .addOperand(Op: GAMCLo)
3061 .addImm(Val: 0));
3062 }
3063
3064 emitAddImm(Reg: AArch64::X16, Addend: Offset, Tmp: AArch64::X17);
3065 Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, ScratchReg: AArch64::X17);
3066
3067 emitPAC(Key, Pointer: AArch64::X16, Disc: DiscReg);
3068}
3069
3070void AArch64AsmPrinter::LowerLOADgotAUTH(const MachineInstr &MI) {
3071 Register DstReg = MI.getOperand(i: 0).getReg();
3072 Register AuthResultReg = STI->hasFPAC() ? DstReg : AArch64::X16;
3073 const MachineOperand &GAMO = MI.getOperand(i: 1);
3074 assert(GAMO.getOffset() == 0);
3075
3076 if (MI.getMF()->getTarget().getCodeModel() == CodeModel::Tiny) {
3077 MCOperand GAMC;
3078 MCInstLowering.lowerOperand(MO: GAMO, MCOp&: GAMC);
3079 EmitToStreamer(
3080 Inst: MCInstBuilder(AArch64::ADR).addReg(Reg: AArch64::X17).addOperand(Op: GAMC));
3081 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
3082 .addReg(Reg: AuthResultReg)
3083 .addReg(Reg: AArch64::X17)
3084 .addImm(Val: 0));
3085 } else {
3086 MachineOperand GAHiOp(GAMO);
3087 MachineOperand GALoOp(GAMO);
3088 GAHiOp.addTargetFlag(F: AArch64II::MO_PAGE);
3089 GALoOp.addTargetFlag(F: AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
3090
3091 MCOperand GAMCHi, GAMCLo;
3092 MCInstLowering.lowerOperand(MO: GAHiOp, MCOp&: GAMCHi);
3093 MCInstLowering.lowerOperand(MO: GALoOp, MCOp&: GAMCLo);
3094
3095 EmitToStreamer(
3096 Inst: MCInstBuilder(AArch64::ADRP).addReg(Reg: AArch64::X17).addOperand(Op: GAMCHi));
3097
3098 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
3099 .addReg(Reg: AArch64::X17)
3100 .addReg(Reg: AArch64::X17)
3101 .addOperand(Op: GAMCLo)
3102 .addImm(Val: 0));
3103
3104 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
3105 .addReg(Reg: AuthResultReg)
3106 .addReg(Reg: AArch64::X17)
3107 .addImm(Val: 0));
3108 }
3109
3110 assert(GAMO.isGlobal());
3111 MCSymbol *UndefWeakSym;
3112 if (GAMO.getGlobal()->hasExternalWeakLinkage()) {
3113 UndefWeakSym = createTempSymbol(Name: "undef_weak");
3114 EmitToStreamer(
3115 Inst: MCInstBuilder(AArch64::CBZX)
3116 .addReg(Reg: AuthResultReg)
3117 .addExpr(Val: MCSymbolRefExpr::create(Symbol: UndefWeakSym, Ctx&: OutContext)));
3118 }
3119
3120 assert(GAMO.getGlobal()->getValueType() != nullptr);
3121
3122 bool IsFunctionTy = GAMO.getGlobal()->getValueType()->isFunctionTy();
3123 auto AuthKey = IsFunctionTy ? AArch64PACKey::IA : AArch64PACKey::DA;
3124 emitAUT(Key: AuthKey, Pointer: AuthResultReg, Disc: AArch64::X17);
3125
3126 if (GAMO.getGlobal()->hasExternalWeakLinkage())
3127 OutStreamer->emitLabel(Symbol: UndefWeakSym);
3128
3129 if (!STI->hasFPAC()) {
3130 emitPtrauthCheckAuthenticatedValue(TestedReg: AuthResultReg, ScratchReg: AArch64::X17, Key: AuthKey,
3131 Method: AArch64PAuth::AuthCheckMethod::XPAC);
3132
3133 emitMovXReg(Dest: DstReg, Src: AuthResultReg);
3134 }
3135}
3136
3137const MCExpr *
3138AArch64AsmPrinter::lowerBlockAddressConstant(const BlockAddress &BA) {
3139 const MCExpr *BAE = AsmPrinter::lowerBlockAddressConstant(BA);
3140 const Function &Fn = *BA.getFunction();
3141
3142 if (std::optional<uint16_t> BADisc =
3143 STI->getPtrAuthBlockAddressDiscriminatorIfEnabled(ParentFn: Fn))
3144 return AArch64AuthMCExpr::create(Expr: BAE, Discriminator: *BADisc, Key: AArch64PACKey::IA,
3145 /*HasAddressDiversity=*/false, Ctx&: OutContext);
3146
3147 return BAE;
3148}
3149
3150void AArch64AsmPrinter::emitCBPseudoExpansion(const MachineInstr *MI) {
3151 bool IsImm = false;
3152 unsigned Width = 0;
3153
3154 switch (MI->getOpcode()) {
3155 default:
3156 llvm_unreachable("This is not a CB pseudo instruction");
3157 case AArch64::CBBAssertExt:
3158 IsImm = false;
3159 Width = 8;
3160 break;
3161 case AArch64::CBHAssertExt:
3162 IsImm = false;
3163 Width = 16;
3164 break;
3165 case AArch64::CBWPrr:
3166 Width = 32;
3167 break;
3168 case AArch64::CBXPrr:
3169 Width = 64;
3170 break;
3171 case AArch64::CBWPri:
3172 IsImm = true;
3173 Width = 32;
3174 break;
3175 case AArch64::CBXPri:
3176 IsImm = true;
3177 Width = 64;
3178 break;
3179 }
3180
3181 AArch64CC::CondCode CC =
3182 static_cast<AArch64CC::CondCode>(MI->getOperand(i: 0).getImm());
3183 bool NeedsRegSwap = false;
3184 bool NeedsImmDec = false;
3185 bool NeedsImmInc = false;
3186
3187#define GET_CB_OPC(IsImm, Width, ImmCond, RegCond) \
3188 (IsImm \
3189 ? (Width == 32 ? AArch64::CB##ImmCond##Wri : AArch64::CB##ImmCond##Xri) \
3190 : (Width == 8 \
3191 ? AArch64::CBB##RegCond##Wrr \
3192 : (Width == 16 ? AArch64::CBH##RegCond##Wrr \
3193 : (Width == 32 ? AArch64::CB##RegCond##Wrr \
3194 : AArch64::CB##RegCond##Xrr))))
3195 unsigned MCOpC;
3196
3197 // Decide if we need to either swap register operands or increment/decrement
3198 // immediate operands
3199 switch (CC) {
3200 default:
3201 llvm_unreachable("Invalid CB condition code");
3202 case AArch64CC::EQ:
3203 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ EQ, /* Reg-Reg */ EQ);
3204 break;
3205 case AArch64CC::NE:
3206 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ NE, /* Reg-Reg */ NE);
3207 break;
3208 case AArch64CC::HS:
3209 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ HI, /* Reg-Reg */ HS);
3210 NeedsImmDec = IsImm;
3211 break;
3212 case AArch64CC::LO:
3213 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ LO, /* Reg-Reg */ HI);
3214 NeedsRegSwap = !IsImm;
3215 break;
3216 case AArch64CC::HI:
3217 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ HI, /* Reg-Reg */ HI);
3218 break;
3219 case AArch64CC::LS:
3220 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ LO, /* Reg-Reg */ HS);
3221 NeedsRegSwap = !IsImm;
3222 NeedsImmInc = IsImm;
3223 break;
3224 case AArch64CC::GE:
3225 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ GT, /* Reg-Reg */ GE);
3226 NeedsImmDec = IsImm;
3227 break;
3228 case AArch64CC::LT:
3229 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ LT, /* Reg-Reg */ GT);
3230 NeedsRegSwap = !IsImm;
3231 break;
3232 case AArch64CC::GT:
3233 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ GT, /* Reg-Reg */ GT);
3234 break;
3235 case AArch64CC::LE:
3236 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ LT, /* Reg-Reg */ GE);
3237 NeedsRegSwap = !IsImm;
3238 NeedsImmInc = IsImm;
3239 break;
3240 }
3241#undef GET_CB_OPC
3242
3243 MCInst Inst;
3244 Inst.setOpcode(MCOpC);
3245
3246 MCOperand Lhs, Rhs, Trgt;
3247 lowerOperand(MO: MI->getOperand(i: 1), MCOp&: Lhs);
3248 lowerOperand(MO: MI->getOperand(i: 2), MCOp&: Rhs);
3249 lowerOperand(MO: MI->getOperand(i: 3), MCOp&: Trgt);
3250
3251 // Now swap, increment or decrement
3252 if (NeedsRegSwap) {
3253 assert(Lhs.isReg() && "Expected register operand for CB");
3254 assert(Rhs.isReg() && "Expected register operand for CB");
3255 Inst.addOperand(Op: Rhs);
3256 Inst.addOperand(Op: Lhs);
3257 } else if (NeedsImmDec) {
3258 Rhs.setImm(Rhs.getImm() - 1);
3259 Inst.addOperand(Op: Lhs);
3260 Inst.addOperand(Op: Rhs);
3261 } else if (NeedsImmInc) {
3262 Rhs.setImm(Rhs.getImm() + 1);
3263 Inst.addOperand(Op: Lhs);
3264 Inst.addOperand(Op: Rhs);
3265 } else {
3266 Inst.addOperand(Op: Lhs);
3267 Inst.addOperand(Op: Rhs);
3268 }
3269
3270 assert((!IsImm || (Rhs.getImm() >= 0 && Rhs.getImm() < 64)) &&
3271 "CB immediate operand out-of-bounds");
3272
3273 Inst.addOperand(Op: Trgt);
3274 EmitToStreamer(S&: *OutStreamer, Inst);
3275}
3276
3277void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI) {
3278
3279 unsigned StOpc;
3280 unsigned Relaxed = MI->getOperand(i: 2).getImm();
3281 assert(Relaxed < 2 && "Atomic hint relaxed immediate out-of-bounds");
3282 switch (MI->getOpcode()) {
3283 case AArch64::ATOMIC_STORE_HINT_B:
3284 StOpc = Relaxed ? AArch64::STRBBui : AArch64::STLRB;
3285 break;
3286 case AArch64::ATOMIC_STORE_HINT_H:
3287 StOpc = Relaxed ? AArch64::STRHHui : AArch64::STLRH;
3288 break;
3289 case AArch64::ATOMIC_STORE_HINT_W:
3290 StOpc = Relaxed ? AArch64::STRWui : AArch64::STLRW;
3291 break;
3292 case AArch64::ATOMIC_STORE_HINT_X:
3293 StOpc = Relaxed ? AArch64::STRXui : AArch64::STLRX;
3294 break;
3295 default:
3296 llvm_unreachable("Unexpected atomic hint size.");
3297 }
3298
3299 EmitToStreamer(
3300 Inst: MCInstBuilder(AArch64::HINT).addImm(Val: MI->getOperand(i: 3).getImm()));
3301
3302 MCInst Store;
3303 Store.setOpcode(StOpc);
3304 Store.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg()));
3305 Store.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 1).getReg()));
3306 Store.setFlags(MI->getFlags());
3307 if (Relaxed)
3308 Store.addOperand(Op: MCOperand::createImm(Val: 0));
3309 EmitToStreamer(S&: *OutStreamer, Inst: Store);
3310}
3311
3312void AArch64AsmPrinter::emitAtomicHintPseudoExpansionRO(
3313 const MachineInstr *MI) {
3314 unsigned StOpc;
3315 assert(MI->getOperand(5).getImm() == 1 &&
3316 "Atomic store addressing mode only supports relaxed stores");
3317
3318 switch (MI->getOpcode()) {
3319 case AArch64::ATOMIC_STORE_HINT_BroW:
3320 StOpc = AArch64::STRBBroW;
3321 break;
3322 case AArch64::ATOMIC_STORE_HINT_HroW:
3323 StOpc = AArch64::STRHHroW;
3324 break;
3325 case AArch64::ATOMIC_STORE_HINT_WroW:
3326 StOpc = AArch64::STRWroW;
3327 break;
3328 case AArch64::ATOMIC_STORE_HINT_XroW:
3329 StOpc = AArch64::STRXroW;
3330 break;
3331 case AArch64::ATOMIC_STORE_HINT_SroW:
3332 StOpc = AArch64::STRSroW;
3333 break;
3334 case AArch64::ATOMIC_STORE_HINT_DroW:
3335 StOpc = AArch64::STRDroW;
3336 break;
3337 case AArch64::ATOMIC_STORE_HINT_BroX:
3338 StOpc = AArch64::STRBBroX;
3339 break;
3340 case AArch64::ATOMIC_STORE_HINT_HroX:
3341 StOpc = AArch64::STRHHroX;
3342 break;
3343 case AArch64::ATOMIC_STORE_HINT_WroX:
3344 StOpc = AArch64::STRWroX;
3345 break;
3346 case AArch64::ATOMIC_STORE_HINT_XroX:
3347 StOpc = AArch64::STRXroX;
3348 break;
3349 case AArch64::ATOMIC_STORE_HINT_SroX:
3350 StOpc = AArch64::STRSroX;
3351 break;
3352 case AArch64::ATOMIC_STORE_HINT_DroX:
3353 StOpc = AArch64::STRDroX;
3354 break;
3355 default:
3356 llvm_unreachable("Unexpected atomic hint opcode.");
3357 }
3358
3359 EmitToStreamer(
3360 Inst: MCInstBuilder(AArch64::HINT).addImm(Val: MI->getOperand(i: 6).getImm()));
3361
3362 MCInst Store;
3363 Store.setOpcode(StOpc);
3364 Store.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg())); // Rn
3365 Store.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 1).getReg())); // Rm
3366 Store.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 2).getReg())); // Data
3367 Store.addOperand(Op: MCOperand::createImm(Val: MI->getOperand(i: 3).getImm())); // Signed
3368 Store.addOperand(Op: MCOperand::createImm(Val: MI->getOperand(i: 4).getImm())); // Shift
3369 Store.setFlags(MI->getFlags());
3370 EmitToStreamer(S&: *OutStreamer, Inst: Store);
3371}
3372
3373void AArch64AsmPrinter::emitAtomicHintPseudoExpansionImm(
3374 const MachineInstr *MI) {
3375 unsigned StOpc;
3376 assert(MI->getOperand(3).getImm() == 1 &&
3377 "Atomic store addressing mode only supports relaxed stores");
3378
3379 switch (MI->getOpcode()) {
3380 case AArch64::ATOMIC_STORE_HINT_Bui:
3381 StOpc = AArch64::STRBBui;
3382 break;
3383 case AArch64::ATOMIC_STORE_HINT_Hui:
3384 StOpc = AArch64::STRHHui;
3385 break;
3386 case AArch64::ATOMIC_STORE_HINT_Wui:
3387 StOpc = AArch64::STRWui;
3388 break;
3389 case AArch64::ATOMIC_STORE_HINT_Xui:
3390 StOpc = AArch64::STRXui;
3391 break;
3392 case AArch64::ATOMIC_STORE_HINT_Sui:
3393 StOpc = AArch64::STRSui;
3394 break;
3395 case AArch64::ATOMIC_STORE_HINT_Dui:
3396 StOpc = AArch64::STRDui;
3397 break;
3398 case AArch64::ATOMIC_STORE_HINT_Bi:
3399 StOpc = AArch64::STURBBi;
3400 break;
3401 case AArch64::ATOMIC_STORE_HINT_Hi:
3402 StOpc = AArch64::STURHHi;
3403 break;
3404 case AArch64::ATOMIC_STORE_HINT_Wi:
3405 StOpc = AArch64::STURWi;
3406 break;
3407 case AArch64::ATOMIC_STORE_HINT_Xi:
3408 StOpc = AArch64::STURXi;
3409 break;
3410 case AArch64::ATOMIC_STORE_HINT_Si:
3411 StOpc = AArch64::STURSi;
3412 break;
3413 case AArch64::ATOMIC_STORE_HINT_Di:
3414 StOpc = AArch64::STURDi;
3415 break;
3416 default:
3417 llvm_unreachable("Unexpected atomic hint opcode.");
3418 }
3419
3420 EmitToStreamer(
3421 Inst: MCInstBuilder(AArch64::HINT).addImm(Val: MI->getOperand(i: 4).getImm()));
3422
3423 MCInst Store;
3424 Store.setOpcode(StOpc);
3425 Store.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg())); // Rn
3426 Store.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 1).getReg())); // Data
3427 Store.addOperand(Op: MCOperand::createImm(Val: MI->getOperand(i: 2).getImm())); // Imm
3428 Store.setFlags(MI->getFlags());
3429 EmitToStreamer(S&: *OutStreamer, Inst: Store);
3430}
3431
3432void AArch64AsmPrinter::emitAtomicFetchHintPseudoExpansion(
3433 const MachineInstr *MI) {
3434 EmitToStreamer(
3435 Inst: MCInstBuilder(AArch64::HINT).addImm(Val: MI->getOperand(i: 4).getImm()));
3436
3437 MCInst Fetch;
3438 Fetch.setOpcode(MI->getOperand(i: 3).getImm());
3439 Fetch.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg()));
3440 Fetch.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 1).getReg()));
3441 Fetch.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 2).getReg()));
3442 Fetch.setFlags(MI->getFlags());
3443 EmitToStreamer(S&: *OutStreamer, Inst: Fetch);
3444}
3445
3446// Simple pseudo-instructions have their lowering (with expansion to real
3447// instructions) auto-generated.
3448#include "AArch64GenMCPseudoLowering.inc"
3449
3450void AArch64AsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) {
3451 S.emitInstruction(Inst, STI: *STI);
3452#ifndef NDEBUG
3453 ++InstsEmitted;
3454#endif
3455}
3456
3457void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
3458 AArch64_MC::verifyInstructionPredicates(Opcode: MI->getOpcode(), Features: STI->getFeatureBits());
3459
3460#ifndef NDEBUG
3461 InstsEmitted = 0;
3462 llvm::scope_exit CheckMISize([&]() {
3463 assert(STI->getInstrInfo()->getInstSizeInBytes(*MI) >= InstsEmitted * 4);
3464 });
3465#endif
3466
3467 // Do any auto-generated pseudo lowerings.
3468 if (MCInst OutInst; lowerPseudoInstExpansion(MI, Inst&: OutInst)) {
3469 EmitToStreamer(S&: *OutStreamer, Inst: OutInst);
3470 return;
3471 }
3472
3473 if (MI->getOpcode() == AArch64::ADRP) {
3474 for (auto &Opd : MI->operands()) {
3475 if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) ==
3476 "swift_async_extendedFramePointerFlags") {
3477 ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true;
3478 }
3479 }
3480 }
3481
3482 if (AArch64FI->getLOHRelated().count(Ptr: MI)) {
3483 // Generate a label for LOH related instruction
3484 MCSymbol *LOHLabel = createTempSymbol(Name: "loh");
3485 // Associate the instruction with the label
3486 LOHInstToLabel[MI] = LOHLabel;
3487 OutStreamer->emitLabel(Symbol: LOHLabel);
3488 }
3489
3490 AArch64TargetStreamer *TS =
3491 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
3492 // Do any manual lowerings.
3493 switch (MI->getOpcode()) {
3494 default:
3495 assert(!AArch64InstrInfo::isTailCallReturnInst(*MI) &&
3496 "Unhandled tail call instruction");
3497 break;
3498 case AArch64::READ_REGISTER_GPR64:
3499 // Read of a named GPR: emit "mov Xt, Xn" (ORR Xt, XZR, Xn). The source
3500 // register is encoded as an immediate operand so that earlier passes do not
3501 // see a use of an undefined physical register.
3502 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ORRXrs)
3503 .addReg(Reg: MI->getOperand(i: 0).getReg())
3504 .addReg(Reg: AArch64::XZR)
3505 .addReg(Reg: MI->getOperand(i: 1).getImm())
3506 .addImm(Val: 0));
3507 return;
3508 case AArch64::READ_REGISTER_FPR64:
3509 // Read of a named FP/SIMD d-register: emit "fmov Dt, Dn".
3510 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::FMOVDr)
3511 .addReg(Reg: MI->getOperand(i: 0).getReg())
3512 .addReg(Reg: MI->getOperand(i: 1).getImm()));
3513 return;
3514 case AArch64::HINT: {
3515 // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
3516 // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
3517 // non-empty. If MI is the initial BTI, place the
3518 // __patchable_function_entries label after BTI.
3519 if (CurrentPatchableFunctionEntrySym &&
3520 CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
3521 MI == &MF->front().front()) {
3522 int64_t Imm = MI->getOperand(i: 0).getImm();
3523 if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38) {
3524 MCInst Inst;
3525 MCInstLowering.Lower(MI, OutMI&: Inst);
3526 EmitToStreamer(S&: *OutStreamer, Inst);
3527 CurrentPatchableFunctionEntrySym = createTempSymbol(Name: "patch");
3528 OutStreamer->emitLabel(Symbol: CurrentPatchableFunctionEntrySym);
3529 return;
3530 }
3531 }
3532 break;
3533 }
3534 case AArch64::MOVMCSym: {
3535 Register DestReg = MI->getOperand(i: 0).getReg();
3536 const MachineOperand &MO_Sym = MI->getOperand(i: 1);
3537 MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
3538 MCOperand Hi_MCSym, Lo_MCSym;
3539
3540 Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
3541 Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
3542
3543 MCInstLowering.lowerOperand(MO: Hi_MOSym, MCOp&: Hi_MCSym);
3544 MCInstLowering.lowerOperand(MO: Lo_MOSym, MCOp&: Lo_MCSym);
3545
3546 MCInst MovZ;
3547 MovZ.setOpcode(AArch64::MOVZXi);
3548 MovZ.addOperand(Op: MCOperand::createReg(Reg: DestReg));
3549 MovZ.addOperand(Op: Hi_MCSym);
3550 MovZ.addOperand(Op: MCOperand::createImm(Val: 16));
3551 EmitToStreamer(S&: *OutStreamer, Inst: MovZ);
3552
3553 MCInst MovK;
3554 MovK.setOpcode(AArch64::MOVKXi);
3555 MovK.addOperand(Op: MCOperand::createReg(Reg: DestReg));
3556 MovK.addOperand(Op: MCOperand::createReg(Reg: DestReg));
3557 MovK.addOperand(Op: Lo_MCSym);
3558 MovK.addOperand(Op: MCOperand::createImm(Val: 0));
3559 EmitToStreamer(S&: *OutStreamer, Inst: MovK);
3560 return;
3561 }
3562 case AArch64::MOVIv2d_ns:
3563 // It is generally beneficial to rewrite "fmov s0, wzr" to "movi d0, #0".
3564 // as movi is more efficient across all cores. Newer cores can eliminate
3565 // fmovs early and there is no difference with movi, but this not true for
3566 // all implementations.
3567 //
3568 // The floating-point version doesn't quite work in rare cases on older
3569 // CPUs, so on those targets we lower this instruction to movi.16b instead.
3570 if (STI->hasZeroCycleZeroingFPWorkaround() &&
3571 MI->getOperand(i: 1).getImm() == 0) {
3572 MCInst TmpInst;
3573 TmpInst.setOpcode(AArch64::MOVIv16b_ns);
3574 TmpInst.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg()));
3575 TmpInst.addOperand(Op: MCOperand::createImm(Val: 0));
3576 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
3577 return;
3578 }
3579 break;
3580
3581 case AArch64::DBG_VALUE:
3582 case AArch64::DBG_VALUE_LIST:
3583 if (isVerbose() && OutStreamer->hasRawTextSupport()) {
3584 SmallString<128> TmpStr;
3585 raw_svector_ostream OS(TmpStr);
3586 PrintDebugValueComment(MI, OS);
3587 OutStreamer->emitRawText(String: StringRef(OS.str()));
3588 }
3589 return;
3590
3591 case AArch64::EMITBKEY: {
3592 ExceptionHandling ExceptionHandlingType = MAI.getExceptionHandlingType();
3593 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
3594 ExceptionHandlingType != ExceptionHandling::ARM)
3595 return;
3596
3597 if (getFunctionCFISectionType(MF: *MF) == CFISection::None)
3598 return;
3599
3600 OutStreamer->emitCFIBKeyFrame();
3601 return;
3602 }
3603
3604 case AArch64::EMITMTETAGGED: {
3605 ExceptionHandling ExceptionHandlingType = MAI.getExceptionHandlingType();
3606 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
3607 ExceptionHandlingType != ExceptionHandling::ARM)
3608 return;
3609
3610 if (getFunctionCFISectionType(MF: *MF) != CFISection::None)
3611 OutStreamer->emitCFIMTETaggedFrame();
3612 return;
3613 }
3614
3615 case AArch64::AUTx16x17: {
3616 const Register Pointer = AArch64::X16;
3617 const Register Scratch = AArch64::X17;
3618
3619 auto AuthSchema = PtrAuthSchema::CreateImmReg(
3620 Key: (AArch64PACKey::ID)MI->getOperand(i: 0).getImm(),
3621 IntDisc: MI->getOperand(i: 1).getImm(), AddrDiscOp: MI->getOperand(i: 2));
3622
3623 emitPtrauthAuthResign(Pointer, Scratch, AuthSchema, SignSchema: std::nullopt,
3624 Addend: std::nullopt, DS: MI->getDeactivationSymbol());
3625 return;
3626 }
3627
3628 case AArch64::AUTxMxN: {
3629 const Register Pointer = MI->getOperand(i: 0).getReg();
3630 const Register Scratch = MI->getOperand(i: 1).getReg();
3631
3632 auto AuthSchema = PtrAuthSchema::CreateImmReg(
3633 Key: (AArch64PACKey::ID)MI->getOperand(i: 3).getImm(),
3634 IntDisc: MI->getOperand(i: 4).getImm(), AddrDiscOp: MI->getOperand(i: 5));
3635
3636 emitPtrauthAuthResign(Pointer, Scratch, AuthSchema, SignSchema: std::nullopt,
3637 Addend: std::nullopt, DS: MI->getDeactivationSymbol());
3638 return;
3639 }
3640
3641 case AArch64::AUTPAC: {
3642 const Register Pointer = AArch64::X16;
3643 const Register Scratch = AArch64::X17;
3644
3645 auto AuthSchema = PtrAuthSchema::CreateImmReg(
3646 Key: (AArch64PACKey::ID)MI->getOperand(i: 0).getImm(),
3647 IntDisc: MI->getOperand(i: 1).getImm(), AddrDiscOp: MI->getOperand(i: 2));
3648
3649 auto SignSchema = PtrAuthSchema::CreateImmReg(
3650 Key: (AArch64PACKey::ID)MI->getOperand(i: 3).getImm(),
3651 IntDisc: MI->getOperand(i: 4).getImm(), AddrDiscOp: MI->getOperand(i: 5));
3652
3653 emitPtrauthAuthResign(Pointer, Scratch, AuthSchema, SignSchema,
3654 Addend: std::nullopt, DS: MI->getDeactivationSymbol());
3655 return;
3656 }
3657
3658 case AArch64::AUTPCPAC: {
3659 auto AuthSchema = PtrAuthSchema::CreateRegReg(
3660 Key: (AArch64PACKey::ID)MI->getOperand(i: 0).getImm(), AddrDisc: AArch64::X16,
3661 PCDisc: AArch64::X15);
3662
3663 auto SignSchema = PtrAuthSchema::CreateImmReg(
3664 Key: (AArch64PACKey::ID)MI->getOperand(i: 1).getImm(),
3665 IntDisc: MI->getOperand(i: 2).getImm(), AddrDiscOp: MI->getOperand(i: 3));
3666
3667 emitPtrauthAuthResign(/*Pointer=*/AArch64::X17, /*Scratch=*/AArch64::X16,
3668 AuthSchema, SignSchema, Addend: std::nullopt,
3669 DS: MI->getDeactivationSymbol());
3670 return;
3671 }
3672
3673 case AArch64::AUTRELLOADPAC: {
3674 const Register Pointer = AArch64::X16;
3675 const Register Scratch = AArch64::X17;
3676
3677 auto AuthSchema = PtrAuthSchema::CreateImmReg(
3678 Key: (AArch64PACKey::ID)MI->getOperand(i: 0).getImm(),
3679 IntDisc: MI->getOperand(i: 1).getImm(), AddrDiscOp: MI->getOperand(i: 2));
3680
3681 auto SignSchema = PtrAuthSchema::CreateImmReg(
3682 Key: (AArch64PACKey::ID)MI->getOperand(i: 3).getImm(),
3683 IntDisc: MI->getOperand(i: 4).getImm(), AddrDiscOp: MI->getOperand(i: 5));
3684
3685 emitPtrauthAuthResign(Pointer, Scratch, AuthSchema, SignSchema,
3686 Addend: MI->getOperand(i: 6).getImm(),
3687 DS: MI->getDeactivationSymbol());
3688
3689 return;
3690 }
3691
3692 case AArch64::PAC:
3693 emitPtrauthSign(MI);
3694 return;
3695
3696 case AArch64::LOADauthptrstatic:
3697 LowerLOADauthptrstatic(MI: *MI);
3698 return;
3699
3700 case AArch64::LOADgotPAC:
3701 case AArch64::MOVaddrPAC:
3702 LowerMOVaddrPAC(MI: *MI);
3703 return;
3704
3705 case AArch64::LOADgotAUTH:
3706 LowerLOADgotAUTH(MI: *MI);
3707 return;
3708
3709 case AArch64::BRA:
3710 case AArch64::BLRA:
3711 emitPtrauthBranch(MI);
3712 return;
3713
3714 // Tail calls use pseudo instructions so they have the proper code-gen
3715 // attributes (isCall, isReturn, etc.). We lower them to the real
3716 // instruction here.
3717 case AArch64::AUTH_TCRETURN:
3718 case AArch64::AUTH_TCRETURN_BTI: {
3719 Register Callee = MI->getOperand(i: 0).getReg();
3720 const auto Key = (AArch64PACKey::ID)MI->getOperand(i: 2).getImm();
3721 const uint64_t Disc = MI->getOperand(i: 3).getImm();
3722
3723 Register AddrDisc = MI->getOperand(i: 4).getReg();
3724
3725 Register ScratchReg = Callee == AArch64::X16 ? AArch64::X17 : AArch64::X16;
3726
3727 emitPtrauthTailCallHardening(TC: MI);
3728
3729 // See the comments in emitPtrauthBranch.
3730 if (Callee == AddrDisc)
3731 report_fatal_error(reason: "Call target is signed with its own value");
3732
3733 // After isX16X17Safer predicate was introduced, emitPtrauthDiscriminator is
3734 // no longer restricted to only reusing AddrDisc when it is X16 or X17
3735 // (which are implicit-def'ed by AUTH_TCRETURN pseudos), thus impose this
3736 // restriction manually not to clobber an unexpected register.
3737 bool AddrDiscIsImplicitDef =
3738 AddrDisc == AArch64::X16 || AddrDisc == AArch64::X17;
3739 Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, ScratchReg,
3740 MayClobberAddrDisc: AddrDiscIsImplicitDef);
3741 emitBLRA(/*IsCall*/ false, Key, Target: Callee, Disc: DiscReg);
3742 return;
3743 }
3744
3745 case AArch64::TCRETURNri:
3746 case AArch64::TCRETURNrix16x17:
3747 case AArch64::TCRETURNrix17:
3748 case AArch64::TCRETURNrinotx16:
3749 case AArch64::TCRETURNriALL: {
3750 emitPtrauthTailCallHardening(TC: MI);
3751
3752 recordIfImportCall(BranchInst: MI);
3753 MCInst TmpInst;
3754 TmpInst.setOpcode(AArch64::BR);
3755 TmpInst.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg()));
3756 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
3757 return;
3758 }
3759 case AArch64::TCRETURNdi: {
3760 emitPtrauthTailCallHardening(TC: MI);
3761
3762 MCOperand Dest;
3763 MCInstLowering.lowerOperand(MO: MI->getOperand(i: 0), MCOp&: Dest);
3764 recordIfImportCall(BranchInst: MI);
3765 MCInst TmpInst;
3766 TmpInst.setOpcode(AArch64::B);
3767 TmpInst.addOperand(Op: Dest);
3768 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
3769 return;
3770 }
3771 case AArch64::SpeculationBarrierISBDSBEndBB: {
3772 // Print DSB SYS + ISB
3773 MCInst TmpInstDSB;
3774 TmpInstDSB.setOpcode(AArch64::DSB);
3775 TmpInstDSB.addOperand(Op: MCOperand::createImm(Val: 0xf));
3776 EmitToStreamer(S&: *OutStreamer, Inst: TmpInstDSB);
3777 MCInst TmpInstISB;
3778 TmpInstISB.setOpcode(AArch64::ISB);
3779 TmpInstISB.addOperand(Op: MCOperand::createImm(Val: 0xf));
3780 EmitToStreamer(S&: *OutStreamer, Inst: TmpInstISB);
3781 return;
3782 }
3783 case AArch64::SpeculationBarrierSBEndBB: {
3784 // Print SB
3785 MCInst TmpInstSB;
3786 TmpInstSB.setOpcode(AArch64::SB);
3787 EmitToStreamer(S&: *OutStreamer, Inst: TmpInstSB);
3788 return;
3789 }
3790 case AArch64::TLSDESC_AUTH_CALLSEQ: {
3791 /// lower this to:
3792 /// adrp x0, :tlsdesc_auth:var
3793 /// ldr x16, [x0, #:tlsdesc_auth_lo12:var]
3794 /// add x0, x0, #:tlsdesc_auth_lo12:var
3795 /// .tlsauthdesccall var
3796 /// blraa x16, x0
3797 /// (TPIDR_EL0 offset now in x0)
3798 const MachineOperand &MO_Sym = MI->getOperand(i: 0);
3799 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
3800 MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
3801 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
3802 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
3803 MCInstLowering.lowerOperand(MO: MO_Sym, MCOp&: Sym);
3804 MCInstLowering.lowerOperand(MO: MO_TLSDESC_LO12, MCOp&: SymTLSDescLo12);
3805 MCInstLowering.lowerOperand(MO: MO_TLSDESC, MCOp&: SymTLSDesc);
3806
3807 MCInst Adrp;
3808 Adrp.setOpcode(AArch64::ADRP);
3809 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3810 Adrp.addOperand(Op: SymTLSDesc);
3811 EmitToStreamer(S&: *OutStreamer, Inst: Adrp);
3812
3813 MCInst Ldr;
3814 Ldr.setOpcode(AArch64::LDRXui);
3815 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
3816 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3817 Ldr.addOperand(Op: SymTLSDescLo12);
3818 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
3819 EmitToStreamer(S&: *OutStreamer, Inst: Ldr);
3820
3821 MCInst Add;
3822 Add.setOpcode(AArch64::ADDXri);
3823 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3824 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3825 Add.addOperand(Op: SymTLSDescLo12);
3826 Add.addOperand(Op: MCOperand::createImm(Val: AArch64_AM::getShiftValue(Imm: 0)));
3827 EmitToStreamer(S&: *OutStreamer, Inst: Add);
3828
3829 // Emit a relocation-annotation. This expands to no code, but requests
3830 // the following instruction gets an R_AARCH64_AUTH_TLSDESC_CALL.
3831 MCInst TLSAuthDescCall;
3832 TLSAuthDescCall.setOpcode(AArch64::TLSAUTHDESCCALL);
3833 TLSAuthDescCall.addOperand(Op: Sym);
3834 EmitToStreamer(S&: *OutStreamer, Inst: TLSAuthDescCall);
3835#ifndef NDEBUG
3836 --InstsEmitted; // no code emitted
3837#endif
3838
3839 MCInst Blraa;
3840 Blraa.setOpcode(AArch64::BLRAA);
3841 Blraa.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
3842 Blraa.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3843 EmitToStreamer(S&: *OutStreamer, Inst: Blraa);
3844
3845 return;
3846 }
3847 case AArch64::TLSDESC_CALLSEQ: {
3848 /// lower this to:
3849 /// adrp x0, :tlsdesc:var
3850 /// ldr x1, [x0, #:tlsdesc_lo12:var]
3851 /// add x0, x0, #:tlsdesc_lo12:var
3852 /// .tlsdesccall var
3853 /// blr x1
3854 /// (TPIDR_EL0 offset now in x0)
3855 const MachineOperand &MO_Sym = MI->getOperand(i: 0);
3856 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
3857 MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
3858 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
3859 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
3860 MCInstLowering.lowerOperand(MO: MO_Sym, MCOp&: Sym);
3861 MCInstLowering.lowerOperand(MO: MO_TLSDESC_LO12, MCOp&: SymTLSDescLo12);
3862 MCInstLowering.lowerOperand(MO: MO_TLSDESC, MCOp&: SymTLSDesc);
3863
3864 MCInst Adrp;
3865 Adrp.setOpcode(AArch64::ADRP);
3866 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3867 Adrp.addOperand(Op: SymTLSDesc);
3868 EmitToStreamer(S&: *OutStreamer, Inst: Adrp);
3869
3870 MCInst Ldr;
3871 if (STI->isTargetILP32()) {
3872 Ldr.setOpcode(AArch64::LDRWui);
3873 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::W1));
3874 } else {
3875 Ldr.setOpcode(AArch64::LDRXui);
3876 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X1));
3877 }
3878 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3879 Ldr.addOperand(Op: SymTLSDescLo12);
3880 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
3881 EmitToStreamer(S&: *OutStreamer, Inst: Ldr);
3882
3883 MCInst Add;
3884 if (STI->isTargetILP32()) {
3885 Add.setOpcode(AArch64::ADDWri);
3886 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::W0));
3887 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::W0));
3888 } else {
3889 Add.setOpcode(AArch64::ADDXri);
3890 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3891 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3892 }
3893 Add.addOperand(Op: SymTLSDescLo12);
3894 Add.addOperand(Op: MCOperand::createImm(Val: AArch64_AM::getShiftValue(Imm: 0)));
3895 EmitToStreamer(S&: *OutStreamer, Inst: Add);
3896
3897 // Emit a relocation-annotation. This expands to no code, but requests
3898 // the following instruction gets an R_AARCH64_TLSDESC_CALL.
3899 MCInst TLSDescCall;
3900 TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
3901 TLSDescCall.addOperand(Op: Sym);
3902 EmitToStreamer(S&: *OutStreamer, Inst: TLSDescCall);
3903#ifndef NDEBUG
3904 --InstsEmitted; // no code emitted
3905#endif
3906
3907 MCInst Blr;
3908 Blr.setOpcode(AArch64::BLR);
3909 Blr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X1));
3910 EmitToStreamer(S&: *OutStreamer, Inst: Blr);
3911
3912 return;
3913 }
3914
3915 case AArch64::JumpTableDest32:
3916 case AArch64::JumpTableDest16:
3917 case AArch64::JumpTableDest8:
3918 LowerJumpTableDest(OutStreamer&: *OutStreamer, MI: *MI);
3919 return;
3920
3921 case AArch64::BR_JumpTable:
3922 LowerHardenedBRJumpTable(MI: *MI);
3923 return;
3924
3925 case AArch64::FMOVH0:
3926 case AArch64::FMOVS0:
3927 case AArch64::FMOVD0:
3928 emitFMov0(MI: *MI);
3929 return;
3930
3931 case AArch64::MOPSMemoryCopyPseudo:
3932 case AArch64::MOPSMemoryMovePseudo:
3933 case AArch64::MOPSMemorySetPseudo:
3934 case AArch64::MOPSMemorySetTaggingPseudo:
3935 LowerMOPS(OutStreamer&: *OutStreamer, MI: *MI);
3936 return;
3937
3938 case TargetOpcode::STACKMAP:
3939 return LowerSTACKMAP(OutStreamer&: *OutStreamer, SM, MI: *MI);
3940
3941 case TargetOpcode::PATCHPOINT:
3942 return LowerPATCHPOINT(OutStreamer&: *OutStreamer, SM, MI: *MI);
3943
3944 case TargetOpcode::STATEPOINT:
3945 return LowerSTATEPOINT(OutStreamer&: *OutStreamer, SM, MI: *MI);
3946
3947 case TargetOpcode::FAULTING_OP:
3948 return LowerFAULTING_OP(FaultingMI: *MI);
3949
3950 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
3951 LowerPATCHABLE_FUNCTION_ENTER(MI: *MI);
3952 return;
3953
3954 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
3955 LowerPATCHABLE_FUNCTION_EXIT(MI: *MI);
3956 return;
3957
3958 case TargetOpcode::PATCHABLE_TAIL_CALL:
3959 LowerPATCHABLE_TAIL_CALL(MI: *MI);
3960 return;
3961 case TargetOpcode::PATCHABLE_EVENT_CALL:
3962 return LowerPATCHABLE_EVENT_CALL(MI: *MI, Typed: false);
3963 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
3964 return LowerPATCHABLE_EVENT_CALL(MI: *MI, Typed: true);
3965
3966 case AArch64::KCFI_CHECK:
3967 LowerKCFI_CHECK(MI: *MI);
3968 return;
3969
3970 case AArch64::HWASAN_CHECK_MEMACCESS:
3971 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
3972 case AArch64::HWASAN_CHECK_MEMACCESS_FIXEDSHADOW:
3973 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW:
3974 LowerHWASAN_CHECK_MEMACCESS(MI: *MI);
3975 return;
3976
3977 case AArch64::SEH_StackAlloc:
3978 TS->emitARM64WinCFIAllocStack(Size: MI->getOperand(i: 0).getImm());
3979 return;
3980
3981 case AArch64::SEH_SaveFPLR:
3982 TS->emitARM64WinCFISaveFPLR(Offset: MI->getOperand(i: 0).getImm());
3983 return;
3984
3985 case AArch64::SEH_SaveFPLR_X:
3986 assert(MI->getOperand(0).getImm() < 0 &&
3987 "Pre increment SEH opcode must have a negative offset");
3988 TS->emitARM64WinCFISaveFPLRX(Offset: -MI->getOperand(i: 0).getImm());
3989 return;
3990
3991 case AArch64::SEH_SaveReg:
3992 TS->emitARM64WinCFISaveReg(Reg: MI->getOperand(i: 0).getImm(),
3993 Offset: MI->getOperand(i: 1).getImm());
3994 return;
3995
3996 case AArch64::SEH_SaveReg_X:
3997 assert(MI->getOperand(1).getImm() < 0 &&
3998 "Pre increment SEH opcode must have a negative offset");
3999 TS->emitARM64WinCFISaveRegX(Reg: MI->getOperand(i: 0).getImm(),
4000 Offset: -MI->getOperand(i: 1).getImm());
4001 return;
4002
4003 case AArch64::SEH_SaveRegP:
4004 if (MI->getOperand(i: 1).getImm() == 30 && MI->getOperand(i: 0).getImm() >= 19 &&
4005 MI->getOperand(i: 0).getImm() <= 28) {
4006 assert((MI->getOperand(0).getImm() - 19) % 2 == 0 &&
4007 "Register paired with LR must be odd");
4008 TS->emitARM64WinCFISaveLRPair(Reg: MI->getOperand(i: 0).getImm(),
4009 Offset: MI->getOperand(i: 2).getImm());
4010 return;
4011 }
4012 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
4013 "Non-consecutive registers not allowed for save_regp");
4014 TS->emitARM64WinCFISaveRegP(Reg: MI->getOperand(i: 0).getImm(),
4015 Offset: MI->getOperand(i: 2).getImm());
4016 return;
4017
4018 case AArch64::SEH_SaveRegP_X:
4019 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
4020 "Non-consecutive registers not allowed for save_regp_x");
4021 assert(MI->getOperand(2).getImm() < 0 &&
4022 "Pre increment SEH opcode must have a negative offset");
4023 TS->emitARM64WinCFISaveRegPX(Reg: MI->getOperand(i: 0).getImm(),
4024 Offset: -MI->getOperand(i: 2).getImm());
4025 return;
4026
4027 case AArch64::SEH_SaveFReg:
4028 TS->emitARM64WinCFISaveFReg(Reg: MI->getOperand(i: 0).getImm(),
4029 Offset: MI->getOperand(i: 1).getImm());
4030 return;
4031
4032 case AArch64::SEH_SaveFReg_X:
4033 assert(MI->getOperand(1).getImm() < 0 &&
4034 "Pre increment SEH opcode must have a negative offset");
4035 TS->emitARM64WinCFISaveFRegX(Reg: MI->getOperand(i: 0).getImm(),
4036 Offset: -MI->getOperand(i: 1).getImm());
4037 return;
4038
4039 case AArch64::SEH_SaveFRegP:
4040 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
4041 "Non-consecutive registers not allowed for save_regp");
4042 TS->emitARM64WinCFISaveFRegP(Reg: MI->getOperand(i: 0).getImm(),
4043 Offset: MI->getOperand(i: 2).getImm());
4044 return;
4045
4046 case AArch64::SEH_SaveFRegP_X:
4047 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
4048 "Non-consecutive registers not allowed for save_regp_x");
4049 assert(MI->getOperand(2).getImm() < 0 &&
4050 "Pre increment SEH opcode must have a negative offset");
4051 TS->emitARM64WinCFISaveFRegPX(Reg: MI->getOperand(i: 0).getImm(),
4052 Offset: -MI->getOperand(i: 2).getImm());
4053 return;
4054
4055 case AArch64::SEH_SetFP:
4056 TS->emitARM64WinCFISetFP();
4057 return;
4058
4059 case AArch64::SEH_AddFP:
4060 TS->emitARM64WinCFIAddFP(Size: MI->getOperand(i: 0).getImm());
4061 return;
4062
4063 case AArch64::SEH_Nop:
4064 TS->emitARM64WinCFINop();
4065 return;
4066
4067 case AArch64::SEH_PrologEnd:
4068 TS->emitARM64WinCFIPrologEnd();
4069 return;
4070
4071 case AArch64::SEH_EpilogStart:
4072 TS->emitARM64WinCFIEpilogStart();
4073 return;
4074
4075 case AArch64::SEH_EpilogEnd:
4076 TS->emitARM64WinCFIEpilogEnd();
4077 return;
4078
4079 case AArch64::SEH_PACSignLR:
4080 TS->emitARM64WinCFIPACSignLR();
4081 return;
4082
4083 case AArch64::SEH_SaveAnyRegI:
4084 assert(MI->getOperand(1).getImm() <= 1008 &&
4085 "SaveAnyRegQP SEH opcode offset must fit into 6 bits");
4086 TS->emitARM64WinCFISaveAnyRegI(Reg: MI->getOperand(i: 0).getImm(),
4087 Offset: MI->getOperand(i: 1).getImm());
4088 return;
4089
4090 case AArch64::SEH_SaveAnyRegIP:
4091 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
4092 "Non-consecutive registers not allowed for save_any_reg");
4093 assert(MI->getOperand(2).getImm() <= 1008 &&
4094 "SaveAnyRegQP SEH opcode offset must fit into 6 bits");
4095 TS->emitARM64WinCFISaveAnyRegIP(Reg: MI->getOperand(i: 0).getImm(),
4096 Offset: MI->getOperand(i: 2).getImm());
4097 return;
4098
4099 case AArch64::SEH_SaveAnyRegQP:
4100 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
4101 "Non-consecutive registers not allowed for save_any_reg");
4102 assert(MI->getOperand(2).getImm() >= 0 &&
4103 "SaveAnyRegQP SEH opcode offset must be non-negative");
4104 assert(MI->getOperand(2).getImm() <= 1008 &&
4105 "SaveAnyRegQP SEH opcode offset must fit into 6 bits");
4106 TS->emitARM64WinCFISaveAnyRegQP(Reg: MI->getOperand(i: 0).getImm(),
4107 Offset: MI->getOperand(i: 2).getImm());
4108 return;
4109
4110 case AArch64::SEH_SaveAnyRegQPX:
4111 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
4112 "Non-consecutive registers not allowed for save_any_reg");
4113 assert(MI->getOperand(2).getImm() < 0 &&
4114 "SaveAnyRegQPX SEH opcode offset must be negative");
4115 assert(MI->getOperand(2).getImm() >= -1008 &&
4116 "SaveAnyRegQPX SEH opcode offset must fit into 6 bits");
4117 TS->emitARM64WinCFISaveAnyRegQPX(Reg: MI->getOperand(i: 0).getImm(),
4118 Offset: -MI->getOperand(i: 2).getImm());
4119 return;
4120
4121 case AArch64::SEH_AllocZ:
4122 assert(MI->getOperand(0).getImm() >= 0 &&
4123 "AllocZ SEH opcode offset must be non-negative");
4124 assert(MI->getOperand(0).getImm() <= 255 &&
4125 "AllocZ SEH opcode offset must fit into 8 bits");
4126 TS->emitARM64WinCFIAllocZ(Offset: MI->getOperand(i: 0).getImm());
4127 return;
4128
4129 case AArch64::SEH_SaveZReg:
4130 assert(MI->getOperand(1).getImm() >= 0 &&
4131 "SaveZReg SEH opcode offset must be non-negative");
4132 assert(MI->getOperand(1).getImm() <= 255 &&
4133 "SaveZReg SEH opcode offset must fit into 8 bits");
4134 TS->emitARM64WinCFISaveZReg(Reg: MI->getOperand(i: 0).getImm(),
4135 Offset: MI->getOperand(i: 1).getImm());
4136 return;
4137
4138 case AArch64::SEH_SavePReg:
4139 assert(MI->getOperand(1).getImm() >= 0 &&
4140 "SavePReg SEH opcode offset must be non-negative");
4141 assert(MI->getOperand(1).getImm() <= 255 &&
4142 "SavePReg SEH opcode offset must fit into 8 bits");
4143 TS->emitARM64WinCFISavePReg(Reg: MI->getOperand(i: 0).getImm(),
4144 Offset: MI->getOperand(i: 1).getImm());
4145 return;
4146
4147 case AArch64::BLR:
4148 case AArch64::BR: {
4149 recordIfImportCall(BranchInst: MI);
4150 MCInst TmpInst;
4151 MCInstLowering.Lower(MI, OutMI&: TmpInst);
4152 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
4153 return;
4154 }
4155 case AArch64::CBWPri:
4156 case AArch64::CBXPri:
4157 case AArch64::CBBAssertExt:
4158 case AArch64::CBHAssertExt:
4159 case AArch64::CBWPrr:
4160 case AArch64::CBXPrr:
4161 emitCBPseudoExpansion(MI);
4162 return;
4163 case AArch64::ATOMIC_STORE_HINT_B:
4164 case AArch64::ATOMIC_STORE_HINT_H:
4165 case AArch64::ATOMIC_STORE_HINT_W:
4166 case AArch64::ATOMIC_STORE_HINT_X:
4167 emitAtomicHintPseudoExpansion(MI);
4168 return;
4169 case AArch64::ATOMIC_FETCH_HINT_W:
4170 case AArch64::ATOMIC_FETCH_HINT_X:
4171 emitAtomicFetchHintPseudoExpansion(MI);
4172 return;
4173 case AArch64::ATOMIC_STORE_HINT_BroW:
4174 case AArch64::ATOMIC_STORE_HINT_HroW:
4175 case AArch64::ATOMIC_STORE_HINT_WroW:
4176 case AArch64::ATOMIC_STORE_HINT_XroW:
4177 case AArch64::ATOMIC_STORE_HINT_SroW:
4178 case AArch64::ATOMIC_STORE_HINT_DroW:
4179 case AArch64::ATOMIC_STORE_HINT_BroX:
4180 case AArch64::ATOMIC_STORE_HINT_HroX:
4181 case AArch64::ATOMIC_STORE_HINT_WroX:
4182 case AArch64::ATOMIC_STORE_HINT_XroX:
4183 case AArch64::ATOMIC_STORE_HINT_SroX:
4184 case AArch64::ATOMIC_STORE_HINT_DroX:
4185 emitAtomicHintPseudoExpansionRO(MI);
4186 return;
4187 case AArch64::ATOMIC_STORE_HINT_Bui:
4188 case AArch64::ATOMIC_STORE_HINT_Hui:
4189 case AArch64::ATOMIC_STORE_HINT_Wui:
4190 case AArch64::ATOMIC_STORE_HINT_Xui:
4191 case AArch64::ATOMIC_STORE_HINT_Sui:
4192 case AArch64::ATOMIC_STORE_HINT_Dui:
4193 case AArch64::ATOMIC_STORE_HINT_Bi:
4194 case AArch64::ATOMIC_STORE_HINT_Hi:
4195 case AArch64::ATOMIC_STORE_HINT_Wi:
4196 case AArch64::ATOMIC_STORE_HINT_Xi:
4197 case AArch64::ATOMIC_STORE_HINT_Si:
4198 case AArch64::ATOMIC_STORE_HINT_Di:
4199 emitAtomicHintPseudoExpansionImm(MI);
4200 return;
4201 }
4202
4203 if (emitDeactivationSymbolRelocation(DS: MI->getDeactivationSymbol()))
4204 return;
4205
4206 // Finally, do the automated lowerings for everything else.
4207 MCInst TmpInst;
4208 MCInstLowering.Lower(MI, OutMI&: TmpInst);
4209 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
4210}
4211
4212void AArch64AsmPrinter::recordIfImportCall(
4213 const llvm::MachineInstr *BranchInst) {
4214 if (!EnableImportCallOptimization)
4215 return;
4216
4217 auto [GV, OpFlags] = BranchInst->getMF()->tryGetCalledGlobal(MI: BranchInst);
4218 if (GV && GV->hasDLLImportStorageClass()) {
4219 auto *CallSiteSymbol = MMI->getContext().createNamedTempSymbol(Name: "impcall");
4220 OutStreamer->emitLabel(Symbol: CallSiteSymbol);
4221
4222 auto *CalledSymbol = MCInstLowering.GetGlobalValueSymbol(GV, TargetFlags: OpFlags);
4223 SectionToImportedFunctionCalls[OutStreamer->getCurrentSectionOnly()]
4224 .push_back(x: {CallSiteSymbol, CalledSymbol});
4225 }
4226}
4227
4228void AArch64AsmPrinter::emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI,
4229 MCSymbol *LazyPointer) {
4230 // _ifunc:
4231 // adrp x16, lazy_pointer@GOTPAGE
4232 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF]
4233 // ldr x16, [x16]
4234 // br x16
4235
4236 {
4237 MCInst Adrp;
4238 Adrp.setOpcode(AArch64::ADRP);
4239 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
4240 MCOperand SymPage;
4241 MCInstLowering.lowerOperand(
4242 MO: MachineOperand::CreateMCSymbol(Sym: LazyPointer,
4243 TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGE),
4244 MCOp&: SymPage);
4245 Adrp.addOperand(Op: SymPage);
4246 EmitToStreamer(Inst: Adrp);
4247 }
4248
4249 {
4250 MCInst Ldr;
4251 Ldr.setOpcode(AArch64::LDRXui);
4252 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
4253 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
4254 MCOperand SymPageOff;
4255 MCInstLowering.lowerOperand(
4256 MO: MachineOperand::CreateMCSymbol(Sym: LazyPointer, TargetFlags: AArch64II::MO_GOT |
4257 AArch64II::MO_PAGEOFF),
4258 MCOp&: SymPageOff);
4259 Ldr.addOperand(Op: SymPageOff);
4260 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
4261 EmitToStreamer(Inst: Ldr);
4262 }
4263
4264 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
4265 .addReg(Reg: AArch64::X16)
4266 .addReg(Reg: AArch64::X16)
4267 .addImm(Val: 0));
4268
4269 EmitToStreamer(Inst: MCInstBuilder(TM.getTargetTriple().isArm64e() ? AArch64::BRAAZ
4270 : AArch64::BR)
4271 .addReg(Reg: AArch64::X16));
4272}
4273
4274void AArch64AsmPrinter::emitMachOIFuncStubHelperBody(Module &M,
4275 const GlobalIFunc &GI,
4276 MCSymbol *LazyPointer) {
4277 // These stub helpers are only ever called once, so here we're optimizing for
4278 // minimum size by using the pre-indexed store variants, which saves a few
4279 // bytes of instructions to bump & restore sp.
4280
4281 // _ifunc.stub_helper:
4282 // stp fp, lr, [sp, #-16]!
4283 // mov fp, sp
4284 // stp x1, x0, [sp, #-16]!
4285 // stp x3, x2, [sp, #-16]!
4286 // stp x5, x4, [sp, #-16]!
4287 // stp x7, x6, [sp, #-16]!
4288 // stp d1, d0, [sp, #-16]!
4289 // stp d3, d2, [sp, #-16]!
4290 // stp d5, d4, [sp, #-16]!
4291 // stp d7, d6, [sp, #-16]!
4292 // bl _resolver
4293 // adrp x16, lazy_pointer@GOTPAGE
4294 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF]
4295 // str x0, [x16]
4296 // mov x16, x0
4297 // ldp d7, d6, [sp], #16
4298 // ldp d5, d4, [sp], #16
4299 // ldp d3, d2, [sp], #16
4300 // ldp d1, d0, [sp], #16
4301 // ldp x7, x6, [sp], #16
4302 // ldp x5, x4, [sp], #16
4303 // ldp x3, x2, [sp], #16
4304 // ldp x1, x0, [sp], #16
4305 // ldp fp, lr, [sp], #16
4306 // br x16
4307
4308 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPXpre)
4309 .addReg(Reg: AArch64::SP)
4310 .addReg(Reg: AArch64::FP)
4311 .addReg(Reg: AArch64::LR)
4312 .addReg(Reg: AArch64::SP)
4313 .addImm(Val: -2));
4314
4315 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
4316 .addReg(Reg: AArch64::FP)
4317 .addReg(Reg: AArch64::SP)
4318 .addImm(Val: 0)
4319 .addImm(Val: 0));
4320
4321 for (int I = 0; I != 4; ++I)
4322 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPXpre)
4323 .addReg(Reg: AArch64::SP)
4324 .addReg(Reg: AArch64::X1 + 2 * I)
4325 .addReg(Reg: AArch64::X0 + 2 * I)
4326 .addReg(Reg: AArch64::SP)
4327 .addImm(Val: -2));
4328
4329 for (int I = 0; I != 4; ++I)
4330 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPDpre)
4331 .addReg(Reg: AArch64::SP)
4332 .addReg(Reg: AArch64::D1 + 2 * I)
4333 .addReg(Reg: AArch64::D0 + 2 * I)
4334 .addReg(Reg: AArch64::SP)
4335 .addImm(Val: -2));
4336
4337 EmitToStreamer(
4338 Inst: MCInstBuilder(AArch64::BL)
4339 .addOperand(Op: MCOperand::createExpr(Val: lowerConstant(CV: GI.getResolver()))));
4340
4341 {
4342 MCInst Adrp;
4343 Adrp.setOpcode(AArch64::ADRP);
4344 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
4345 MCOperand SymPage;
4346 MCInstLowering.lowerOperand(
4347 MO: MachineOperand::CreateES(SymName: LazyPointer->getName().data() + 1,
4348 TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGE),
4349 MCOp&: SymPage);
4350 Adrp.addOperand(Op: SymPage);
4351 EmitToStreamer(Inst: Adrp);
4352 }
4353
4354 {
4355 MCInst Ldr;
4356 Ldr.setOpcode(AArch64::LDRXui);
4357 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
4358 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
4359 MCOperand SymPageOff;
4360 MCInstLowering.lowerOperand(
4361 MO: MachineOperand::CreateES(SymName: LazyPointer->getName().data() + 1,
4362 TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGEOFF),
4363 MCOp&: SymPageOff);
4364 Ldr.addOperand(Op: SymPageOff);
4365 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
4366 EmitToStreamer(Inst: Ldr);
4367 }
4368
4369 EmitToStreamer(Inst: MCInstBuilder(AArch64::STRXui)
4370 .addReg(Reg: AArch64::X0)
4371 .addReg(Reg: AArch64::X16)
4372 .addImm(Val: 0));
4373
4374 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
4375 .addReg(Reg: AArch64::X16)
4376 .addReg(Reg: AArch64::X0)
4377 .addImm(Val: 0)
4378 .addImm(Val: 0));
4379
4380 for (int I = 3; I != -1; --I)
4381 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDPDpost)
4382 .addReg(Reg: AArch64::SP)
4383 .addReg(Reg: AArch64::D1 + 2 * I)
4384 .addReg(Reg: AArch64::D0 + 2 * I)
4385 .addReg(Reg: AArch64::SP)
4386 .addImm(Val: 2));
4387
4388 for (int I = 3; I != -1; --I)
4389 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDPXpost)
4390 .addReg(Reg: AArch64::SP)
4391 .addReg(Reg: AArch64::X1 + 2 * I)
4392 .addReg(Reg: AArch64::X0 + 2 * I)
4393 .addReg(Reg: AArch64::SP)
4394 .addImm(Val: 2));
4395
4396 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDPXpost)
4397 .addReg(Reg: AArch64::SP)
4398 .addReg(Reg: AArch64::FP)
4399 .addReg(Reg: AArch64::LR)
4400 .addReg(Reg: AArch64::SP)
4401 .addImm(Val: 2));
4402
4403 EmitToStreamer(Inst: MCInstBuilder(TM.getTargetTriple().isArm64e() ? AArch64::BRAAZ
4404 : AArch64::BR)
4405 .addReg(Reg: AArch64::X16));
4406}
4407
4408const MCExpr *AArch64AsmPrinter::lowerConstant(const Constant *CV,
4409 const Constant *BaseCV,
4410 uint64_t Offset) {
4411 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: CV)) {
4412 return MCSymbolRefExpr::create(Symbol: MCInstLowering.GetGlobalValueSymbol(GV, TargetFlags: 0),
4413 Ctx&: OutContext);
4414 }
4415
4416 return AsmPrinter::lowerConstant(CV, BaseCV, Offset);
4417}
4418
4419char AArch64AsmPrinter::ID = 0;
4420
4421INITIALIZE_PASS(AArch64AsmPrinter, "aarch64-asm-printer",
4422 "AArch64 Assembly Printer", false, false)
4423
4424// Force static initialization.
4425extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
4426LLVMInitializeAArch64AsmPrinter() {
4427 RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
4428 RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
4429 RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
4430 RegisterAsmPrinter<AArch64AsmPrinter> W(getTheARM64_32Target());
4431 RegisterAsmPrinter<AArch64AsmPrinter> V(getTheAArch64_32Target());
4432}
4433
4434PreservedAnalyses AArch64AsmPrinterBeginPass::run(Module &M,
4435 ModuleAnalysisManager &MAM) {
4436 AArch64AsmPrinter &AsmPrinter = static_cast<AArch64AsmPrinter &>(
4437 MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter());
4438 setupModuleAsmPrinter(M, MAM, AsmPrinter);
4439 AsmPrinter.doInitialization(M);
4440 return PreservedAnalyses::all();
4441}
4442
4443PreservedAnalyses
4444AArch64AsmPrinterPass::run(MachineFunction &MF,
4445 MachineFunctionAnalysisManager &MFAM) {
4446 AArch64AsmPrinter &AsmPrinter = static_cast<AArch64AsmPrinter &>(
4447 MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(IR&: MF)
4448 .getCachedResult<AsmPrinterAnalysis>(IR&: *MF.getFunction().getParent())
4449 ->getPrinter());
4450 setupMachineFunctionAsmPrinter(MFAM, MF, AsmPrinter);
4451 AsmPrinter.runOnMachineFunction(MF);
4452 return PreservedAnalyses::all();
4453}
4454
4455PreservedAnalyses AArch64AsmPrinterEndPass::run(Module &M,
4456 ModuleAnalysisManager &MAM) {
4457 AArch64AsmPrinter &AsmPrinter = static_cast<AArch64AsmPrinter &>(
4458 MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter());
4459 setupModuleAsmPrinter(M, MAM, AsmPrinter);
4460 AsmPrinter.doFinalization(M);
4461 return PreservedAnalyses::all();
4462}
4463