1//===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a printer that converts from our internal representation
10// of machine-dependent LLVM code to the AArch64 assembly language.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AArch64.h"
15#include "AArch64MCInstLower.h"
16#include "AArch64MachineFunctionInfo.h"
17#include "AArch64RegisterInfo.h"
18#include "AArch64Subtarget.h"
19#include "AArch64TargetObjectFile.h"
20#include "MCTargetDesc/AArch64AddressingModes.h"
21#include "MCTargetDesc/AArch64InstPrinter.h"
22#include "MCTargetDesc/AArch64MCAsmInfo.h"
23#include "MCTargetDesc/AArch64MCTargetDesc.h"
24#include "MCTargetDesc/AArch64TargetStreamer.h"
25#include "TargetInfo/AArch64TargetInfo.h"
26#include "Utils/AArch64BaseInfo.h"
27#include "llvm/ADT/DenseMap.h"
28#include "llvm/ADT/ScopeExit.h"
29#include "llvm/ADT/SmallString.h"
30#include "llvm/ADT/SmallVector.h"
31#include "llvm/ADT/Statistic.h"
32#include "llvm/ADT/StringRef.h"
33#include "llvm/ADT/Twine.h"
34#include "llvm/BinaryFormat/COFF.h"
35#include "llvm/BinaryFormat/ELF.h"
36#include "llvm/BinaryFormat/MachO.h"
37#include "llvm/CodeGen/AsmPrinter.h"
38#include "llvm/CodeGen/FaultMaps.h"
39#include "llvm/CodeGen/MachineBasicBlock.h"
40#include "llvm/CodeGen/MachineFunction.h"
41#include "llvm/CodeGen/MachineInstr.h"
42#include "llvm/CodeGen/MachineJumpTableInfo.h"
43#include "llvm/CodeGen/MachineModuleInfoImpls.h"
44#include "llvm/CodeGen/MachineOperand.h"
45#include "llvm/CodeGen/StackMaps.h"
46#include "llvm/CodeGen/TargetRegisterInfo.h"
47#include "llvm/IR/DataLayout.h"
48#include "llvm/IR/DebugInfoMetadata.h"
49#include "llvm/IR/Mangler.h"
50#include "llvm/IR/Module.h"
51#include "llvm/MC/MCAsmInfo.h"
52#include "llvm/MC/MCContext.h"
53#include "llvm/MC/MCExpr.h"
54#include "llvm/MC/MCInst.h"
55#include "llvm/MC/MCInstBuilder.h"
56#include "llvm/MC/MCSectionELF.h"
57#include "llvm/MC/MCSectionMachO.h"
58#include "llvm/MC/MCStreamer.h"
59#include "llvm/MC/MCSymbol.h"
60#include "llvm/MC/MCValue.h"
61#include "llvm/MC/TargetRegistry.h"
62#include "llvm/Support/Casting.h"
63#include "llvm/Support/CommandLine.h"
64#include "llvm/Support/Compiler.h"
65#include "llvm/Support/ErrorHandling.h"
66#include "llvm/Support/raw_ostream.h"
67#include "llvm/Target/TargetMachine.h"
68#include "llvm/TargetParser/Triple.h"
69#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
70#include <cassert>
71#include <cstdint>
72#include <map>
73#include <memory>
74
75using namespace llvm;
76
77#define DEBUG_TYPE "AArch64AsmPrinter"
78
79// Doesn't count FPR128 ZCZ instructions which are handled
80// by TableGen pattern matching
81STATISTIC(NumZCZeroingInstrsFPR,
82 "Number of zero-cycle FPR zeroing instructions expanded from "
83 "canonical pseudo instructions");
84
85enum PtrauthCheckMode { Default, Unchecked, Poison, Trap };
86static cl::opt<PtrauthCheckMode> PtrauthAuthChecks(
87 "aarch64-ptrauth-auth-checks", cl::Hidden,
88 cl::values(clEnumValN(Unchecked, "none", "don't test for failure"),
89 clEnumValN(Poison, "poison", "poison on failure"),
90 clEnumValN(Trap, "trap", "trap on failure")),
91 cl::desc("Check pointer authentication auth/resign failures"),
92 cl::init(Val: Default));
93
94namespace {
95
96class AArch64AsmPrinter : public AsmPrinter {
97 AArch64MCInstLower MCInstLowering;
98 FaultMaps FM;
99 const AArch64Subtarget *STI;
100 bool ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = false;
101#ifndef NDEBUG
102 unsigned InstsEmitted;
103#endif
104 bool EnableImportCallOptimization = false;
105 MapVector<MCSection *, std::vector<std::pair<MCSymbol *, MCSymbol *>>>
106 SectionToImportedFunctionCalls;
107 unsigned PAuthIFuncNextUniqueID = 1;
108
109public:
110 static char ID;
111
112 AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
113 : AsmPrinter(TM, std::move(Streamer), ID),
114 MCInstLowering(OutContext, *this), FM(*this) {}
115
116 StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
117
118 /// Wrapper for MCInstLowering.lowerOperand() for the
119 /// tblgen'erated pseudo lowering.
120 bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
121 return MCInstLowering.lowerOperand(MO, MCOp);
122 }
123
124 const MCExpr *lowerConstantPtrAuth(const ConstantPtrAuth &CPA) override;
125
126 const MCExpr *lowerBlockAddressConstant(const BlockAddress &BA) override;
127
128 void emitStartOfAsmFile(Module &M) override;
129 void emitJumpTableImpl(const MachineJumpTableInfo &MJTI,
130 ArrayRef<unsigned> JumpTableIndices) override;
131 std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
132 codeview::JumpTableEntrySize>
133 getCodeViewJumpTableInfo(int JTI, const MachineInstr *BranchInstr,
134 const MCSymbol *BranchLabel) const override;
135
136 void emitFunctionEntryLabel() override;
137
138 void emitXXStructor(const DataLayout &DL, const Constant *CV) override;
139
140 void LowerJumpTableDest(MCStreamer &OutStreamer, const MachineInstr &MI);
141
142 void LowerHardenedBRJumpTable(const MachineInstr &MI);
143
144 void LowerMOPS(MCStreamer &OutStreamer, const MachineInstr &MI);
145
146 void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
147 const MachineInstr &MI);
148 void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
149 const MachineInstr &MI);
150 void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
151 const MachineInstr &MI);
152 void LowerFAULTING_OP(const MachineInstr &MI);
153
154 void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
155 void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
156 void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
157 void LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI, bool Typed);
158
159 typedef std::tuple<unsigned, bool, uint32_t, bool, uint64_t>
160 HwasanMemaccessTuple;
161 std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
162 void LowerKCFI_CHECK(const MachineInstr &MI);
163 void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
164 void emitHwasanMemaccessSymbols(Module &M);
165
166 void emitSled(const MachineInstr &MI, SledKind Kind);
167
168 // Returns whether Reg may be used to store sensitive temporary values when
169 // expanding PtrAuth pseudos. Some OSes may take extra care to protect a
170 // small subset of GPRs on context switches - use these registers then.
171 //
172 // If there are no preferred registers, returns true for any Reg.
173 bool isPtrauthRegSafe(Register Reg) const {
174 if (STI->isX16X17Safer())
175 return Reg == AArch64::X16 || Reg == AArch64::X17;
176
177 return true;
178 }
179
180 // Emit the sequence for BRA/BLRA (authenticate + branch/call).
181 void emitPtrauthBranch(const MachineInstr *MI);
182
183 void emitPtrauthCheckAuthenticatedValue(Register TestedReg,
184 Register ScratchReg,
185 AArch64PACKey::ID Key,
186 AArch64PAuth::AuthCheckMethod Method,
187 const MCSymbol *OnFailure = nullptr);
188
189 // Check authenticated LR before tail calling.
190 void emitPtrauthTailCallHardening(const MachineInstr *TC);
191
192 struct PtrAuthSchema {
193 PtrAuthSchema(AArch64PACKey::ID Key, uint64_t IntDisc,
194 const MachineOperand &AddrDiscOp);
195
196 AArch64PACKey::ID Key;
197 uint64_t IntDisc;
198 Register AddrDisc;
199 bool AddrDiscIsKilled;
200 };
201
202 // Emit the sequence for AUT or AUTPAC. Addend if AUTRELLOADPAC
203 void emitPtrauthAuthResign(Register Pointer, Register Scratch,
204 PtrAuthSchema AuthSchema,
205 std::optional<PtrAuthSchema> SignSchema,
206 std::optional<uint64_t> Addend, Value *DS);
207
208 // Emit R_AARCH64_PATCHINST, the deactivation symbol relocation. Returns true
209 // if no instruction should be emitted because the deactivation symbol is
210 // defined in the current module so this function emitted a NOP instead.
211 bool emitDeactivationSymbolRelocation(Value *DS);
212
213 // Emit the sequence for PAC.
214 void emitPtrauthSign(const MachineInstr *MI);
215
216 // Emit the sequence to compute the discriminator.
217 //
218 // The Scratch register passed to this function must be safe, as returned by
219 // isPtrauthRegSafe(ScratchReg).
220 //
221 // The returned register is either ScratchReg, AddrDisc, or XZR. Furthermore,
222 // it is guaranteed to be safe (or XZR), with the only exception of
223 // passing-through an *unmodified* unsafe AddrDisc register.
224 //
225 // If the expanded pseudo is allowed to clobber AddrDisc register, setting
226 // MayClobberAddrDisc may save one MOV instruction, provided
227 // isPtrauthRegSafe(AddrDisc) is true:
228 //
229 // mov x17, x16
230 // movk x17, #1234, lsl #48
231 // ; x16 is not used anymore
232 //
233 // can be replaced by
234 //
235 // movk x16, #1234, lsl #48
236 Register emitPtrauthDiscriminator(uint64_t Disc, Register AddrDisc,
237 Register ScratchReg,
238 bool MayClobberAddrDisc = false);
239
240 // Emit the sequence for LOADauthptrstatic
241 void LowerLOADauthptrstatic(const MachineInstr &MI);
242
243 // Emit the sequence for LOADgotPAC/MOVaddrPAC (either GOT adrp-ldr or
244 // adrp-add followed by PAC sign)
245 void LowerMOVaddrPAC(const MachineInstr &MI);
246
247 // Emit the sequence for LOADgotAUTH (load signed pointer from signed ELF GOT
248 // and authenticate it with, if FPAC bit is not set, check+trap sequence after
249 // authenticating)
250 void LowerLOADgotAUTH(const MachineInstr &MI);
251
252 void emitAddImm(MCRegister Val, int64_t Addend, MCRegister Tmp);
253 void emitAddress(MCRegister Reg, const MCExpr *Expr, MCRegister Tmp,
254 bool DSOLocal, const MCSubtargetInfo &STI);
255
256 const MCExpr *emitPAuthRelocationAsIRelative(
257 const MCExpr *Target, uint64_t Disc, AArch64PACKey::ID KeyID,
258 bool HasAddressDiversity, bool IsDSOLocal, const MCExpr *DSExpr);
259
260 /// tblgen'erated driver function for lowering simple MI->MC
261 /// pseudo instructions.
262 bool lowerPseudoInstExpansion(const MachineInstr *MI, MCInst &Inst);
263
264 // Emit Build Attributes
265 void emitAttributes(unsigned Flags, uint64_t PAuthABIPlatform,
266 uint64_t PAuthABIVersion, AArch64TargetStreamer *TS);
267
268 // Emit expansion of Compare-and-branch pseudo instructions
269 void emitCBPseudoExpansion(const MachineInstr *MI);
270
271 void EmitToStreamer(MCStreamer &S, const MCInst &Inst);
272 void EmitToStreamer(const MCInst &Inst) {
273 EmitToStreamer(S&: *OutStreamer, Inst);
274 }
275
276 void emitInstruction(const MachineInstr *MI) override;
277
278 void emitFunctionHeaderComment() override;
279
280 void getAnalysisUsage(AnalysisUsage &AU) const override {
281 AsmPrinter::getAnalysisUsage(AU);
282 AU.setPreservesAll();
283 }
284
285 bool runOnMachineFunction(MachineFunction &MF) override {
286 if (auto *PSIW = getAnalysisIfAvailable<ProfileSummaryInfoWrapperPass>())
287 PSI = &PSIW->getPSI();
288 if (auto *SDPIW =
289 getAnalysisIfAvailable<StaticDataProfileInfoWrapperPass>())
290 SDPI = &SDPIW->getStaticDataProfileInfo();
291
292 AArch64FI = MF.getInfo<AArch64FunctionInfo>();
293 STI = &MF.getSubtarget<AArch64Subtarget>();
294
295 SetupMachineFunction(MF);
296
297 if (STI->isTargetCOFF()) {
298 bool Local = MF.getFunction().hasLocalLinkage();
299 COFF::SymbolStorageClass Scl =
300 Local ? COFF::IMAGE_SYM_CLASS_STATIC : COFF::IMAGE_SYM_CLASS_EXTERNAL;
301 int Type =
302 COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
303
304 OutStreamer->beginCOFFSymbolDef(Symbol: CurrentFnSym);
305 OutStreamer->emitCOFFSymbolStorageClass(StorageClass: Scl);
306 OutStreamer->emitCOFFSymbolType(Type);
307 OutStreamer->endCOFFSymbolDef();
308 }
309
310 // Emit the rest of the function body.
311 emitFunctionBody();
312
313 // Emit the XRay table for this function.
314 emitXRayTable();
315
316 // We didn't modify anything.
317 return false;
318 }
319
320 const MCExpr *lowerConstant(const Constant *CV,
321 const Constant *BaseCV = nullptr,
322 uint64_t Offset = 0) override;
323
324private:
325 void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
326 bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
327 bool printAsmRegInClass(const MachineOperand &MO,
328 const TargetRegisterClass *RC, unsigned AltName,
329 raw_ostream &O);
330
331 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
332 const char *ExtraCode, raw_ostream &O) override;
333 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
334 const char *ExtraCode, raw_ostream &O) override;
335
336 void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
337
338 void emitFunctionBodyEnd() override;
339 void emitGlobalAlias(const Module &M, const GlobalAlias &GA) override;
340
341 MCSymbol *GetCPISymbol(unsigned CPID) const override;
342 void emitEndOfAsmFile(Module &M) override;
343
344 AArch64FunctionInfo *AArch64FI = nullptr;
345
346 /// Emit the LOHs contained in AArch64FI.
347 void emitLOHs();
348
349 void emitMovXReg(Register Dest, Register Src);
350 void emitMOVZ(Register Dest, uint64_t Imm, unsigned Shift);
351 void emitMOVK(Register Dest, uint64_t Imm, unsigned Shift);
352
353 void emitAUT(AArch64PACKey::ID Key, Register Pointer, Register Disc);
354 void emitPAC(AArch64PACKey::ID Key, Register Pointer, Register Disc);
355 void emitBLRA(bool IsCall, AArch64PACKey::ID Key, Register Target,
356 Register Disc);
357
358 /// Emit instruction to set float register to zero.
359 void emitFMov0(const MachineInstr &MI);
360 void emitFMov0AsFMov(const MachineInstr &MI, Register DestReg);
361
362 using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
363
364 MInstToMCSymbol LOHInstToLabel;
365
366 bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const override {
367 return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags;
368 }
369
370 const MCSubtargetInfo *getIFuncMCSubtargetInfo() const override {
371 assert(STI);
372 return STI;
373 }
374 void emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI,
375 MCSymbol *LazyPointer) override;
376 void emitMachOIFuncStubHelperBody(Module &M, const GlobalIFunc &GI,
377 MCSymbol *LazyPointer) override;
378
379 /// Checks if this instruction is part of a sequence that is eligle for import
380 /// call optimization and, if so, records it to be emitted in the import call
381 /// section.
382 void recordIfImportCall(const MachineInstr *BranchInst);
383};
384
385} // end anonymous namespace
386
387void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
388 const Triple &TT = TM.getTargetTriple();
389
390 if (TT.isOSBinFormatCOFF()) {
391 emitCOFFFeatureSymbol(M);
392 emitCOFFReplaceableFunctionData(M);
393
394 if (M.getModuleFlag(Key: "import-call-optimization"))
395 EnableImportCallOptimization = true;
396 }
397
398 if (!TT.isOSBinFormatELF())
399 return;
400
401 // For emitting build attributes and .note.gnu.property section
402 auto *TS =
403 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
404 // Assemble feature flags that may require creation of build attributes and a
405 // note section.
406 unsigned BAFlags = 0;
407 unsigned GNUFlags = 0;
408 if (const auto *BTE = mdconst::extract_or_null<ConstantInt>(
409 MD: M.getModuleFlag(Key: "branch-target-enforcement"))) {
410 if (!BTE->isZero()) {
411 BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_BTI_Flag;
412 GNUFlags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
413 }
414 }
415
416 if (const auto *GCS = mdconst::extract_or_null<ConstantInt>(
417 MD: M.getModuleFlag(Key: "guarded-control-stack"))) {
418 if (!GCS->isZero()) {
419 BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_GCS_Flag;
420 GNUFlags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_GCS;
421 }
422 }
423
424 if (const auto *Sign = mdconst::extract_or_null<ConstantInt>(
425 MD: M.getModuleFlag(Key: "sign-return-address"))) {
426 if (!Sign->isZero()) {
427 BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_PAC_Flag;
428 GNUFlags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
429 }
430 }
431
432 uint64_t PAuthABIPlatform = -1;
433 if (const auto *PAP = mdconst::extract_or_null<ConstantInt>(
434 MD: M.getModuleFlag(Key: "aarch64-elf-pauthabi-platform"))) {
435 PAuthABIPlatform = PAP->getZExtValue();
436 }
437
438 uint64_t PAuthABIVersion = -1;
439 if (const auto *PAV = mdconst::extract_or_null<ConstantInt>(
440 MD: M.getModuleFlag(Key: "aarch64-elf-pauthabi-version"))) {
441 PAuthABIVersion = PAV->getZExtValue();
442 }
443
444 // Emit AArch64 Build Attributes
445 emitAttributes(Flags: BAFlags, PAuthABIPlatform, PAuthABIVersion, TS);
446 // Emit a .note.gnu.property section with the flags.
447 TS->emitNoteSection(Flags: GNUFlags, PAuthABIPlatform, PAuthABIVersion);
448}
449
450void AArch64AsmPrinter::emitFunctionHeaderComment() {
451 const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>();
452 std::optional<std::string> OutlinerString = FI->getOutliningStyle();
453 if (OutlinerString != std::nullopt)
454 OutStreamer->getCommentOS() << ' ' << OutlinerString;
455}
456
457void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
458{
459 const Function &F = MF->getFunction();
460 if (F.hasFnAttribute(Kind: "patchable-function-entry")) {
461 unsigned Num;
462 if (F.getFnAttribute(Kind: "patchable-function-entry")
463 .getValueAsString()
464 .getAsInteger(Radix: 10, Result&: Num))
465 return;
466 emitNops(N: Num);
467 return;
468 }
469
470 emitSled(MI, Kind: SledKind::FUNCTION_ENTER);
471}
472
473void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI) {
474 emitSled(MI, Kind: SledKind::FUNCTION_EXIT);
475}
476
477void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) {
478 emitSled(MI, Kind: SledKind::TAIL_CALL);
479}
480
481void AArch64AsmPrinter::emitSled(const MachineInstr &MI, SledKind Kind) {
482 static const int8_t NoopsInSledCount = 7;
483 // We want to emit the following pattern:
484 //
485 // .Lxray_sled_N:
486 // ALIGN
487 // B #32
488 // ; 7 NOP instructions (28 bytes)
489 // .tmpN
490 //
491 // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
492 // over the full 32 bytes (8 instructions) with the following pattern:
493 //
494 // STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
495 // LDR W17, #12 ; W17 := function ID
496 // LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
497 // BLR X16 ; call the tracing trampoline
498 // ;DATA: 32 bits of function ID
499 // ;DATA: lower 32 bits of the address of the trampoline
500 // ;DATA: higher 32 bits of the address of the trampoline
501 // LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
502 //
503 OutStreamer->emitCodeAlignment(Alignment: Align(4), STI: &getSubtargetInfo());
504 auto CurSled = OutContext.createTempSymbol(Name: "xray_sled_", AlwaysAddSuffix: true);
505 OutStreamer->emitLabel(Symbol: CurSled);
506 auto Target = OutContext.createTempSymbol();
507
508 // Emit "B #32" instruction, which jumps over the next 28 bytes.
509 // The operand has to be the number of 4-byte instructions to jump over,
510 // including the current instruction.
511 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::B).addImm(Val: 8));
512
513 for (int8_t I = 0; I < NoopsInSledCount; I++)
514 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::NOP));
515
516 OutStreamer->emitLabel(Symbol: Target);
517 recordSled(Sled: CurSled, MI, Kind, Version: 2);
518}
519
520void AArch64AsmPrinter::emitAttributes(unsigned Flags,
521 uint64_t PAuthABIPlatform,
522 uint64_t PAuthABIVersion,
523 AArch64TargetStreamer *TS) {
524
525 PAuthABIPlatform = (uint64_t(-1) == PAuthABIPlatform) ? 0 : PAuthABIPlatform;
526 PAuthABIVersion = (uint64_t(-1) == PAuthABIVersion) ? 0 : PAuthABIVersion;
527
528 if (PAuthABIPlatform || PAuthABIVersion) {
529 TS->emitAttributesSubsection(
530 VendorName: AArch64BuildAttributes::getVendorName(
531 Vendor: AArch64BuildAttributes::AEABI_PAUTHABI),
532 IsOptional: AArch64BuildAttributes::SubsectionOptional::REQUIRED,
533 ParameterType: AArch64BuildAttributes::SubsectionType::ULEB128);
534 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
535 Vendor: AArch64BuildAttributes::AEABI_PAUTHABI),
536 Tag: AArch64BuildAttributes::TAG_PAUTH_PLATFORM,
537 Value: PAuthABIPlatform, String: "");
538 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
539 Vendor: AArch64BuildAttributes::AEABI_PAUTHABI),
540 Tag: AArch64BuildAttributes::TAG_PAUTH_SCHEMA, Value: PAuthABIVersion,
541 String: "");
542 }
543
544 unsigned BTIValue =
545 (Flags & AArch64BuildAttributes::Feature_BTI_Flag) ? 1 : 0;
546 unsigned PACValue =
547 (Flags & AArch64BuildAttributes::Feature_PAC_Flag) ? 1 : 0;
548 unsigned GCSValue =
549 (Flags & AArch64BuildAttributes::Feature_GCS_Flag) ? 1 : 0;
550
551 if (BTIValue || PACValue || GCSValue) {
552 TS->emitAttributesSubsection(
553 VendorName: AArch64BuildAttributes::getVendorName(
554 Vendor: AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
555 IsOptional: AArch64BuildAttributes::SubsectionOptional::OPTIONAL,
556 ParameterType: AArch64BuildAttributes::SubsectionType::ULEB128);
557 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
558 Vendor: AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
559 Tag: AArch64BuildAttributes::TAG_FEATURE_BTI, Value: BTIValue, String: "");
560 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
561 Vendor: AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
562 Tag: AArch64BuildAttributes::TAG_FEATURE_PAC, Value: PACValue, String: "");
563 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
564 Vendor: AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
565 Tag: AArch64BuildAttributes::TAG_FEATURE_GCS, Value: GCSValue, String: "");
566 }
567}
568
569// Emit the following code for Intrinsic::{xray_customevent,xray_typedevent}
570// (built-in functions __xray_customevent/__xray_typedevent).
571//
572// .Lxray_event_sled_N:
573// b 1f
574// save x0 and x1 (and also x2 for TYPED_EVENT_CALL)
575// set up x0 and x1 (and also x2 for TYPED_EVENT_CALL)
576// bl __xray_CustomEvent or __xray_TypedEvent
577// restore x0 and x1 (and also x2 for TYPED_EVENT_CALL)
578// 1:
579//
580// There are 6 instructions for EVENT_CALL and 9 for TYPED_EVENT_CALL.
581//
582// Then record a sled of kind CUSTOM_EVENT or TYPED_EVENT.
583// After patching, b .+N will become a nop.
584void AArch64AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI,
585 bool Typed) {
586 auto &O = *OutStreamer;
587 MCSymbol *CurSled = OutContext.createTempSymbol(Name: "xray_sled_", AlwaysAddSuffix: true);
588 O.emitLabel(Symbol: CurSled);
589 bool MachO = TM.getTargetTriple().isOSBinFormatMachO();
590 auto *Sym = MCSymbolRefExpr::create(
591 Symbol: OutContext.getOrCreateSymbol(
592 Name: Twine(MachO ? "_" : "") +
593 (Typed ? "__xray_TypedEvent" : "__xray_CustomEvent")),
594 Ctx&: OutContext);
595 if (Typed) {
596 O.AddComment(T: "Begin XRay typed event");
597 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::B).addImm(Val: 9));
598 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::STPXpre)
599 .addReg(Reg: AArch64::SP)
600 .addReg(Reg: AArch64::X0)
601 .addReg(Reg: AArch64::X1)
602 .addReg(Reg: AArch64::SP)
603 .addImm(Val: -4));
604 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::STRXui)
605 .addReg(Reg: AArch64::X2)
606 .addReg(Reg: AArch64::SP)
607 .addImm(Val: 2));
608 emitMovXReg(Dest: AArch64::X0, Src: MI.getOperand(i: 0).getReg());
609 emitMovXReg(Dest: AArch64::X1, Src: MI.getOperand(i: 1).getReg());
610 emitMovXReg(Dest: AArch64::X2, Src: MI.getOperand(i: 2).getReg());
611 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::BL).addExpr(Val: Sym));
612 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::LDRXui)
613 .addReg(Reg: AArch64::X2)
614 .addReg(Reg: AArch64::SP)
615 .addImm(Val: 2));
616 O.AddComment(T: "End XRay typed event");
617 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::LDPXpost)
618 .addReg(Reg: AArch64::SP)
619 .addReg(Reg: AArch64::X0)
620 .addReg(Reg: AArch64::X1)
621 .addReg(Reg: AArch64::SP)
622 .addImm(Val: 4));
623
624 recordSled(Sled: CurSled, MI, Kind: SledKind::TYPED_EVENT, Version: 2);
625 } else {
626 O.AddComment(T: "Begin XRay custom event");
627 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::B).addImm(Val: 6));
628 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::STPXpre)
629 .addReg(Reg: AArch64::SP)
630 .addReg(Reg: AArch64::X0)
631 .addReg(Reg: AArch64::X1)
632 .addReg(Reg: AArch64::SP)
633 .addImm(Val: -2));
634 emitMovXReg(Dest: AArch64::X0, Src: MI.getOperand(i: 0).getReg());
635 emitMovXReg(Dest: AArch64::X1, Src: MI.getOperand(i: 1).getReg());
636 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::BL).addExpr(Val: Sym));
637 O.AddComment(T: "End XRay custom event");
638 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::LDPXpost)
639 .addReg(Reg: AArch64::SP)
640 .addReg(Reg: AArch64::X0)
641 .addReg(Reg: AArch64::X1)
642 .addReg(Reg: AArch64::SP)
643 .addImm(Val: 2));
644
645 recordSled(Sled: CurSled, MI, Kind: SledKind::CUSTOM_EVENT, Version: 2);
646 }
647}
648
649void AArch64AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
650 Register AddrReg = MI.getOperand(i: 0).getReg();
651 assert(std::next(MI.getIterator())->isCall() &&
652 "KCFI_CHECK not followed by a call instruction");
653 assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg &&
654 "KCFI_CHECK call target doesn't match call operand");
655
656 // Default to using the intra-procedure-call temporary registers for
657 // comparing the hashes.
658 unsigned ScratchRegs[] = {AArch64::W16, AArch64::W17};
659 if (AddrReg == AArch64::XZR) {
660 // Checking XZR makes no sense. Instead of emitting a load, zero
661 // ScratchRegs[0] and use it for the ESR AddrIndex below.
662 AddrReg = getXRegFromWReg(Reg: ScratchRegs[0]);
663 emitMovXReg(Dest: AddrReg, Src: AArch64::XZR);
664 } else {
665 // If one of the scratch registers is used for the call target (e.g.
666 // with AArch64::TCRETURNriBTI), we can clobber another caller-saved
667 // temporary register instead (in this case, AArch64::W9) as the check
668 // is immediately followed by the call instruction.
669 for (auto &Reg : ScratchRegs) {
670 if (Reg == getWRegFromXReg(Reg: AddrReg)) {
671 Reg = AArch64::W9;
672 break;
673 }
674 }
675 assert(ScratchRegs[0] != AddrReg && ScratchRegs[1] != AddrReg &&
676 "Invalid scratch registers for KCFI_CHECK");
677
678 // Adjust the offset for patchable-function-prefix. This assumes that
679 // patchable-function-prefix is the same for all functions.
680 int64_t PrefixNops = 0;
681 (void)MI.getMF()
682 ->getFunction()
683 .getFnAttribute(Kind: "patchable-function-prefix")
684 .getValueAsString()
685 .getAsInteger(Radix: 10, Result&: PrefixNops);
686
687 // Load the target function type hash.
688 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::LDURWi)
689 .addReg(Reg: ScratchRegs[0])
690 .addReg(Reg: AddrReg)
691 .addImm(Val: -(PrefixNops * 4 + 4)));
692 }
693
694 // Load the expected type hash.
695 const int64_t Type = MI.getOperand(i: 1).getImm();
696 emitMOVK(Dest: ScratchRegs[1], Imm: Type & 0xFFFF, Shift: 0);
697 emitMOVK(Dest: ScratchRegs[1], Imm: (Type >> 16) & 0xFFFF, Shift: 16);
698
699 // Compare the hashes and trap if there's a mismatch.
700 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::SUBSWrs)
701 .addReg(Reg: AArch64::WZR)
702 .addReg(Reg: ScratchRegs[0])
703 .addReg(Reg: ScratchRegs[1])
704 .addImm(Val: 0));
705
706 MCSymbol *Pass = OutContext.createTempSymbol();
707 EmitToStreamer(S&: *OutStreamer,
708 Inst: MCInstBuilder(AArch64::Bcc)
709 .addImm(Val: AArch64CC::EQ)
710 .addExpr(Val: MCSymbolRefExpr::create(Symbol: Pass, Ctx&: OutContext)));
711
712 // The base ESR is 0x8000 and the register information is encoded in bits
713 // 0-9 as follows:
714 // - 0-4: n, where the register Xn contains the target address
715 // - 5-9: m, where the register Wm contains the expected type hash
716 // Where n, m are in [0, 30].
717 unsigned TypeIndex = ScratchRegs[1] - AArch64::W0;
718 unsigned AddrIndex;
719 switch (AddrReg) {
720 default:
721 AddrIndex = AddrReg - AArch64::X0;
722 break;
723 case AArch64::FP:
724 AddrIndex = 29;
725 break;
726 case AArch64::LR:
727 AddrIndex = 30;
728 break;
729 }
730
731 assert(AddrIndex < 31 && TypeIndex < 31);
732
733 unsigned ESR = 0x8000 | ((TypeIndex & 31) << 5) | (AddrIndex & 31);
734 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::BRK).addImm(Val: ESR));
735 OutStreamer->emitLabel(Symbol: Pass);
736}
737
738void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
739 Register Reg = MI.getOperand(i: 0).getReg();
740
741 // The HWASan pass won't emit a CHECK_MEMACCESS intrinsic with a pointer
742 // statically known to be zero. However, conceivably, the HWASan pass may
743 // encounter a "cannot currently statically prove to be null" pointer (and is
744 // therefore unable to omit the intrinsic) that later optimization passes
745 // convert into a statically known-null pointer.
746 if (Reg == AArch64::XZR)
747 return;
748
749 bool IsShort =
750 ((MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES) ||
751 (MI.getOpcode() ==
752 AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW));
753 uint32_t AccessInfo = MI.getOperand(i: 1).getImm();
754 bool IsFixedShadow =
755 ((MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_FIXEDSHADOW) ||
756 (MI.getOpcode() ==
757 AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW));
758 uint64_t FixedShadowOffset = IsFixedShadow ? MI.getOperand(i: 2).getImm() : 0;
759
760 MCSymbol *&Sym = HwasanMemaccessSymbols[HwasanMemaccessTuple(
761 Reg, IsShort, AccessInfo, IsFixedShadow, FixedShadowOffset)];
762 if (!Sym) {
763 // FIXME: Make this work on non-ELF.
764 if (!TM.getTargetTriple().isOSBinFormatELF())
765 report_fatal_error(reason: "llvm.hwasan.check.memaccess only supported on ELF");
766
767 std::string SymName = "__hwasan_check_x" + utostr(X: Reg - AArch64::X0) + "_" +
768 utostr(X: AccessInfo);
769 if (IsFixedShadow)
770 SymName += "_fixed_" + utostr(X: FixedShadowOffset);
771 if (IsShort)
772 SymName += "_short_v2";
773 Sym = OutContext.getOrCreateSymbol(Name: SymName);
774 }
775
776 EmitToStreamer(S&: *OutStreamer,
777 Inst: MCInstBuilder(AArch64::BL)
778 .addExpr(Val: MCSymbolRefExpr::create(Symbol: Sym, Ctx&: OutContext)));
779}
780
781void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) {
782 if (HwasanMemaccessSymbols.empty())
783 return;
784
785 const Triple &TT = TM.getTargetTriple();
786 assert(TT.isOSBinFormatELF());
787 // AArch64Subtarget is huge, so heap allocate it so we don't run out of stack
788 // space.
789 auto STI = std::make_unique<AArch64Subtarget>(
790 args: TT, args: TM.getTargetCPU(), args: TM.getTargetCPU(), args: TM.getTargetFeatureString(), args&: TM,
791 args: true);
792 this->STI = STI.get();
793
794 MCSymbol *HwasanTagMismatchV1Sym =
795 OutContext.getOrCreateSymbol(Name: "__hwasan_tag_mismatch");
796 MCSymbol *HwasanTagMismatchV2Sym =
797 OutContext.getOrCreateSymbol(Name: "__hwasan_tag_mismatch_v2");
798
799 const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
800 MCSymbolRefExpr::create(Symbol: HwasanTagMismatchV1Sym, Ctx&: OutContext);
801 const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
802 MCSymbolRefExpr::create(Symbol: HwasanTagMismatchV2Sym, Ctx&: OutContext);
803
804 for (auto &P : HwasanMemaccessSymbols) {
805 unsigned Reg = std::get<0>(t: P.first);
806 bool IsShort = std::get<1>(t: P.first);
807 uint32_t AccessInfo = std::get<2>(t: P.first);
808 bool IsFixedShadow = std::get<3>(t: P.first);
809 uint64_t FixedShadowOffset = std::get<4>(t: P.first);
810 const MCSymbolRefExpr *HwasanTagMismatchRef =
811 IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
812 MCSymbol *Sym = P.second;
813
814 bool HasMatchAllTag =
815 (AccessInfo >> HWASanAccessInfo::HasMatchAllShift) & 1;
816 uint8_t MatchAllTag =
817 (AccessInfo >> HWASanAccessInfo::MatchAllShift) & 0xff;
818 unsigned Size =
819 1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf);
820 bool CompileKernel =
821 (AccessInfo >> HWASanAccessInfo::CompileKernelShift) & 1;
822
823 OutStreamer->switchSection(Section: OutContext.getELFSection(
824 Section: ".text.hot", Type: ELF::SHT_PROGBITS,
825 Flags: ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, EntrySize: 0, Group: Sym->getName(),
826 /*IsComdat=*/true));
827
828 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_ELF_TypeFunction);
829 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Weak);
830 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Hidden);
831 OutStreamer->emitLabel(Symbol: Sym);
832
833 EmitToStreamer(Inst: MCInstBuilder(AArch64::SBFMXri)
834 .addReg(Reg: AArch64::X16)
835 .addReg(Reg)
836 .addImm(Val: 4)
837 .addImm(Val: 55));
838
839 if (IsFixedShadow) {
840 // Aarch64 makes it difficult to embed large constants in the code.
841 // Fortuitously, kShadowBaseAlignment == 32, so we use the 32-bit
842 // left-shift option in the MOV instruction. Combined with the 16-bit
843 // immediate, this is enough to represent any offset up to 2**48.
844 emitMOVZ(Dest: AArch64::X17, Imm: FixedShadowOffset >> 32, Shift: 32);
845 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRBBroX)
846 .addReg(Reg: AArch64::W16)
847 .addReg(Reg: AArch64::X17)
848 .addReg(Reg: AArch64::X16)
849 .addImm(Val: 0)
850 .addImm(Val: 0));
851 } else {
852 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRBBroX)
853 .addReg(Reg: AArch64::W16)
854 .addReg(Reg: IsShort ? AArch64::X20 : AArch64::X9)
855 .addReg(Reg: AArch64::X16)
856 .addImm(Val: 0)
857 .addImm(Val: 0));
858 }
859
860 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSXrs)
861 .addReg(Reg: AArch64::XZR)
862 .addReg(Reg: AArch64::X16)
863 .addReg(Reg)
864 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSR, Imm: 56)));
865 MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
866 EmitToStreamer(Inst: MCInstBuilder(AArch64::Bcc)
867 .addImm(Val: AArch64CC::NE)
868 .addExpr(Val: MCSymbolRefExpr::create(
869 Symbol: HandleMismatchOrPartialSym, Ctx&: OutContext)));
870 MCSymbol *ReturnSym = OutContext.createTempSymbol();
871 OutStreamer->emitLabel(Symbol: ReturnSym);
872 EmitToStreamer(Inst: MCInstBuilder(AArch64::RET).addReg(Reg: AArch64::LR));
873 OutStreamer->emitLabel(Symbol: HandleMismatchOrPartialSym);
874
875 if (HasMatchAllTag) {
876 EmitToStreamer(Inst: MCInstBuilder(AArch64::UBFMXri)
877 .addReg(Reg: AArch64::X17)
878 .addReg(Reg)
879 .addImm(Val: 56)
880 .addImm(Val: 63));
881 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSXri)
882 .addReg(Reg: AArch64::XZR)
883 .addReg(Reg: AArch64::X17)
884 .addImm(Val: MatchAllTag)
885 .addImm(Val: 0));
886 EmitToStreamer(
887 Inst: MCInstBuilder(AArch64::Bcc)
888 .addImm(Val: AArch64CC::EQ)
889 .addExpr(Val: MCSymbolRefExpr::create(Symbol: ReturnSym, Ctx&: OutContext)));
890 }
891
892 if (IsShort) {
893 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSWri)
894 .addReg(Reg: AArch64::WZR)
895 .addReg(Reg: AArch64::W16)
896 .addImm(Val: 15)
897 .addImm(Val: 0));
898 MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
899 EmitToStreamer(
900 Inst: MCInstBuilder(AArch64::Bcc)
901 .addImm(Val: AArch64CC::HI)
902 .addExpr(Val: MCSymbolRefExpr::create(Symbol: HandleMismatchSym, Ctx&: OutContext)));
903
904 EmitToStreamer(Inst: MCInstBuilder(AArch64::ANDXri)
905 .addReg(Reg: AArch64::X17)
906 .addReg(Reg)
907 .addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: 0xf, regSize: 64)));
908 if (Size != 1)
909 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
910 .addReg(Reg: AArch64::X17)
911 .addReg(Reg: AArch64::X17)
912 .addImm(Val: Size - 1)
913 .addImm(Val: 0));
914 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSWrs)
915 .addReg(Reg: AArch64::WZR)
916 .addReg(Reg: AArch64::W16)
917 .addReg(Reg: AArch64::W17)
918 .addImm(Val: 0));
919 EmitToStreamer(
920 Inst: MCInstBuilder(AArch64::Bcc)
921 .addImm(Val: AArch64CC::LS)
922 .addExpr(Val: MCSymbolRefExpr::create(Symbol: HandleMismatchSym, Ctx&: OutContext)));
923
924 EmitToStreamer(Inst: MCInstBuilder(AArch64::ORRXri)
925 .addReg(Reg: AArch64::X16)
926 .addReg(Reg)
927 .addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: 0xf, regSize: 64)));
928 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRBBui)
929 .addReg(Reg: AArch64::W16)
930 .addReg(Reg: AArch64::X16)
931 .addImm(Val: 0));
932 EmitToStreamer(
933 Inst: MCInstBuilder(AArch64::SUBSXrs)
934 .addReg(Reg: AArch64::XZR)
935 .addReg(Reg: AArch64::X16)
936 .addReg(Reg)
937 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSR, Imm: 56)));
938 EmitToStreamer(
939 Inst: MCInstBuilder(AArch64::Bcc)
940 .addImm(Val: AArch64CC::EQ)
941 .addExpr(Val: MCSymbolRefExpr::create(Symbol: ReturnSym, Ctx&: OutContext)));
942
943 OutStreamer->emitLabel(Symbol: HandleMismatchSym);
944 }
945
946 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPXpre)
947 .addReg(Reg: AArch64::SP)
948 .addReg(Reg: AArch64::X0)
949 .addReg(Reg: AArch64::X1)
950 .addReg(Reg: AArch64::SP)
951 .addImm(Val: -32));
952 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPXi)
953 .addReg(Reg: AArch64::FP)
954 .addReg(Reg: AArch64::LR)
955 .addReg(Reg: AArch64::SP)
956 .addImm(Val: 29));
957
958 if (Reg != AArch64::X0)
959 emitMovXReg(Dest: AArch64::X0, Src: Reg);
960 emitMOVZ(Dest: AArch64::X1, Imm: AccessInfo & HWASanAccessInfo::RuntimeMask, Shift: 0);
961
962 if (CompileKernel) {
963 // The Linux kernel's dynamic loader doesn't support GOT relative
964 // relocations, but it doesn't support late binding either, so just call
965 // the function directly.
966 EmitToStreamer(Inst: MCInstBuilder(AArch64::B).addExpr(Val: HwasanTagMismatchRef));
967 } else {
968 // Intentionally load the GOT entry and branch to it, rather than possibly
969 // late binding the function, which may clobber the registers before we
970 // have a chance to save them.
971 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADRP)
972 .addReg(Reg: AArch64::X16)
973 .addExpr(Val: MCSpecifierExpr::create(Expr: HwasanTagMismatchRef,
974 S: AArch64::S_GOT_PAGE,
975 Ctx&: OutContext)));
976 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
977 .addReg(Reg: AArch64::X16)
978 .addReg(Reg: AArch64::X16)
979 .addExpr(Val: MCSpecifierExpr::create(Expr: HwasanTagMismatchRef,
980 S: AArch64::S_GOT_LO12,
981 Ctx&: OutContext)));
982 EmitToStreamer(Inst: MCInstBuilder(AArch64::BR).addReg(Reg: AArch64::X16));
983 }
984 }
985 this->STI = nullptr;
986}
987
988static void emitAuthenticatedPointer(MCStreamer &OutStreamer,
989 MCSymbol *StubLabel,
990 const MCExpr *StubAuthPtrRef) {
991 // sym$auth_ptr$key$disc:
992 OutStreamer.emitLabel(Symbol: StubLabel);
993 OutStreamer.emitValue(Value: StubAuthPtrRef, /*size=*/Size: 8);
994}
995
996void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
997 emitHwasanMemaccessSymbols(M);
998
999 const Triple &TT = TM.getTargetTriple();
1000 if (TT.isOSBinFormatMachO()) {
1001 // Output authenticated pointers as indirect symbols, if we have any.
1002 MachineModuleInfoMachO &MMIMacho =
1003 MMI->getObjFileInfo<MachineModuleInfoMachO>();
1004
1005 auto Stubs = MMIMacho.getAuthGVStubList();
1006
1007 if (!Stubs.empty()) {
1008 // Switch to the "__auth_ptr" section.
1009 OutStreamer->switchSection(
1010 Section: OutContext.getMachOSection(Segment: "__DATA", Section: "__auth_ptr", TypeAndAttributes: MachO::S_REGULAR,
1011 K: SectionKind::getMetadata()));
1012 emitAlignment(Alignment: Align(8));
1013
1014 for (const auto &Stub : Stubs)
1015 emitAuthenticatedPointer(OutStreamer&: *OutStreamer, StubLabel: Stub.first, StubAuthPtrRef: Stub.second);
1016
1017 OutStreamer->addBlankLine();
1018 }
1019
1020 // Funny Darwin hack: This flag tells the linker that no global symbols
1021 // contain code that falls through to other global symbols (e.g. the obvious
1022 // implementation of multiple entry points). If this doesn't occur, the
1023 // linker can safely perform dead code stripping. Since LLVM never
1024 // generates code that does this, it is always safe to set.
1025 OutStreamer->emitSubsectionsViaSymbols();
1026 }
1027
1028 if (TT.isOSBinFormatELF()) {
1029 // Output authenticated pointers as indirect symbols, if we have any.
1030 MachineModuleInfoELF &MMIELF = MMI->getObjFileInfo<MachineModuleInfoELF>();
1031
1032 auto Stubs = MMIELF.getAuthGVStubList();
1033
1034 if (!Stubs.empty()) {
1035 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
1036 OutStreamer->switchSection(Section: TLOF.getDataSection());
1037 emitAlignment(Alignment: Align(8));
1038
1039 for (const auto &Stub : Stubs)
1040 emitAuthenticatedPointer(OutStreamer&: *OutStreamer, StubLabel: Stub.first, StubAuthPtrRef: Stub.second);
1041
1042 OutStreamer->addBlankLine();
1043 }
1044
1045 // With signed ELF GOT enabled, the linker looks at the symbol type to
1046 // choose between keys IA (for STT_FUNC) and DA (for other types). Symbols
1047 // for functions not defined in the module have STT_NOTYPE type by default.
1048 // This makes linker to emit signing schema with DA key (instead of IA) for
1049 // corresponding R_AARCH64_AUTH_GLOB_DAT dynamic reloc. To avoid that, force
1050 // all function symbols used in the module to have STT_FUNC type. See
1051 // https://github.com/ARM-software/abi-aa/blob/main/pauthabielf64/pauthabielf64.rst#default-signing-schema
1052 const auto *PtrAuthELFGOTFlag = mdconst::extract_or_null<ConstantInt>(
1053 MD: M.getModuleFlag(Key: "ptrauth-elf-got"));
1054 if (PtrAuthELFGOTFlag && PtrAuthELFGOTFlag->getZExtValue() == 1)
1055 for (const GlobalValue &GV : M.global_values())
1056 if (!GV.use_empty() && isa<Function>(Val: GV) &&
1057 !GV.getName().starts_with(Prefix: "llvm."))
1058 OutStreamer->emitSymbolAttribute(Symbol: getSymbol(GV: &GV),
1059 Attribute: MCSA_ELF_TypeFunction);
1060 }
1061
1062 // Emit stack and fault map information.
1063 FM.serializeToFaultMapSection();
1064
1065 // If import call optimization is enabled, emit the appropriate section.
1066 // We do this whether or not we recorded any import calls.
1067 if (EnableImportCallOptimization && TT.isOSBinFormatCOFF()) {
1068 OutStreamer->switchSection(Section: getObjFileLowering().getImportCallSection());
1069
1070 // Section always starts with some magic.
1071 constexpr char ImpCallMagic[12] = "Imp_Call_V1";
1072 OutStreamer->emitBytes(Data: StringRef{ImpCallMagic, sizeof(ImpCallMagic)});
1073
1074 // Layout of this section is:
1075 // Per section that contains calls to imported functions:
1076 // uint32_t SectionSize: Size in bytes for information in this section.
1077 // uint32_t Section Number
1078 // Per call to imported function in section:
1079 // uint32_t Kind: the kind of imported function.
1080 // uint32_t BranchOffset: the offset of the branch instruction in its
1081 // parent section.
1082 // uint32_t TargetSymbolId: the symbol id of the called function.
1083 for (auto &[Section, CallsToImportedFuncs] :
1084 SectionToImportedFunctionCalls) {
1085 unsigned SectionSize =
1086 sizeof(uint32_t) * (2 + 3 * CallsToImportedFuncs.size());
1087 OutStreamer->emitInt32(Value: SectionSize);
1088 OutStreamer->emitCOFFSecNumber(Symbol: Section->getBeginSymbol());
1089 for (auto &[CallsiteSymbol, CalledSymbol] : CallsToImportedFuncs) {
1090 // Kind is always IMAGE_REL_ARM64_DYNAMIC_IMPORT_CALL (0x13).
1091 OutStreamer->emitInt32(Value: 0x13);
1092 OutStreamer->emitCOFFSecOffset(Symbol: CallsiteSymbol);
1093 OutStreamer->emitCOFFSymbolIndex(Symbol: CalledSymbol);
1094 }
1095 }
1096 }
1097}
1098
1099void AArch64AsmPrinter::emitLOHs() {
1100 SmallVector<MCSymbol *, 3> MCArgs;
1101
1102 for (const auto &D : AArch64FI->getLOHContainer()) {
1103 for (const MachineInstr *MI : D.getArgs()) {
1104 MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(x: MI);
1105 assert(LabelIt != LOHInstToLabel.end() &&
1106 "Label hasn't been inserted for LOH related instruction");
1107 MCArgs.push_back(Elt: LabelIt->second);
1108 }
1109 OutStreamer->emitLOHDirective(Kind: D.getKind(), Args: MCArgs);
1110 MCArgs.clear();
1111 }
1112}
1113
1114void AArch64AsmPrinter::emitFunctionBodyEnd() {
1115 if (!AArch64FI->getLOHRelated().empty())
1116 emitLOHs();
1117}
1118
1119/// GetCPISymbol - Return the symbol for the specified constant pool entry.
1120MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
1121 // Darwin uses a linker-private symbol name for constant-pools (to
1122 // avoid addends on the relocation?), ELF has no such concept and
1123 // uses a normal private symbol.
1124 if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
1125 return OutContext.getOrCreateSymbol(
1126 Name: Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
1127 Twine(getFunctionNumber()) + "_" + Twine(CPID));
1128
1129 return AsmPrinter::GetCPISymbol(CPID);
1130}
1131
1132void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
1133 raw_ostream &O) {
1134 const MachineOperand &MO = MI->getOperand(i: OpNum);
1135 switch (MO.getType()) {
1136 default:
1137 llvm_unreachable("<unknown operand type>");
1138 case MachineOperand::MO_Register: {
1139 Register Reg = MO.getReg();
1140 assert(Reg.isPhysical());
1141 assert(!MO.getSubReg() && "Subregs should be eliminated!");
1142 O << AArch64InstPrinter::getRegisterName(Reg);
1143 break;
1144 }
1145 case MachineOperand::MO_Immediate: {
1146 O << MO.getImm();
1147 break;
1148 }
1149 case MachineOperand::MO_GlobalAddress: {
1150 PrintSymbolOperand(MO, OS&: O);
1151 break;
1152 }
1153 case MachineOperand::MO_BlockAddress: {
1154 MCSymbol *Sym = GetBlockAddressSymbol(BA: MO.getBlockAddress());
1155 Sym->print(OS&: O, MAI);
1156 break;
1157 }
1158 }
1159}
1160
1161bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
1162 raw_ostream &O) {
1163 Register Reg = MO.getReg();
1164 switch (Mode) {
1165 default:
1166 return true; // Unknown mode.
1167 case 'w':
1168 Reg = getWRegFromXReg(Reg);
1169 break;
1170 case 'x':
1171 Reg = getXRegFromWReg(Reg);
1172 break;
1173 case 't':
1174 Reg = getXRegFromXRegTuple(RegTuple: Reg);
1175 break;
1176 }
1177
1178 O << AArch64InstPrinter::getRegisterName(Reg);
1179 return false;
1180}
1181
1182// Prints the register in MO using class RC using the offset in the
1183// new register class. This should not be used for cross class
1184// printing.
1185bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
1186 const TargetRegisterClass *RC,
1187 unsigned AltName, raw_ostream &O) {
1188 assert(MO.isReg() && "Should only get here with a register!");
1189 const TargetRegisterInfo *RI = STI->getRegisterInfo();
1190 Register Reg = MO.getReg();
1191 MCRegister RegToPrint = RC->getRegister(i: RI->getEncodingValue(Reg));
1192 if (!RI->regsOverlap(RegA: RegToPrint, RegB: Reg))
1193 return true;
1194 O << AArch64InstPrinter::getRegisterName(Reg: RegToPrint, AltIdx: AltName);
1195 return false;
1196}
1197
1198bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
1199 const char *ExtraCode, raw_ostream &O) {
1200 const MachineOperand &MO = MI->getOperand(i: OpNum);
1201
1202 // First try the generic code, which knows about modifiers like 'c' and 'n'.
1203 if (!AsmPrinter::PrintAsmOperand(MI, OpNo: OpNum, ExtraCode, OS&: O))
1204 return false;
1205
1206 // Does this asm operand have a single letter operand modifier?
1207 if (ExtraCode && ExtraCode[0]) {
1208 if (ExtraCode[1] != 0)
1209 return true; // Unknown modifier.
1210
1211 switch (ExtraCode[0]) {
1212 default:
1213 return true; // Unknown modifier.
1214 case 'w': // Print W register
1215 case 'x': // Print X register
1216 if (MO.isReg())
1217 return printAsmMRegister(MO, Mode: ExtraCode[0], O);
1218 if (MO.isImm() && MO.getImm() == 0) {
1219 unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
1220 O << AArch64InstPrinter::getRegisterName(Reg);
1221 return false;
1222 }
1223 printOperand(MI, OpNum, O);
1224 return false;
1225 case 'b': // Print B register.
1226 case 'h': // Print H register.
1227 case 's': // Print S register.
1228 case 'd': // Print D register.
1229 case 'q': // Print Q register.
1230 case 'z': // Print Z register.
1231 if (MO.isReg()) {
1232 const TargetRegisterClass *RC;
1233 switch (ExtraCode[0]) {
1234 case 'b':
1235 RC = &AArch64::FPR8RegClass;
1236 break;
1237 case 'h':
1238 RC = &AArch64::FPR16RegClass;
1239 break;
1240 case 's':
1241 RC = &AArch64::FPR32RegClass;
1242 break;
1243 case 'd':
1244 RC = &AArch64::FPR64RegClass;
1245 break;
1246 case 'q':
1247 RC = &AArch64::FPR128RegClass;
1248 break;
1249 case 'z':
1250 RC = &AArch64::ZPRRegClass;
1251 break;
1252 default:
1253 return true;
1254 }
1255 return printAsmRegInClass(MO, RC, AltName: AArch64::NoRegAltName, O);
1256 }
1257 printOperand(MI, OpNum, O);
1258 return false;
1259 }
1260 }
1261
1262 // According to ARM, we should emit x and v registers unless we have a
1263 // modifier.
1264 if (MO.isReg()) {
1265 Register Reg = MO.getReg();
1266
1267 // If this is a w or x register, print an x register.
1268 if (AArch64::GPR32allRegClass.contains(Reg) ||
1269 AArch64::GPR64allRegClass.contains(Reg))
1270 return printAsmMRegister(MO, Mode: 'x', O);
1271
1272 // If this is an x register tuple, print an x register.
1273 if (AArch64::GPR64x8ClassRegClass.contains(Reg))
1274 return printAsmMRegister(MO, Mode: 't', O);
1275
1276 unsigned AltName = AArch64::NoRegAltName;
1277 const TargetRegisterClass *RegClass;
1278 if (AArch64::ZPRRegClass.contains(Reg)) {
1279 RegClass = &AArch64::ZPRRegClass;
1280 } else if (AArch64::PPRRegClass.contains(Reg)) {
1281 RegClass = &AArch64::PPRRegClass;
1282 } else if (AArch64::PNRRegClass.contains(Reg)) {
1283 RegClass = &AArch64::PNRRegClass;
1284 } else {
1285 RegClass = &AArch64::FPR128RegClass;
1286 AltName = AArch64::vreg;
1287 }
1288
1289 // If this is a b, h, s, d, or q register, print it as a v register.
1290 return printAsmRegInClass(MO, RC: RegClass, AltName, O);
1291 }
1292
1293 printOperand(MI, OpNum, O);
1294 return false;
1295}
1296
1297bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
1298 unsigned OpNum,
1299 const char *ExtraCode,
1300 raw_ostream &O) {
1301 if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
1302 return true; // Unknown modifier.
1303
1304 const MachineOperand &MO = MI->getOperand(i: OpNum);
1305 assert(MO.isReg() && "unexpected inline asm memory operand");
1306 O << "[" << AArch64InstPrinter::getRegisterName(Reg: MO.getReg()) << "]";
1307 return false;
1308}
1309
1310void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
1311 raw_ostream &OS) {
1312 unsigned NOps = MI->getNumOperands();
1313 assert(NOps == 4);
1314 OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
1315 // cast away const; DIetc do not take const operands for some reason.
1316 OS << MI->getDebugVariable()->getName();
1317 OS << " <- ";
1318 // Frame address. Currently handles register +- offset only.
1319 assert(MI->isIndirectDebugValue());
1320 OS << '[';
1321 for (unsigned I = 0, E = llvm::size(Range: MI->debug_operands()); I < E; ++I) {
1322 if (I != 0)
1323 OS << ", ";
1324 printOperand(MI, OpNum: I, O&: OS);
1325 }
1326 OS << ']';
1327 OS << "+";
1328 printOperand(MI, OpNum: NOps - 2, O&: OS);
1329}
1330
1331void AArch64AsmPrinter::emitJumpTableImpl(const MachineJumpTableInfo &MJTI,
1332 ArrayRef<unsigned> JumpTableIndices) {
1333 // Fast return if there is nothing to emit to avoid creating empty sections.
1334 if (JumpTableIndices.empty())
1335 return;
1336 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
1337 const auto &F = MF->getFunction();
1338 ArrayRef<MachineJumpTableEntry> JT = MJTI.getJumpTables();
1339
1340 MCSection *ReadOnlySec = nullptr;
1341 if (TM.Options.EnableStaticDataPartitioning) {
1342 ReadOnlySec =
1343 TLOF.getSectionForJumpTable(F, TM, JTE: &JT[JumpTableIndices.front()]);
1344 } else {
1345 ReadOnlySec = TLOF.getSectionForJumpTable(F, TM);
1346 }
1347 OutStreamer->switchSection(Section: ReadOnlySec);
1348
1349 auto AFI = MF->getInfo<AArch64FunctionInfo>();
1350 for (unsigned JTI : JumpTableIndices) {
1351 const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
1352
1353 // If this jump table was deleted, ignore it.
1354 if (JTBBs.empty()) continue;
1355
1356 unsigned Size = AFI->getJumpTableEntrySize(Idx: JTI);
1357 emitAlignment(Alignment: Align(Size));
1358 OutStreamer->emitLabel(Symbol: GetJTISymbol(JTID: JTI));
1359
1360 const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(Idx: JTI);
1361 const MCExpr *Base = MCSymbolRefExpr::create(Symbol: BaseSym, Ctx&: OutContext);
1362
1363 for (auto *JTBB : JTBBs) {
1364 const MCExpr *Value =
1365 MCSymbolRefExpr::create(Symbol: JTBB->getSymbol(), Ctx&: OutContext);
1366
1367 // Each entry is:
1368 // .byte/.hword (LBB - Lbase)>>2
1369 // or plain:
1370 // .word LBB - Lbase
1371 Value = MCBinaryExpr::createSub(LHS: Value, RHS: Base, Ctx&: OutContext);
1372 if (Size != 4)
1373 Value = MCBinaryExpr::createLShr(
1374 LHS: Value, RHS: MCConstantExpr::create(Value: 2, Ctx&: OutContext), Ctx&: OutContext);
1375
1376 OutStreamer->emitValue(Value, Size);
1377 }
1378 }
1379}
1380
1381std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
1382 codeview::JumpTableEntrySize>
1383AArch64AsmPrinter::getCodeViewJumpTableInfo(int JTI,
1384 const MachineInstr *BranchInstr,
1385 const MCSymbol *BranchLabel) const {
1386 const auto AFI = MF->getInfo<AArch64FunctionInfo>();
1387 const auto Base = AArch64FI->getJumpTableEntryPCRelSymbol(Idx: JTI);
1388 codeview::JumpTableEntrySize EntrySize;
1389 switch (AFI->getJumpTableEntrySize(Idx: JTI)) {
1390 case 1:
1391 EntrySize = codeview::JumpTableEntrySize::UInt8ShiftLeft;
1392 break;
1393 case 2:
1394 EntrySize = codeview::JumpTableEntrySize::UInt16ShiftLeft;
1395 break;
1396 case 4:
1397 EntrySize = codeview::JumpTableEntrySize::Int32;
1398 break;
1399 default:
1400 llvm_unreachable("Unexpected jump table entry size");
1401 }
1402 return std::make_tuple(args: Base, args: 0, args&: BranchLabel, args&: EntrySize);
1403}
1404
1405void AArch64AsmPrinter::emitFunctionEntryLabel() {
1406 const Triple &TT = TM.getTargetTriple();
1407 if (TT.isOSBinFormatELF() &&
1408 (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall ||
1409 MF->getFunction().getCallingConv() ==
1410 CallingConv::AArch64_SVE_VectorCall ||
1411 MF->getInfo<AArch64FunctionInfo>()->isSVECC())) {
1412 auto *TS =
1413 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
1414 TS->emitDirectiveVariantPCS(Symbol: CurrentFnSym);
1415 }
1416
1417 AsmPrinter::emitFunctionEntryLabel();
1418
1419 if (TT.isWindowsArm64EC() && !MF->getFunction().hasLocalLinkage()) {
1420 // For ARM64EC targets, a function definition's name is mangled differently
1421 // from the normal symbol, emit required aliases here.
1422 auto emitFunctionAlias = [&](MCSymbol *Src, MCSymbol *Dst) {
1423 OutStreamer->emitSymbolAttribute(Symbol: Src, Attribute: MCSA_WeakAntiDep);
1424 OutStreamer->emitAssignment(
1425 Symbol: Src, Value: MCSymbolRefExpr::create(Symbol: Dst, Ctx&: MMI->getContext()));
1426 };
1427
1428 auto getSymbolFromMetadata = [&](StringRef Name) {
1429 MCSymbol *Sym = nullptr;
1430 if (MDNode *Node = MF->getFunction().getMetadata(Kind: Name)) {
1431 StringRef NameStr = cast<MDString>(Val: Node->getOperand(I: 0))->getString();
1432 Sym = MMI->getContext().getOrCreateSymbol(Name: NameStr);
1433 }
1434 return Sym;
1435 };
1436
1437 SmallVector<MDNode *> UnmangledNames;
1438 MF->getFunction().getMetadata(Kind: "arm64ec_unmangled_name", MDs&: UnmangledNames);
1439 for (MDNode *Node : UnmangledNames) {
1440 StringRef NameStr = cast<MDString>(Val: Node->getOperand(I: 0))->getString();
1441 MCSymbol *UnmangledSym = MMI->getContext().getOrCreateSymbol(Name: NameStr);
1442 if (std::optional<std::string> MangledName =
1443 getArm64ECMangledFunctionName(Name: UnmangledSym->getName())) {
1444 MCSymbol *ECMangledSym =
1445 MMI->getContext().getOrCreateSymbol(Name: *MangledName);
1446 emitFunctionAlias(UnmangledSym, ECMangledSym);
1447 }
1448 }
1449 if (MCSymbol *ECMangledSym =
1450 getSymbolFromMetadata("arm64ec_ecmangled_name"))
1451 emitFunctionAlias(ECMangledSym, CurrentFnSym);
1452 }
1453}
1454
1455void AArch64AsmPrinter::emitXXStructor(const DataLayout &DL,
1456 const Constant *CV) {
1457 if (const auto *CPA = dyn_cast<ConstantPtrAuth>(Val: CV))
1458 if (CPA->hasAddressDiscriminator() &&
1459 !CPA->hasSpecialAddressDiscriminator(
1460 Value: ConstantPtrAuth::AddrDiscriminator_CtorsDtors))
1461 report_fatal_error(
1462 reason: "unexpected address discrimination value for ctors/dtors entry, only "
1463 "'ptr inttoptr (i64 1 to ptr)' is allowed");
1464 // If we have signed pointers in xxstructors list, they'll be lowered to @AUTH
1465 // MCExpr's via AArch64AsmPrinter::lowerConstantPtrAuth. It does not look at
1466 // actual address discrimination value and only checks
1467 // hasAddressDiscriminator(), so it's OK to leave special address
1468 // discrimination value here.
1469 AsmPrinter::emitXXStructor(DL, CV);
1470}
1471
1472void AArch64AsmPrinter::emitGlobalAlias(const Module &M,
1473 const GlobalAlias &GA) {
1474 if (auto F = dyn_cast_or_null<Function>(Val: GA.getAliasee())) {
1475 // Global aliases must point to a definition, but unmangled patchable
1476 // symbols are special and need to point to an undefined symbol with "EXP+"
1477 // prefix. Such undefined symbol is resolved by the linker by creating
1478 // x86 thunk that jumps back to the actual EC target.
1479 if (MDNode *Node = F->getMetadata(Kind: "arm64ec_exp_name")) {
1480 StringRef ExpStr = cast<MDString>(Val: Node->getOperand(I: 0))->getString();
1481 MCSymbol *ExpSym = MMI->getContext().getOrCreateSymbol(Name: ExpStr);
1482 MCSymbol *Sym = MMI->getContext().getOrCreateSymbol(Name: GA.getName());
1483
1484 OutStreamer->beginCOFFSymbolDef(Symbol: ExpSym);
1485 OutStreamer->emitCOFFSymbolStorageClass(StorageClass: COFF::IMAGE_SYM_CLASS_EXTERNAL);
1486 OutStreamer->emitCOFFSymbolType(Type: COFF::IMAGE_SYM_DTYPE_FUNCTION
1487 << COFF::SCT_COMPLEX_TYPE_SHIFT);
1488 OutStreamer->endCOFFSymbolDef();
1489
1490 OutStreamer->beginCOFFSymbolDef(Symbol: Sym);
1491 OutStreamer->emitCOFFSymbolStorageClass(StorageClass: COFF::IMAGE_SYM_CLASS_EXTERNAL);
1492 OutStreamer->emitCOFFSymbolType(Type: COFF::IMAGE_SYM_DTYPE_FUNCTION
1493 << COFF::SCT_COMPLEX_TYPE_SHIFT);
1494 OutStreamer->endCOFFSymbolDef();
1495 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Weak);
1496 OutStreamer->emitAssignment(
1497 Symbol: Sym, Value: MCSymbolRefExpr::create(Symbol: ExpSym, Ctx&: MMI->getContext()));
1498 return;
1499 }
1500 }
1501 AsmPrinter::emitGlobalAlias(M, GA);
1502}
1503
1504/// Small jump tables contain an unsigned byte or half, representing the offset
1505/// from the lowest-addressed possible destination to the desired basic
1506/// block. Since all instructions are 4-byte aligned, this is further compressed
1507/// by counting in instructions rather than bytes (i.e. divided by 4). So, to
1508/// materialize the correct destination we need:
1509///
1510/// adr xDest, .LBB0_0
1511/// ldrb wScratch, [xTable, xEntry] (with "lsl #1" for ldrh).
1512/// add xDest, xDest, xScratch (with "lsl #2" for smaller entries)
1513void AArch64AsmPrinter::LowerJumpTableDest(llvm::MCStreamer &OutStreamer,
1514 const llvm::MachineInstr &MI) {
1515 Register DestReg = MI.getOperand(i: 0).getReg();
1516 Register ScratchReg = MI.getOperand(i: 1).getReg();
1517 Register ScratchRegW =
1518 STI->getRegisterInfo()->getSubReg(Reg: ScratchReg, Idx: AArch64::sub_32);
1519 Register TableReg = MI.getOperand(i: 2).getReg();
1520 Register EntryReg = MI.getOperand(i: 3).getReg();
1521 int JTIdx = MI.getOperand(i: 4).getIndex();
1522 int Size = AArch64FI->getJumpTableEntrySize(Idx: JTIdx);
1523
1524 // This has to be first because the compression pass based its reachability
1525 // calculations on the start of the JumpTableDest instruction.
1526 auto Label =
1527 MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(Idx: JTIdx);
1528
1529 // If we don't already have a symbol to use as the base, use the ADR
1530 // instruction itself.
1531 if (!Label) {
1532 Label = MF->getContext().createTempSymbol();
1533 AArch64FI->setJumpTableEntryInfo(Idx: JTIdx, Size, PCRelSym: Label);
1534 OutStreamer.emitLabel(Symbol: Label);
1535 }
1536
1537 auto LabelExpr = MCSymbolRefExpr::create(Symbol: Label, Ctx&: MF->getContext());
1538 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::ADR)
1539 .addReg(Reg: DestReg)
1540 .addExpr(Val: LabelExpr));
1541
1542 // Load the number of instruction-steps to offset from the label.
1543 unsigned LdrOpcode;
1544 switch (Size) {
1545 case 1: LdrOpcode = AArch64::LDRBBroX; break;
1546 case 2: LdrOpcode = AArch64::LDRHHroX; break;
1547 case 4: LdrOpcode = AArch64::LDRSWroX; break;
1548 default:
1549 llvm_unreachable("Unknown jump table size");
1550 }
1551
1552 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(LdrOpcode)
1553 .addReg(Reg: Size == 4 ? ScratchReg : ScratchRegW)
1554 .addReg(Reg: TableReg)
1555 .addReg(Reg: EntryReg)
1556 .addImm(Val: 0)
1557 .addImm(Val: Size == 1 ? 0 : 1));
1558
1559 // Add to the already materialized base label address, multiplying by 4 if
1560 // compressed.
1561 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::ADDXrs)
1562 .addReg(Reg: DestReg)
1563 .addReg(Reg: DestReg)
1564 .addReg(Reg: ScratchReg)
1565 .addImm(Val: Size == 4 ? 0 : 2));
1566}
1567
1568void AArch64AsmPrinter::LowerHardenedBRJumpTable(const MachineInstr &MI) {
1569 const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
1570 assert(MJTI && "Can't lower jump-table dispatch without JTI");
1571
1572 const std::vector<MachineJumpTableEntry> &JTs = MJTI->getJumpTables();
1573 assert(!JTs.empty() && "Invalid JT index for jump-table dispatch");
1574
1575 // Emit:
1576 // mov x17, #<size of table> ; depending on table size, with MOVKs
1577 // cmp x16, x17 ; or #imm if table size fits in 12-bit
1578 // csel x16, x16, xzr, ls ; check for index overflow
1579 //
1580 // adrp x17, Ltable@PAGE ; materialize table address
1581 // add x17, Ltable@PAGEOFF
1582 // ldrsw x16, [x17, x16, lsl #2] ; load table entry
1583 //
1584 // Lanchor:
1585 // adr x17, Lanchor ; compute target address
1586 // add x16, x17, x16
1587 // br x16 ; branch to target
1588
1589 MachineOperand JTOp = MI.getOperand(i: 0);
1590
1591 unsigned JTI = JTOp.getIndex();
1592 assert(!AArch64FI->getJumpTableEntryPCRelSymbol(JTI) &&
1593 "unsupported compressed jump table");
1594
1595 const uint64_t NumTableEntries = JTs[JTI].MBBs.size();
1596
1597 // cmp only supports a 12-bit immediate. If we need more, materialize the
1598 // immediate, using x17 as a scratch register.
1599 uint64_t MaxTableEntry = NumTableEntries - 1;
1600 if (isUInt<12>(x: MaxTableEntry)) {
1601 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::SUBSXri)
1602 .addReg(Reg: AArch64::XZR)
1603 .addReg(Reg: AArch64::X16)
1604 .addImm(Val: MaxTableEntry)
1605 .addImm(Val: 0));
1606 } else {
1607 emitMOVZ(Dest: AArch64::X17, Imm: static_cast<uint16_t>(MaxTableEntry), Shift: 0);
1608 // It's sad that we have to manually materialize instructions, but we can't
1609 // trivially reuse the main pseudo expansion logic.
1610 // A MOVK sequence is easy enough to generate and handles the general case.
1611 for (int Offset = 16; Offset < 64; Offset += 16) {
1612 if ((MaxTableEntry >> Offset) == 0)
1613 break;
1614 emitMOVK(Dest: AArch64::X17, Imm: static_cast<uint16_t>(MaxTableEntry >> Offset),
1615 Shift: Offset);
1616 }
1617 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::SUBSXrs)
1618 .addReg(Reg: AArch64::XZR)
1619 .addReg(Reg: AArch64::X16)
1620 .addReg(Reg: AArch64::X17)
1621 .addImm(Val: 0));
1622 }
1623
1624 // This picks entry #0 on failure.
1625 // We might want to trap instead.
1626 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::CSELXr)
1627 .addReg(Reg: AArch64::X16)
1628 .addReg(Reg: AArch64::X16)
1629 .addReg(Reg: AArch64::XZR)
1630 .addImm(Val: AArch64CC::LS));
1631
1632 // Prepare the @PAGE/@PAGEOFF low/high operands.
1633 MachineOperand JTMOHi(JTOp), JTMOLo(JTOp);
1634 MCOperand JTMCHi, JTMCLo;
1635
1636 JTMOHi.setTargetFlags(AArch64II::MO_PAGE);
1637 JTMOLo.setTargetFlags(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
1638
1639 MCInstLowering.lowerOperand(MO: JTMOHi, MCOp&: JTMCHi);
1640 MCInstLowering.lowerOperand(MO: JTMOLo, MCOp&: JTMCLo);
1641
1642 EmitToStreamer(
1643 S&: *OutStreamer,
1644 Inst: MCInstBuilder(AArch64::ADRP).addReg(Reg: AArch64::X17).addOperand(Op: JTMCHi));
1645
1646 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ADDXri)
1647 .addReg(Reg: AArch64::X17)
1648 .addReg(Reg: AArch64::X17)
1649 .addOperand(Op: JTMCLo)
1650 .addImm(Val: 0));
1651
1652 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::LDRSWroX)
1653 .addReg(Reg: AArch64::X16)
1654 .addReg(Reg: AArch64::X17)
1655 .addReg(Reg: AArch64::X16)
1656 .addImm(Val: 0)
1657 .addImm(Val: 1));
1658
1659 MCSymbol *AdrLabel = MF->getContext().createTempSymbol();
1660 const auto *AdrLabelE = MCSymbolRefExpr::create(Symbol: AdrLabel, Ctx&: MF->getContext());
1661 AArch64FI->setJumpTableEntryInfo(Idx: JTI, Size: 4, PCRelSym: AdrLabel);
1662
1663 OutStreamer->emitLabel(Symbol: AdrLabel);
1664 EmitToStreamer(
1665 S&: *OutStreamer,
1666 Inst: MCInstBuilder(AArch64::ADR).addReg(Reg: AArch64::X17).addExpr(Val: AdrLabelE));
1667
1668 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ADDXrs)
1669 .addReg(Reg: AArch64::X16)
1670 .addReg(Reg: AArch64::X17)
1671 .addReg(Reg: AArch64::X16)
1672 .addImm(Val: 0));
1673
1674 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::BR).addReg(Reg: AArch64::X16));
1675}
1676
1677void AArch64AsmPrinter::LowerMOPS(llvm::MCStreamer &OutStreamer,
1678 const llvm::MachineInstr &MI) {
1679 unsigned Opcode = MI.getOpcode();
1680 assert(STI->hasMOPS());
1681 assert(STI->hasMTE() || Opcode != AArch64::MOPSMemorySetTaggingPseudo);
1682
1683 const auto Ops = [Opcode]() -> std::array<unsigned, 3> {
1684 if (Opcode == AArch64::MOPSMemoryCopyPseudo)
1685 return {AArch64::CPYFP, AArch64::CPYFM, AArch64::CPYFE};
1686 if (Opcode == AArch64::MOPSMemoryMovePseudo)
1687 return {AArch64::CPYP, AArch64::CPYM, AArch64::CPYE};
1688 if (Opcode == AArch64::MOPSMemorySetPseudo)
1689 return {AArch64::SETP, AArch64::SETM, AArch64::SETE};
1690 if (Opcode == AArch64::MOPSMemorySetTaggingPseudo)
1691 return {AArch64::SETGP, AArch64::SETGM, AArch64::MOPSSETGE};
1692 llvm_unreachable("Unhandled memory operation pseudo");
1693 }();
1694 const bool IsSet = Opcode == AArch64::MOPSMemorySetPseudo ||
1695 Opcode == AArch64::MOPSMemorySetTaggingPseudo;
1696
1697 for (auto Op : Ops) {
1698 int i = 0;
1699 auto MCIB = MCInstBuilder(Op);
1700 // Destination registers
1701 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1702 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1703 if (!IsSet)
1704 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1705 // Input registers
1706 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1707 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1708 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1709
1710 EmitToStreamer(S&: OutStreamer, Inst: MCIB);
1711 }
1712}
1713
1714void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
1715 const MachineInstr &MI) {
1716 unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
1717
1718 auto &Ctx = OutStreamer.getContext();
1719 MCSymbol *MILabel = Ctx.createTempSymbol();
1720 OutStreamer.emitLabel(Symbol: MILabel);
1721
1722 SM.recordStackMap(L: *MILabel, MI);
1723 assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1724
1725 // Scan ahead to trim the shadow.
1726 const MachineBasicBlock &MBB = *MI.getParent();
1727 MachineBasicBlock::const_iterator MII(MI);
1728 ++MII;
1729 while (NumNOPBytes > 0) {
1730 if (MII == MBB.end() || MII->isCall() ||
1731 MII->getOpcode() == AArch64::DBG_VALUE ||
1732 MII->getOpcode() == TargetOpcode::PATCHPOINT ||
1733 MII->getOpcode() == TargetOpcode::STACKMAP)
1734 break;
1735 ++MII;
1736 NumNOPBytes -= 4;
1737 }
1738
1739 // Emit nops.
1740 for (unsigned i = 0; i < NumNOPBytes; i += 4)
1741 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::NOP));
1742}
1743
1744// Lower a patchpoint of the form:
1745// [<def>], <id>, <numBytes>, <target>, <numArgs>
1746void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1747 const MachineInstr &MI) {
1748 auto &Ctx = OutStreamer.getContext();
1749 MCSymbol *MILabel = Ctx.createTempSymbol();
1750 OutStreamer.emitLabel(Symbol: MILabel);
1751 SM.recordPatchPoint(L: *MILabel, MI);
1752
1753 PatchPointOpers Opers(&MI);
1754
1755 int64_t CallTarget = Opers.getCallTarget().getImm();
1756 unsigned EncodedBytes = 0;
1757 if (CallTarget) {
1758 assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
1759 "High 16 bits of call target should be zero.");
1760 Register ScratchReg = MI.getOperand(i: Opers.getNextScratchIdx()).getReg();
1761 EncodedBytes = 16;
1762 // Materialize the jump address:
1763 emitMOVZ(Dest: ScratchReg, Imm: (CallTarget >> 32) & 0xFFFF, Shift: 32);
1764 emitMOVK(Dest: ScratchReg, Imm: (CallTarget >> 16) & 0xFFFF, Shift: 16);
1765 emitMOVK(Dest: ScratchReg, Imm: CallTarget & 0xFFFF, Shift: 0);
1766 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::BLR).addReg(Reg: ScratchReg));
1767 }
1768 // Emit padding.
1769 unsigned NumBytes = Opers.getNumPatchBytes();
1770 assert(NumBytes >= EncodedBytes &&
1771 "Patchpoint can't request size less than the length of a call.");
1772 assert((NumBytes - EncodedBytes) % 4 == 0 &&
1773 "Invalid number of NOP bytes requested!");
1774 for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
1775 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::NOP));
1776}
1777
1778void AArch64AsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1779 const MachineInstr &MI) {
1780 StatepointOpers SOpers(&MI);
1781 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
1782 assert(PatchBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1783 for (unsigned i = 0; i < PatchBytes; i += 4)
1784 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::NOP));
1785 } else {
1786 // Lower call target and choose correct opcode
1787 const MachineOperand &CallTarget = SOpers.getCallTarget();
1788 MCOperand CallTargetMCOp;
1789 unsigned CallOpcode;
1790 switch (CallTarget.getType()) {
1791 case MachineOperand::MO_GlobalAddress:
1792 case MachineOperand::MO_ExternalSymbol:
1793 MCInstLowering.lowerOperand(MO: CallTarget, MCOp&: CallTargetMCOp);
1794 CallOpcode = AArch64::BL;
1795 break;
1796 case MachineOperand::MO_Immediate:
1797 CallTargetMCOp = MCOperand::createImm(Val: CallTarget.getImm());
1798 CallOpcode = AArch64::BL;
1799 break;
1800 case MachineOperand::MO_Register:
1801 CallTargetMCOp = MCOperand::createReg(Reg: CallTarget.getReg());
1802 CallOpcode = AArch64::BLR;
1803 break;
1804 default:
1805 llvm_unreachable("Unsupported operand type in statepoint call target");
1806 break;
1807 }
1808
1809 EmitToStreamer(S&: OutStreamer,
1810 Inst: MCInstBuilder(CallOpcode).addOperand(Op: CallTargetMCOp));
1811 }
1812
1813 auto &Ctx = OutStreamer.getContext();
1814 MCSymbol *MILabel = Ctx.createTempSymbol();
1815 OutStreamer.emitLabel(Symbol: MILabel);
1816 SM.recordStatepoint(L: *MILabel, MI);
1817}
1818
1819void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) {
1820 // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
1821 // <opcode>, <operands>
1822
1823 Register DefRegister = FaultingMI.getOperand(i: 0).getReg();
1824 FaultMaps::FaultKind FK =
1825 static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(i: 1).getImm());
1826 MCSymbol *HandlerLabel = FaultingMI.getOperand(i: 2).getMBB()->getSymbol();
1827 unsigned Opcode = FaultingMI.getOperand(i: 3).getImm();
1828 unsigned OperandsBeginIdx = 4;
1829
1830 auto &Ctx = OutStreamer->getContext();
1831 MCSymbol *FaultingLabel = Ctx.createTempSymbol();
1832 OutStreamer->emitLabel(Symbol: FaultingLabel);
1833
1834 assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
1835 FM.recordFaultingOp(FaultTy: FK, FaultingLabel, HandlerLabel);
1836
1837 MCInst MI;
1838 MI.setOpcode(Opcode);
1839
1840 if (DefRegister != (Register)0)
1841 MI.addOperand(Op: MCOperand::createReg(Reg: DefRegister));
1842
1843 for (const MachineOperand &MO :
1844 llvm::drop_begin(RangeOrContainer: FaultingMI.operands(), N: OperandsBeginIdx)) {
1845 MCOperand Dest;
1846 lowerOperand(MO, MCOp&: Dest);
1847 MI.addOperand(Op: Dest);
1848 }
1849
1850 OutStreamer->AddComment(T: "on-fault: " + HandlerLabel->getName());
1851 EmitToStreamer(Inst: MI);
1852}
1853
1854void AArch64AsmPrinter::emitMovXReg(Register Dest, Register Src) {
1855 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ORRXrs)
1856 .addReg(Reg: Dest)
1857 .addReg(Reg: AArch64::XZR)
1858 .addReg(Reg: Src)
1859 .addImm(Val: 0));
1860}
1861
1862void AArch64AsmPrinter::emitMOVZ(Register Dest, uint64_t Imm, unsigned Shift) {
1863 bool Is64Bit = AArch64::GPR64RegClass.contains(Reg: Dest);
1864 EmitToStreamer(S&: *OutStreamer,
1865 Inst: MCInstBuilder(Is64Bit ? AArch64::MOVZXi : AArch64::MOVZWi)
1866 .addReg(Reg: Dest)
1867 .addImm(Val: Imm)
1868 .addImm(Val: Shift));
1869}
1870
1871void AArch64AsmPrinter::emitMOVK(Register Dest, uint64_t Imm, unsigned Shift) {
1872 bool Is64Bit = AArch64::GPR64RegClass.contains(Reg: Dest);
1873 EmitToStreamer(S&: *OutStreamer,
1874 Inst: MCInstBuilder(Is64Bit ? AArch64::MOVKXi : AArch64::MOVKWi)
1875 .addReg(Reg: Dest)
1876 .addReg(Reg: Dest)
1877 .addImm(Val: Imm)
1878 .addImm(Val: Shift));
1879}
1880
1881void AArch64AsmPrinter::emitAUT(AArch64PACKey::ID Key, Register Pointer,
1882 Register Disc) {
1883 bool IsZeroDisc = Disc == AArch64::XZR;
1884 unsigned Opcode = getAUTOpcodeForKey(K: Key, Zero: IsZeroDisc);
1885
1886 // autiza x16 ; if IsZeroDisc
1887 // autia x16, x17 ; if !IsZeroDisc
1888 MCInst AUTInst;
1889 AUTInst.setOpcode(Opcode);
1890 AUTInst.addOperand(Op: MCOperand::createReg(Reg: Pointer));
1891 AUTInst.addOperand(Op: MCOperand::createReg(Reg: Pointer));
1892 if (!IsZeroDisc)
1893 AUTInst.addOperand(Op: MCOperand::createReg(Reg: Disc));
1894
1895 EmitToStreamer(Inst: AUTInst);
1896}
1897
1898void AArch64AsmPrinter::emitPAC(AArch64PACKey::ID Key, Register Pointer,
1899 Register Disc) {
1900 bool IsZeroDisc = Disc == AArch64::XZR;
1901 unsigned Opcode = getPACOpcodeForKey(K: Key, Zero: IsZeroDisc);
1902
1903 // paciza x16 ; if IsZeroDisc
1904 // pacia x16, x17 ; if !IsZeroDisc
1905 MCInst PACInst;
1906 PACInst.setOpcode(Opcode);
1907 PACInst.addOperand(Op: MCOperand::createReg(Reg: Pointer));
1908 PACInst.addOperand(Op: MCOperand::createReg(Reg: Pointer));
1909 if (!IsZeroDisc)
1910 PACInst.addOperand(Op: MCOperand::createReg(Reg: Disc));
1911
1912 EmitToStreamer(Inst: PACInst);
1913}
1914
1915void AArch64AsmPrinter::emitBLRA(bool IsCall, AArch64PACKey::ID Key,
1916 Register Target, Register Disc) {
1917 bool IsZeroDisc = Disc == AArch64::XZR;
1918 unsigned Opcode = getBranchOpcodeForKey(IsCall, K: Key, Zero: IsZeroDisc);
1919
1920 // blraaz x16 ; if IsZeroDisc
1921 // blraa x16, x17 ; if !IsZeroDisc
1922 MCInst Inst;
1923 Inst.setOpcode(Opcode);
1924 Inst.addOperand(Op: MCOperand::createReg(Reg: Target));
1925 if (!IsZeroDisc)
1926 Inst.addOperand(Op: MCOperand::createReg(Reg: Disc));
1927 EmitToStreamer(Inst);
1928}
1929
1930void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) {
1931 Register DestReg = MI.getOperand(i: 0).getReg();
1932 if (!STI->hasZeroCycleZeroingFPWorkaround() && STI->isNeonAvailable()) {
1933 if (STI->hasZeroCycleZeroingFPR64()) {
1934 // Convert H/S register to corresponding D register
1935 const AArch64RegisterInfo *TRI = STI->getRegisterInfo();
1936 if (AArch64::FPR16RegClass.contains(Reg: DestReg))
1937 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
1938 RC: &AArch64::FPR64RegClass);
1939 else if (AArch64::FPR32RegClass.contains(Reg: DestReg))
1940 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::ssub,
1941 RC: &AArch64::FPR64RegClass);
1942 else
1943 assert(AArch64::FPR64RegClass.contains(DestReg));
1944
1945 MCInst MOVI;
1946 MOVI.setOpcode(AArch64::MOVID);
1947 MOVI.addOperand(Op: MCOperand::createReg(Reg: DestReg));
1948 MOVI.addOperand(Op: MCOperand::createImm(Val: 0));
1949 EmitToStreamer(S&: *OutStreamer, Inst: MOVI);
1950 ++NumZCZeroingInstrsFPR;
1951 } else if (STI->hasZeroCycleZeroingFPR128()) {
1952 // Convert H/S/D register to corresponding Q register
1953 const AArch64RegisterInfo *TRI = STI->getRegisterInfo();
1954 if (AArch64::FPR16RegClass.contains(Reg: DestReg)) {
1955 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
1956 RC: &AArch64::FPR128RegClass);
1957 } else if (AArch64::FPR32RegClass.contains(Reg: DestReg)) {
1958 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::ssub,
1959 RC: &AArch64::FPR128RegClass);
1960 } else {
1961 assert(AArch64::FPR64RegClass.contains(DestReg));
1962 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::dsub,
1963 RC: &AArch64::FPR128RegClass);
1964 }
1965
1966 MCInst MOVI;
1967 MOVI.setOpcode(AArch64::MOVIv2d_ns);
1968 MOVI.addOperand(Op: MCOperand::createReg(Reg: DestReg));
1969 MOVI.addOperand(Op: MCOperand::createImm(Val: 0));
1970 EmitToStreamer(S&: *OutStreamer, Inst: MOVI);
1971 ++NumZCZeroingInstrsFPR;
1972 } else {
1973 emitFMov0AsFMov(MI, DestReg);
1974 }
1975 } else {
1976 emitFMov0AsFMov(MI, DestReg);
1977 }
1978}
1979
1980void AArch64AsmPrinter::emitFMov0AsFMov(const MachineInstr &MI,
1981 Register DestReg) {
1982 MCInst FMov;
1983 switch (MI.getOpcode()) {
1984 default:
1985 llvm_unreachable("Unexpected opcode");
1986 case AArch64::FMOVH0:
1987 FMov.setOpcode(STI->hasFullFP16() ? AArch64::FMOVWHr : AArch64::FMOVWSr);
1988 if (!STI->hasFullFP16())
1989 DestReg = (AArch64::S0 + (DestReg - AArch64::H0));
1990 FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg));
1991 FMov.addOperand(Op: MCOperand::createReg(Reg: AArch64::WZR));
1992 break;
1993 case AArch64::FMOVS0:
1994 FMov.setOpcode(AArch64::FMOVWSr);
1995 FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg));
1996 FMov.addOperand(Op: MCOperand::createReg(Reg: AArch64::WZR));
1997 break;
1998 case AArch64::FMOVD0:
1999 FMov.setOpcode(AArch64::FMOVXDr);
2000 FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2001 FMov.addOperand(Op: MCOperand::createReg(Reg: AArch64::XZR));
2002 break;
2003 }
2004 EmitToStreamer(S&: *OutStreamer, Inst: FMov);
2005}
2006
2007Register AArch64AsmPrinter::emitPtrauthDiscriminator(uint64_t Disc,
2008 Register AddrDisc,
2009 Register ScratchReg,
2010 bool MayClobberAddrDisc) {
2011 assert(isPtrauthRegSafe(ScratchReg) &&
2012 "Safe scratch register must be provided by the caller");
2013 assert(isUInt<16>(Disc) && "Constant discriminator is too wide");
2014
2015 // So far we've used NoRegister in pseudos. Now we need real encodings.
2016 if (AddrDisc == AArch64::NoRegister)
2017 AddrDisc = AArch64::XZR;
2018
2019 // If there is no constant discriminator, there's no blend involved:
2020 // just use the address discriminator register as-is (XZR or not).
2021 if (!Disc)
2022 return AddrDisc;
2023
2024 // If there's only a constant discriminator, MOV it into the scratch register.
2025 if (AddrDisc == AArch64::XZR) {
2026 emitMOVZ(Dest: ScratchReg, Imm: Disc, Shift: 0);
2027 return ScratchReg;
2028 }
2029
2030 // If there are both, emit a blend into the scratch register.
2031
2032 // Check if we can save one MOV instruction.
2033 if (MayClobberAddrDisc && isPtrauthRegSafe(Reg: AddrDisc)) {
2034 ScratchReg = AddrDisc;
2035 } else {
2036 emitMovXReg(Dest: ScratchReg, Src: AddrDisc);
2037 assert(ScratchReg != AddrDisc &&
2038 "Forbidden to clobber AddrDisc, but have to");
2039 }
2040
2041 emitMOVK(Dest: ScratchReg, Imm: Disc, Shift: 48);
2042 return ScratchReg;
2043}
2044
2045/// Emit a code sequence to check an authenticated pointer value.
2046///
2047/// This function emits a sequence of instructions that checks if TestedReg was
2048/// authenticated successfully. On success, execution continues at the next
2049/// instruction after the sequence.
2050///
2051/// The action performed on failure depends on the OnFailure argument:
2052/// * if OnFailure is not nullptr, control is transferred to that label after
2053/// clearing the PAC field
2054/// * otherwise, BRK instruction is emitted to generate an error
2055void AArch64AsmPrinter::emitPtrauthCheckAuthenticatedValue(
2056 Register TestedReg, Register ScratchReg, AArch64PACKey::ID Key,
2057 AArch64PAuth::AuthCheckMethod Method, const MCSymbol *OnFailure) {
2058 // Insert a sequence to check if authentication of TestedReg succeeded,
2059 // such as:
2060 //
2061 // - checked and clearing:
2062 // ; x16 is TestedReg, x17 is ScratchReg
2063 // mov x17, x16
2064 // xpaci x17
2065 // cmp x16, x17
2066 // b.eq Lsuccess
2067 // mov x16, x17
2068 // b Lend
2069 // Lsuccess:
2070 // ; skipped if authentication failed
2071 // Lend:
2072 // ...
2073 //
2074 // - checked and trapping:
2075 // mov x17, x16
2076 // xpaci x17
2077 // cmp x16, x17
2078 // b.eq Lsuccess
2079 // brk #<0xc470 + aut key>
2080 // Lsuccess:
2081 // ...
2082 //
2083 // See the documentation on AuthCheckMethod enumeration constants for
2084 // the specific code sequences that can be used to perform the check.
2085 using AArch64PAuth::AuthCheckMethod;
2086
2087 if (Method == AuthCheckMethod::None)
2088 return;
2089 if (Method == AuthCheckMethod::DummyLoad) {
2090 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRWui)
2091 .addReg(Reg: getWRegFromXReg(Reg: ScratchReg))
2092 .addReg(Reg: TestedReg)
2093 .addImm(Val: 0));
2094 assert(!OnFailure && "DummyLoad always traps on error");
2095 return;
2096 }
2097
2098 MCSymbol *SuccessSym = createTempSymbol(Name: "auth_success_");
2099 if (Method == AuthCheckMethod::XPAC || Method == AuthCheckMethod::XPACHint) {
2100 // mov Xscratch, Xtested
2101 emitMovXReg(Dest: ScratchReg, Src: TestedReg);
2102
2103 if (Method == AuthCheckMethod::XPAC) {
2104 // xpac(i|d) Xscratch
2105 unsigned XPACOpc = getXPACOpcodeForKey(K: Key);
2106 EmitToStreamer(
2107 Inst: MCInstBuilder(XPACOpc).addReg(Reg: ScratchReg).addReg(Reg: ScratchReg));
2108 } else {
2109 // xpaclri
2110
2111 // Note that this method applies XPAC to TestedReg instead of ScratchReg.
2112 assert(TestedReg == AArch64::LR &&
2113 "XPACHint mode is only compatible with checking the LR register");
2114 assert((Key == AArch64PACKey::IA || Key == AArch64PACKey::IB) &&
2115 "XPACHint mode is only compatible with I-keys");
2116 EmitToStreamer(Inst: MCInstBuilder(AArch64::XPACLRI));
2117 }
2118
2119 // cmp Xtested, Xscratch
2120 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSXrs)
2121 .addReg(Reg: AArch64::XZR)
2122 .addReg(Reg: TestedReg)
2123 .addReg(Reg: ScratchReg)
2124 .addImm(Val: 0));
2125
2126 // b.eq Lsuccess
2127 EmitToStreamer(
2128 Inst: MCInstBuilder(AArch64::Bcc)
2129 .addImm(Val: AArch64CC::EQ)
2130 .addExpr(Val: MCSymbolRefExpr::create(Symbol: SuccessSym, Ctx&: OutContext)));
2131 } else if (Method == AuthCheckMethod::HighBitsNoTBI) {
2132 // eor Xscratch, Xtested, Xtested, lsl #1
2133 EmitToStreamer(Inst: MCInstBuilder(AArch64::EORXrs)
2134 .addReg(Reg: ScratchReg)
2135 .addReg(Reg: TestedReg)
2136 .addReg(Reg: TestedReg)
2137 .addImm(Val: 1));
2138 // tbz Xscratch, #62, Lsuccess
2139 EmitToStreamer(
2140 Inst: MCInstBuilder(AArch64::TBZX)
2141 .addReg(Reg: ScratchReg)
2142 .addImm(Val: 62)
2143 .addExpr(Val: MCSymbolRefExpr::create(Symbol: SuccessSym, Ctx&: OutContext)));
2144 } else {
2145 llvm_unreachable("Unsupported check method");
2146 }
2147
2148 if (!OnFailure) {
2149 // Trapping sequences do a 'brk'.
2150 // brk #<0xc470 + aut key>
2151 EmitToStreamer(Inst: MCInstBuilder(AArch64::BRK).addImm(Val: 0xc470 | Key));
2152 } else {
2153 // Non-trapping checked sequences return the stripped result in TestedReg,
2154 // skipping over success-only code (such as re-signing the pointer) by
2155 // jumping to OnFailure label.
2156 // Note that this can introduce an authentication oracle (such as based on
2157 // the high bits of the re-signed value).
2158
2159 // FIXME: The XPAC method can be optimized by applying XPAC to TestedReg
2160 // instead of ScratchReg, thus eliminating one `mov` instruction.
2161 // Both XPAC and XPACHint can be further optimized by not using a
2162 // conditional branch jumping over an unconditional one.
2163
2164 switch (Method) {
2165 case AuthCheckMethod::XPACHint:
2166 // LR is already XPAC-ed at this point.
2167 break;
2168 case AuthCheckMethod::XPAC:
2169 // mov Xtested, Xscratch
2170 emitMovXReg(Dest: TestedReg, Src: ScratchReg);
2171 break;
2172 default:
2173 // If Xtested was not XPAC-ed so far, emit XPAC here.
2174 // xpac(i|d) Xtested
2175 unsigned XPACOpc = getXPACOpcodeForKey(K: Key);
2176 EmitToStreamer(
2177 Inst: MCInstBuilder(XPACOpc).addReg(Reg: TestedReg).addReg(Reg: TestedReg));
2178 }
2179
2180 // b Lend
2181 const auto *OnFailureExpr = MCSymbolRefExpr::create(Symbol: OnFailure, Ctx&: OutContext);
2182 EmitToStreamer(Inst: MCInstBuilder(AArch64::B).addExpr(Val: OnFailureExpr));
2183 }
2184
2185 // If the auth check succeeds, we can continue.
2186 // Lsuccess:
2187 OutStreamer->emitLabel(Symbol: SuccessSym);
2188}
2189
2190// With Pointer Authentication, it may be needed to explicitly check the
2191// authenticated value in LR before performing a tail call.
2192// Otherwise, the callee may re-sign the invalid return address,
2193// introducing a signing oracle.
2194void AArch64AsmPrinter::emitPtrauthTailCallHardening(const MachineInstr *TC) {
2195 if (!AArch64FI->shouldSignReturnAddress(MF: *MF))
2196 return;
2197
2198 auto LRCheckMethod = STI->getAuthenticatedLRCheckMethod(MF: *MF);
2199 if (LRCheckMethod == AArch64PAuth::AuthCheckMethod::None)
2200 return;
2201
2202 const AArch64RegisterInfo *TRI = STI->getRegisterInfo();
2203 Register ScratchReg =
2204 TC->readsRegister(Reg: AArch64::X16, TRI) ? AArch64::X17 : AArch64::X16;
2205 assert(!TC->readsRegister(ScratchReg, TRI) &&
2206 "Neither x16 nor x17 is available as a scratch register");
2207 AArch64PACKey::ID Key =
2208 AArch64FI->shouldSignWithBKey() ? AArch64PACKey::IB : AArch64PACKey::IA;
2209 emitPtrauthCheckAuthenticatedValue(TestedReg: AArch64::LR, ScratchReg, Key,
2210 Method: LRCheckMethod);
2211}
2212
2213bool AArch64AsmPrinter::emitDeactivationSymbolRelocation(Value *DS) {
2214 if (!DS)
2215 return false;
2216
2217 if (isa<GlobalAlias>(Val: DS)) {
2218 // Just emit the nop directly.
2219 EmitToStreamer(Inst: MCInstBuilder(AArch64::NOP));
2220 return true;
2221 }
2222 MCSymbol *Dot = OutContext.createTempSymbol();
2223 OutStreamer->emitLabel(Symbol: Dot);
2224 const MCExpr *DeactDotExpr = MCSymbolRefExpr::create(Symbol: Dot, Ctx&: OutContext);
2225
2226 const MCExpr *DSExpr = MCSymbolRefExpr::create(
2227 Symbol: OutContext.getOrCreateSymbol(Name: DS->getName()), Ctx&: OutContext);
2228 OutStreamer->emitRelocDirective(Offset: *DeactDotExpr, Name: "R_AARCH64_PATCHINST", Expr: DSExpr,
2229 Loc: SMLoc());
2230 return false;
2231}
2232
2233AArch64AsmPrinter::PtrAuthSchema::PtrAuthSchema(
2234 AArch64PACKey::ID Key, uint64_t IntDisc, const MachineOperand &AddrDiscOp)
2235 : Key(Key), IntDisc(IntDisc), AddrDisc(AddrDiscOp.getReg()),
2236 AddrDiscIsKilled(AddrDiscOp.isKill()) {}
2237
2238void AArch64AsmPrinter::emitPtrauthAuthResign(
2239 Register Pointer, Register Scratch, PtrAuthSchema AuthSchema,
2240 std::optional<PtrAuthSchema> SignSchema, std::optional<uint64_t> OptAddend,
2241 Value *DS) {
2242 const bool IsResign = SignSchema.has_value();
2243 const bool HasLoad = OptAddend.has_value();
2244 // We expand AUT/AUTPAC into a sequence of the form
2245 //
2246 // ; authenticate x16
2247 // ; check pointer in x16
2248 // Lsuccess:
2249 // ; sign x16 (if AUTPAC)
2250 // Lend: ; if not trapping on failure
2251 //
2252 // with the checking sequence chosen depending on whether/how we should check
2253 // the pointer and whether we should trap on failure.
2254
2255 // By default, auth/resign sequences check for auth failures.
2256 bool ShouldCheck = true;
2257 // In the checked sequence, we only trap if explicitly requested.
2258 bool ShouldTrap = MF->getFunction().hasFnAttribute(Kind: "ptrauth-auth-traps");
2259
2260 // On an FPAC CPU, you get traps whether you want them or not: there's
2261 // no point in emitting checks or traps.
2262 if (STI->hasFPAC())
2263 ShouldCheck = ShouldTrap = false;
2264
2265 // However, command-line flags can override this, for experimentation.
2266 switch (PtrauthAuthChecks) {
2267 case PtrauthCheckMode::Default:
2268 break;
2269 case PtrauthCheckMode::Unchecked:
2270 ShouldCheck = ShouldTrap = false;
2271 break;
2272 case PtrauthCheckMode::Poison:
2273 ShouldCheck = true;
2274 ShouldTrap = false;
2275 break;
2276 case PtrauthCheckMode::Trap:
2277 ShouldCheck = ShouldTrap = true;
2278 break;
2279 }
2280
2281 // Compute aut discriminator
2282 Register AUTDiscReg =
2283 emitPtrauthDiscriminator(Disc: AuthSchema.IntDisc, AddrDisc: AuthSchema.AddrDisc, ScratchReg: Scratch,
2284 MayClobberAddrDisc: AuthSchema.AddrDiscIsKilled);
2285
2286 if (!emitDeactivationSymbolRelocation(DS))
2287 emitAUT(Key: AuthSchema.Key, Pointer, Disc: AUTDiscReg);
2288
2289 // Unchecked or checked-but-non-trapping AUT is just an "AUT": we're done.
2290 if (!IsResign && (!ShouldCheck || !ShouldTrap))
2291 return;
2292
2293 MCSymbol *EndSym = nullptr;
2294
2295 if (ShouldCheck) {
2296 if (IsResign && !ShouldTrap)
2297 EndSym = createTempSymbol(Name: "resign_end_");
2298
2299 emitPtrauthCheckAuthenticatedValue(TestedReg: Pointer, ScratchReg: Scratch, Key: AuthSchema.Key,
2300 Method: AArch64PAuth::AuthCheckMethod::XPAC,
2301 OnFailure: EndSym);
2302 }
2303
2304 // We already emitted unchecked and checked-but-non-trapping AUTs.
2305 // That left us with trapping AUTs, and AUTPA/AUTRELLOADPACs.
2306 // Trapping AUTs don't need PAC: we're done.
2307 if (!IsResign)
2308 return;
2309
2310 if (HasLoad) {
2311 int64_t Addend = *OptAddend;
2312 // incoming rawpointer in X16, X17 is not live at this point.
2313 // LDSRWpre x17, x16, simm9 ; note: x16+simm9 used later.
2314 if (isInt<9>(x: Addend)) {
2315 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::LDRSWpre)
2316 .addReg(Reg: AArch64::X16)
2317 .addReg(Reg: AArch64::X17)
2318 .addReg(Reg: AArch64::X16)
2319 .addImm(/*simm9:*/ Val: Addend));
2320 } else {
2321 // x16 = x16 + Addend computation has 2 variants
2322 if (isUInt<24>(x: Addend)) {
2323 // variant 1: add x16, x16, Addend >> shift12 ls shift12
2324 // This can take upto 2 instructions.
2325 for (int BitPos = 0; BitPos != 24 && (Addend >> BitPos); BitPos += 12) {
2326 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ADDXri)
2327 .addReg(Reg: AArch64::X16)
2328 .addReg(Reg: AArch64::X16)
2329 .addImm(Val: (Addend >> BitPos) & 0xfff)
2330 .addImm(Val: AArch64_AM::getShifterImm(
2331 ST: AArch64_AM::LSL, Imm: BitPos)));
2332 }
2333 } else {
2334 // variant 2: accumulate constant in X17 16 bits at a time, and add to
2335 // X16 This can take 2-5 instructions.
2336 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::MOVZXi)
2337 .addReg(Reg: AArch64::X17)
2338 .addImm(Val: Addend & 0xffff)
2339 .addImm(Val: AArch64_AM::getShifterImm(
2340 ST: AArch64_AM::LSL, Imm: 0)));
2341
2342 for (int Offset = 16; Offset < 64; Offset += 16) {
2343 uint16_t Fragment = static_cast<uint16_t>(Addend >> Offset);
2344 if (!Fragment)
2345 continue;
2346 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::MOVKXi)
2347 .addReg(Reg: AArch64::X17)
2348 .addReg(Reg: AArch64::X17)
2349 .addImm(Val: Fragment)
2350 .addImm(/*shift:*/ Val: Offset));
2351 }
2352 // addx x16, x16, x17
2353 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ADDXrs)
2354 .addReg(Reg: AArch64::X16)
2355 .addReg(Reg: AArch64::X16)
2356 .addReg(Reg: AArch64::X17)
2357 .addImm(Val: 0));
2358 }
2359 // ldrsw x17,x16(0)
2360 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::LDRSWui)
2361 .addReg(Reg: AArch64::X17)
2362 .addReg(Reg: AArch64::X16)
2363 .addImm(Val: 0));
2364 }
2365 // addx x16, x16, x17
2366 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ADDXrs)
2367 .addReg(Reg: AArch64::X16)
2368 .addReg(Reg: AArch64::X16)
2369 .addReg(Reg: AArch64::X17)
2370 .addImm(Val: 0));
2371
2372 } /* HasLoad == true */
2373
2374 // Compute pac discriminator into x17
2375 Register PACDiscReg = emitPtrauthDiscriminator(Disc: SignSchema->IntDisc,
2376 AddrDisc: SignSchema->AddrDisc, ScratchReg: Scratch);
2377 emitPAC(Key: SignSchema->Key, Pointer, Disc: PACDiscReg);
2378
2379 // Lend:
2380 if (EndSym)
2381 OutStreamer->emitLabel(Symbol: EndSym);
2382}
2383
2384void AArch64AsmPrinter::emitPtrauthSign(const MachineInstr *MI) {
2385 Register Val = MI->getOperand(i: 1).getReg();
2386 auto Key = (AArch64PACKey::ID)MI->getOperand(i: 2).getImm();
2387 uint64_t Disc = MI->getOperand(i: 3).getImm();
2388 Register AddrDisc = MI->getOperand(i: 4).getReg();
2389 bool AddrDiscKilled = MI->getOperand(i: 4).isKill();
2390
2391 // As long as at least one of Val and AddrDisc is in GPR64noip, a scratch
2392 // register is available.
2393 Register ScratchReg = Val == AArch64::X16 ? AArch64::X17 : AArch64::X16;
2394 assert(ScratchReg != AddrDisc &&
2395 "Neither X16 nor X17 is available as a scratch register");
2396
2397 // Compute pac discriminator
2398 Register DiscReg = emitPtrauthDiscriminator(
2399 Disc, AddrDisc, ScratchReg, /*MayClobberAddrDisc=*/AddrDiscKilled);
2400
2401 if (emitDeactivationSymbolRelocation(DS: MI->getDeactivationSymbol()))
2402 return;
2403
2404 emitPAC(Key, Pointer: Val, Disc: DiscReg);
2405}
2406
2407void AArch64AsmPrinter::emitPtrauthBranch(const MachineInstr *MI) {
2408 bool IsCall = MI->getOpcode() == AArch64::BLRA;
2409 unsigned BrTarget = MI->getOperand(i: 0).getReg();
2410
2411 auto Key = (AArch64PACKey::ID)MI->getOperand(i: 1).getImm();
2412 uint64_t Disc = MI->getOperand(i: 2).getImm();
2413
2414 unsigned AddrDisc = MI->getOperand(i: 3).getReg();
2415
2416 // Make sure AddrDisc is solely used to compute the discriminator.
2417 // While hardly meaningful, it is still possible to describe an authentication
2418 // of a pointer against its own value (instead of storage address) with
2419 // intrinsics, so use report_fatal_error instead of assert.
2420 if (BrTarget == AddrDisc)
2421 report_fatal_error(reason: "Branch target is signed with its own value");
2422
2423 // If we are printing BLRA pseudo, try to save one MOV by making use of the
2424 // fact that x16 and x17 are described as clobbered by the MI instruction and
2425 // AddrDisc is not used as any other input.
2426 //
2427 // Back in the day, emitPtrauthDiscriminator was restricted to only returning
2428 // either x16 or x17, meaning the returned register is always among the
2429 // implicit-def'ed registers of BLRA pseudo. Now this property can be violated
2430 // if isX16X17Safer predicate is false, thus manually check if AddrDisc is
2431 // among x16 and x17 to prevent clobbering unexpected registers.
2432 //
2433 // Unlike BLRA, BRA pseudo is used to perform computed goto, and thus not
2434 // declared as clobbering x16/x17.
2435 //
2436 // FIXME: Make use of `killed` flags and register masks instead.
2437 bool AddrDiscIsImplicitDef =
2438 IsCall && (AddrDisc == AArch64::X16 || AddrDisc == AArch64::X17);
2439 Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, ScratchReg: AArch64::X17,
2440 MayClobberAddrDisc: AddrDiscIsImplicitDef);
2441 emitBLRA(IsCall, Key, Target: BrTarget, Disc: DiscReg);
2442}
2443
2444void AArch64AsmPrinter::emitAddImm(MCRegister Reg, int64_t Addend,
2445 MCRegister Tmp) {
2446 if (Addend != 0) {
2447 const uint64_t AbsOffset = (Addend > 0 ? Addend : -((uint64_t)Addend));
2448 const bool IsNeg = Addend < 0;
2449 if (isUInt<24>(x: AbsOffset)) {
2450 for (int BitPos = 0; BitPos != 24 && (AbsOffset >> BitPos);
2451 BitPos += 12) {
2452 EmitToStreamer(
2453 Inst: MCInstBuilder(IsNeg ? AArch64::SUBXri : AArch64::ADDXri)
2454 .addReg(Reg)
2455 .addReg(Reg)
2456 .addImm(Val: (AbsOffset >> BitPos) & 0xfff)
2457 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: BitPos)));
2458 }
2459 } else {
2460 const uint64_t UAddend = Addend;
2461 EmitToStreamer(Inst: MCInstBuilder(IsNeg ? AArch64::MOVNXi : AArch64::MOVZXi)
2462 .addReg(Reg: Tmp)
2463 .addImm(Val: (IsNeg ? ~UAddend : UAddend) & 0xffff)
2464 .addImm(/*shift=*/Val: 0));
2465 auto NeedMovk = [IsNeg, UAddend](int BitPos) -> bool {
2466 assert(BitPos == 16 || BitPos == 32 || BitPos == 48);
2467 uint64_t Shifted = UAddend >> BitPos;
2468 if (!IsNeg)
2469 return Shifted != 0;
2470 for (int I = 0; I != 64 - BitPos; I += 16)
2471 if (((Shifted >> I) & 0xffff) != 0xffff)
2472 return true;
2473 return false;
2474 };
2475 for (int BitPos = 16; BitPos != 64 && NeedMovk(BitPos); BitPos += 16)
2476 emitMOVK(Dest: Tmp, Imm: (UAddend >> BitPos) & 0xffff, Shift: BitPos);
2477
2478 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXrs)
2479 .addReg(Reg)
2480 .addReg(Reg)
2481 .addReg(Reg: Tmp)
2482 .addImm(/*shift=*/Val: 0));
2483 }
2484 }
2485}
2486
2487void AArch64AsmPrinter::emitAddress(MCRegister Reg, const MCExpr *Expr,
2488 MCRegister Tmp, bool DSOLocal,
2489 const MCSubtargetInfo &STI) {
2490 MCValue Val;
2491 if (!Expr->evaluateAsRelocatable(Res&: Val, Asm: nullptr))
2492 report_fatal_error(reason: "emitAddress could not evaluate");
2493 if (DSOLocal) {
2494 EmitToStreamer(
2495 Inst: MCInstBuilder(AArch64::ADRP)
2496 .addReg(Reg)
2497 .addExpr(Val: MCSpecifierExpr::create(Expr, S: AArch64::S_ABS_PAGE,
2498 Ctx&: OutStreamer->getContext())));
2499 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
2500 .addReg(Reg)
2501 .addReg(Reg)
2502 .addExpr(Val: MCSpecifierExpr::create(
2503 Expr, S: AArch64::S_LO12, Ctx&: OutStreamer->getContext()))
2504 .addImm(Val: 0));
2505 } else {
2506 auto *SymRef =
2507 MCSymbolRefExpr::create(Symbol: Val.getAddSym(), Ctx&: OutStreamer->getContext());
2508 EmitToStreamer(
2509 Inst: MCInstBuilder(AArch64::ADRP)
2510 .addReg(Reg)
2511 .addExpr(Val: MCSpecifierExpr::create(Expr: SymRef, S: AArch64::S_GOT_PAGE,
2512 Ctx&: OutStreamer->getContext())));
2513 EmitToStreamer(
2514 Inst: MCInstBuilder(AArch64::LDRXui)
2515 .addReg(Reg)
2516 .addReg(Reg)
2517 .addExpr(Val: MCSpecifierExpr::create(Expr: SymRef, S: AArch64::S_GOT_LO12,
2518 Ctx&: OutStreamer->getContext())));
2519 emitAddImm(Reg, Addend: Val.getConstant(), Tmp);
2520 }
2521}
2522
2523static bool targetSupportsIRelativeRelocation(const Triple &TT) {
2524 // IFUNCs are ELF-only.
2525 if (!TT.isOSBinFormatELF())
2526 return false;
2527
2528 // IFUNCs are supported on glibc, bionic, and some but not all of the BSDs.
2529 return TT.isOSGlibc() || TT.isAndroid() || TT.isOSFreeBSD() ||
2530 TT.isOSDragonFly() || TT.isOSNetBSD();
2531}
2532
2533// Emit an ifunc resolver that returns a signed pointer to the specified target,
2534// and return a FUNCINIT reference to the resolver. In the linked binary, this
2535// function becomes the target of an IRELATIVE relocation. This resolver is used
2536// to relocate signed pointers in global variable initializers in special cases
2537// where the standard R_AARCH64_AUTH_ABS64 relocation would not work.
2538//
2539// Example (signed null pointer, not address discriminated):
2540//
2541// .8byte .Lpauth_ifunc0
2542// .pushsection .text.startup,"ax",@progbits
2543// .Lpauth_ifunc0:
2544// mov x0, #0
2545// mov x1, #12345
2546// b __emupac_pacda
2547//
2548// Example (signed null pointer, address discriminated):
2549//
2550// .Ltmp:
2551// .8byte .Lpauth_ifunc0
2552// .pushsection .text.startup,"ax",@progbits
2553// .Lpauth_ifunc0:
2554// mov x0, #0
2555// adrp x1, .Ltmp
2556// add x1, x1, :lo12:.Ltmp
2557// b __emupac_pacda
2558// .popsection
2559//
2560// Example (signed pointer to symbol, not address discriminated):
2561//
2562// .Ltmp:
2563// .8byte .Lpauth_ifunc0
2564// .pushsection .text.startup,"ax",@progbits
2565// .Lpauth_ifunc0:
2566// adrp x0, symbol
2567// add x0, x0, :lo12:symbol
2568// mov x1, #12345
2569// b __emupac_pacda
2570// .popsection
2571//
2572// Example (signed null pointer, not address discriminated, with deactivation
2573// symbol ds):
2574//
2575// .8byte .Lpauth_ifunc0
2576// .pushsection .text.startup,"ax",@progbits
2577// .Lpauth_ifunc0:
2578// mov x0, #0
2579// mov x1, #12345
2580// .reloc ., R_AARCH64_PATCHINST, ds
2581// b __emupac_pacda
2582// ret
2583// .popsection
2584const MCExpr *AArch64AsmPrinter::emitPAuthRelocationAsIRelative(
2585 const MCExpr *Target, uint64_t Disc, AArch64PACKey::ID KeyID,
2586 bool HasAddressDiversity, bool IsDSOLocal, const MCExpr *DSExpr) {
2587 const Triple &TT = TM.getTargetTriple();
2588
2589 // We only emit an IRELATIVE relocation if the target supports IRELATIVE.
2590 if (!targetSupportsIRelativeRelocation(TT))
2591 return nullptr;
2592
2593 // For now, only the DA key is supported.
2594 if (KeyID != AArch64PACKey::DA)
2595 return nullptr;
2596
2597 // AArch64Subtarget is huge, so heap allocate it so we don't run out of stack
2598 // space.
2599 auto STI = std::make_unique<AArch64Subtarget>(
2600 args: TT, args: TM.getTargetCPU(), args: TM.getTargetCPU(), args: TM.getTargetFeatureString(), args&: TM,
2601 args: true);
2602 this->STI = STI.get();
2603
2604 MCSymbol *Place = OutStreamer->getContext().createTempSymbol();
2605 OutStreamer->emitLabel(Symbol: Place);
2606 OutStreamer->pushSection();
2607
2608 const MCSymbolELF *Group =
2609 static_cast<MCSectionELF *>(OutStreamer->getCurrentSectionOnly())
2610 ->getGroup();
2611 auto Flags = ELF::SHF_ALLOC | ELF::SHF_EXECINSTR;
2612 if (Group)
2613 Flags |= ELF::SHF_GROUP;
2614 OutStreamer->switchSection(Section: OutStreamer->getContext().getELFSection(
2615 Section: ".text.startup", Type: ELF::SHT_PROGBITS, Flags, EntrySize: 0, Group, IsComdat: true,
2616 UniqueID: Group ? MCSection::NonUniqueID : PAuthIFuncNextUniqueID++, LinkedToSym: nullptr));
2617
2618 MCSymbol *IRelativeSym =
2619 OutStreamer->getContext().createLinkerPrivateSymbol(Name: "pauth_ifunc");
2620 OutStreamer->emitLabel(Symbol: IRelativeSym);
2621 if (isa<MCConstantExpr>(Val: Target)) {
2622 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::MOVZXi)
2623 .addReg(Reg: AArch64::X0)
2624 .addExpr(Val: Target)
2625 .addImm(Val: 0),
2626 STI: *STI);
2627 } else {
2628 emitAddress(Reg: AArch64::X0, Expr: Target, Tmp: AArch64::X16, DSOLocal: IsDSOLocal, STI: *STI);
2629 }
2630 if (HasAddressDiversity) {
2631 auto *PlacePlusDisc = MCBinaryExpr::createAdd(
2632 LHS: MCSymbolRefExpr::create(Symbol: Place, Ctx&: OutStreamer->getContext()),
2633 RHS: MCConstantExpr::create(Value: Disc, Ctx&: OutStreamer->getContext()),
2634 Ctx&: OutStreamer->getContext());
2635 emitAddress(Reg: AArch64::X1, Expr: PlacePlusDisc, Tmp: AArch64::X16, /*IsDSOLocal=*/DSOLocal: true,
2636 STI: *STI);
2637 } else {
2638 if (!isUInt<16>(x: Disc)) {
2639 OutContext.reportError(L: SMLoc(), Msg: "AArch64 PAC Discriminator '" +
2640 Twine(Disc) +
2641 "' out of range [0, 0xFFFF]");
2642 }
2643 emitMOVZ(Dest: AArch64::X1, Imm: Disc, Shift: 0);
2644 }
2645
2646 if (DSExpr) {
2647 MCSymbol *PrePACInst = OutStreamer->getContext().createTempSymbol();
2648 OutStreamer->emitLabel(Symbol: PrePACInst);
2649
2650 auto *PrePACInstExpr =
2651 MCSymbolRefExpr::create(Symbol: PrePACInst, Ctx&: OutStreamer->getContext());
2652 OutStreamer->emitRelocDirective(Offset: *PrePACInstExpr, Name: "R_AARCH64_PATCHINST",
2653 Expr: DSExpr, Loc: SMLoc());
2654 }
2655
2656 // We don't know the subtarget because this is being emitted for a global
2657 // initializer. Because the performance of IFUNC resolvers is unimportant, we
2658 // always call the EmuPAC runtime, which will end up using the PAC instruction
2659 // if the target supports PAC.
2660 MCSymbol *EmuPAC =
2661 OutStreamer->getContext().getOrCreateSymbol(Name: "__emupac_pacda");
2662 const MCSymbolRefExpr *EmuPACRef =
2663 MCSymbolRefExpr::create(Symbol: EmuPAC, Ctx&: OutStreamer->getContext());
2664 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::B).addExpr(Val: EmuPACRef),
2665 STI: *STI);
2666
2667 // We need a RET despite the above tail call because the deactivation symbol
2668 // may replace the tail call with a NOP.
2669 if (DSExpr)
2670 OutStreamer->emitInstruction(
2671 Inst: MCInstBuilder(AArch64::RET).addReg(Reg: AArch64::LR), STI: *STI);
2672 OutStreamer->popSection();
2673
2674 return MCSymbolRefExpr::create(Symbol: IRelativeSym, specifier: AArch64::S_FUNCINIT,
2675 Ctx&: OutStreamer->getContext());
2676}
2677
2678const MCExpr *
2679AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) {
2680 MCContext &Ctx = OutContext;
2681
2682 // Figure out the base symbol and the addend, if any.
2683 APInt Offset(64, 0);
2684 const Value *BaseGV = CPA.getPointer()->stripAndAccumulateConstantOffsets(
2685 DL: getDataLayout(), Offset, /*AllowNonInbounds=*/true);
2686
2687 auto *BaseGVB = dyn_cast<GlobalValue>(Val: BaseGV);
2688
2689 const MCExpr *Sym;
2690 if (BaseGVB) {
2691 // If there is an addend, turn that into the appropriate MCExpr.
2692 Sym = MCSymbolRefExpr::create(Symbol: getSymbol(GV: BaseGVB), Ctx);
2693 if (Offset.sgt(RHS: 0))
2694 Sym = MCBinaryExpr::createAdd(
2695 LHS: Sym, RHS: MCConstantExpr::create(Value: Offset.getSExtValue(), Ctx), Ctx);
2696 else if (Offset.slt(RHS: 0))
2697 Sym = MCBinaryExpr::createSub(
2698 LHS: Sym, RHS: MCConstantExpr::create(Value: (-Offset).getSExtValue(), Ctx), Ctx);
2699 } else {
2700 Sym = MCConstantExpr::create(Value: Offset.getSExtValue(), Ctx);
2701 }
2702
2703 const MCExpr *DSExpr = nullptr;
2704 if (auto *DS = dyn_cast<GlobalValue>(Val: CPA.getDeactivationSymbol())) {
2705 if (isa<GlobalAlias>(Val: DS))
2706 return Sym;
2707 DSExpr = MCSymbolRefExpr::create(Symbol: getSymbol(GV: DS), Ctx);
2708 }
2709
2710 uint64_t KeyID = CPA.getKey()->getZExtValue();
2711 // We later rely on valid KeyID value in AArch64PACKeyIDToString call from
2712 // AArch64AuthMCExpr::printImpl, so fail fast.
2713 if (KeyID > AArch64PACKey::LAST) {
2714 CPA.getContext().emitError(ErrorStr: "AArch64 PAC Key ID '" + Twine(KeyID) +
2715 "' out of range [0, " +
2716 Twine((unsigned)AArch64PACKey::LAST) + "]");
2717 KeyID = 0;
2718 }
2719
2720 uint64_t Disc = CPA.getDiscriminator()->getZExtValue();
2721
2722 // Check if we can represent this with an IRELATIVE and emit it if so.
2723 if (auto *IFuncSym = emitPAuthRelocationAsIRelative(
2724 Target: Sym, Disc, KeyID: AArch64PACKey::ID(KeyID), HasAddressDiversity: CPA.hasAddressDiscriminator(),
2725 IsDSOLocal: BaseGVB && BaseGVB->isDSOLocal(), DSExpr))
2726 return IFuncSym;
2727
2728 if (!isUInt<16>(x: Disc)) {
2729 CPA.getContext().emitError(ErrorStr: "AArch64 PAC Discriminator '" + Twine(Disc) +
2730 "' out of range [0, 0xFFFF]");
2731 Disc = 0;
2732 }
2733
2734 if (DSExpr)
2735 report_fatal_error(reason: "deactivation symbols unsupported in constant "
2736 "expressions on this target");
2737
2738 // Finally build the complete @AUTH expr.
2739 return AArch64AuthMCExpr::create(Expr: Sym, Discriminator: Disc, Key: AArch64PACKey::ID(KeyID),
2740 HasAddressDiversity: CPA.hasAddressDiscriminator(), Ctx);
2741}
2742
2743void AArch64AsmPrinter::LowerLOADauthptrstatic(const MachineInstr &MI) {
2744 unsigned DstReg = MI.getOperand(i: 0).getReg();
2745 const MachineOperand &GAOp = MI.getOperand(i: 1);
2746 const uint64_t KeyC = MI.getOperand(i: 2).getImm();
2747 assert(KeyC <= AArch64PACKey::LAST &&
2748 "key is out of range [0, AArch64PACKey::LAST]");
2749 const auto Key = (AArch64PACKey::ID)KeyC;
2750 const uint64_t Disc = MI.getOperand(i: 3).getImm();
2751 assert(isUInt<16>(Disc) &&
2752 "constant discriminator is out of range [0, 0xffff]");
2753
2754 // Emit instruction sequence like the following:
2755 // ADRP x16, symbol$auth_ptr$key$disc
2756 // LDR x16, [x16, :lo12:symbol$auth_ptr$key$disc]
2757 //
2758 // Where the $auth_ptr$ symbol is the stub slot containing the signed pointer
2759 // to symbol.
2760 MCSymbol *AuthPtrStubSym;
2761 if (TM.getTargetTriple().isOSBinFormatELF()) {
2762 const auto &TLOF =
2763 static_cast<const AArch64_ELFTargetObjectFile &>(getObjFileLowering());
2764
2765 assert(GAOp.getOffset() == 0 &&
2766 "non-zero offset for $auth_ptr$ stub slots is not supported");
2767 const MCSymbol *GASym = TM.getSymbol(GV: GAOp.getGlobal());
2768 AuthPtrStubSym = TLOF.getAuthPtrSlotSymbol(TM, MMI, RawSym: GASym, Key, Discriminator: Disc);
2769 } else {
2770 assert(TM.getTargetTriple().isOSBinFormatMachO() &&
2771 "LOADauthptrstatic is implemented only for MachO/ELF");
2772
2773 const auto &TLOF = static_cast<const AArch64_MachoTargetObjectFile &>(
2774 getObjFileLowering());
2775
2776 assert(GAOp.getOffset() == 0 &&
2777 "non-zero offset for $auth_ptr$ stub slots is not supported");
2778 const MCSymbol *GASym = TM.getSymbol(GV: GAOp.getGlobal());
2779 AuthPtrStubSym = TLOF.getAuthPtrSlotSymbol(TM, MMI, RawSym: GASym, Key, Discriminator: Disc);
2780 }
2781
2782 MachineOperand StubMOHi =
2783 MachineOperand::CreateMCSymbol(Sym: AuthPtrStubSym, TargetFlags: AArch64II::MO_PAGE);
2784 MachineOperand StubMOLo = MachineOperand::CreateMCSymbol(
2785 Sym: AuthPtrStubSym, TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2786 MCOperand StubMCHi, StubMCLo;
2787
2788 MCInstLowering.lowerOperand(MO: StubMOHi, MCOp&: StubMCHi);
2789 MCInstLowering.lowerOperand(MO: StubMOLo, MCOp&: StubMCLo);
2790
2791 EmitToStreamer(
2792 S&: *OutStreamer,
2793 Inst: MCInstBuilder(AArch64::ADRP).addReg(Reg: DstReg).addOperand(Op: StubMCHi));
2794
2795 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::LDRXui)
2796 .addReg(Reg: DstReg)
2797 .addReg(Reg: DstReg)
2798 .addOperand(Op: StubMCLo));
2799}
2800
2801void AArch64AsmPrinter::LowerMOVaddrPAC(const MachineInstr &MI) {
2802 const bool IsGOTLoad = MI.getOpcode() == AArch64::LOADgotPAC;
2803 const bool IsELFSignedGOT = MI.getParent()
2804 ->getParent()
2805 ->getInfo<AArch64FunctionInfo>()
2806 ->hasELFSignedGOT();
2807 MachineOperand GAOp = MI.getOperand(i: 0);
2808 const uint64_t KeyC = MI.getOperand(i: 1).getImm();
2809 assert(KeyC <= AArch64PACKey::LAST &&
2810 "key is out of range [0, AArch64PACKey::LAST]");
2811 const auto Key = (AArch64PACKey::ID)KeyC;
2812 const unsigned AddrDisc = MI.getOperand(i: 2).getReg();
2813 const uint64_t Disc = MI.getOperand(i: 3).getImm();
2814
2815 const int64_t Offset = GAOp.getOffset();
2816 GAOp.setOffset(0);
2817
2818 // Emit:
2819 // target materialization:
2820 // - via GOT:
2821 // - unsigned GOT:
2822 // adrp x16, :got:target
2823 // ldr x16, [x16, :got_lo12:target]
2824 // add offset to x16 if offset != 0
2825 // - ELF signed GOT:
2826 // adrp x17, :got:target
2827 // add x17, x17, :got_auth_lo12:target
2828 // ldr x16, [x17]
2829 // aut{i|d}a x16, x17
2830 // check+trap sequence (if no FPAC)
2831 // add offset to x16 if offset != 0
2832 //
2833 // - direct:
2834 // adrp x16, target
2835 // add x16, x16, :lo12:target
2836 // add offset to x16 if offset != 0
2837 //
2838 // add offset to x16:
2839 // - abs(offset) fits 24 bits:
2840 // add/sub x16, x16, #<offset>[, #lsl 12] (up to 2 instructions)
2841 // - abs(offset) does not fit 24 bits:
2842 // - offset < 0:
2843 // movn+movk sequence filling x17 register with the offset (up to 4
2844 // instructions)
2845 // add x16, x16, x17
2846 // - offset > 0:
2847 // movz+movk sequence filling x17 register with the offset (up to 4
2848 // instructions)
2849 // add x16, x16, x17
2850 //
2851 // signing:
2852 // - 0 discriminator:
2853 // paciza x16
2854 // - Non-0 discriminator, no address discriminator:
2855 // mov x17, #Disc
2856 // pacia x16, x17
2857 // - address discriminator (with potentially folded immediate discriminator):
2858 // pacia x16, xAddrDisc
2859
2860 MachineOperand GAMOHi(GAOp), GAMOLo(GAOp);
2861 MCOperand GAMCHi, GAMCLo;
2862
2863 GAMOHi.setTargetFlags(AArch64II::MO_PAGE);
2864 GAMOLo.setTargetFlags(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2865 if (IsGOTLoad) {
2866 GAMOHi.addTargetFlag(F: AArch64II::MO_GOT);
2867 GAMOLo.addTargetFlag(F: AArch64II::MO_GOT);
2868 }
2869
2870 MCInstLowering.lowerOperand(MO: GAMOHi, MCOp&: GAMCHi);
2871 MCInstLowering.lowerOperand(MO: GAMOLo, MCOp&: GAMCLo);
2872
2873 EmitToStreamer(
2874 Inst: MCInstBuilder(AArch64::ADRP)
2875 .addReg(Reg: IsGOTLoad && IsELFSignedGOT ? AArch64::X17 : AArch64::X16)
2876 .addOperand(Op: GAMCHi));
2877
2878 if (IsGOTLoad) {
2879 if (IsELFSignedGOT) {
2880 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
2881 .addReg(Reg: AArch64::X17)
2882 .addReg(Reg: AArch64::X17)
2883 .addOperand(Op: GAMCLo)
2884 .addImm(Val: 0));
2885
2886 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
2887 .addReg(Reg: AArch64::X16)
2888 .addReg(Reg: AArch64::X17)
2889 .addImm(Val: 0));
2890
2891 assert(GAOp.isGlobal());
2892 assert(GAOp.getGlobal()->getValueType() != nullptr);
2893
2894 bool IsFunctionTy = GAOp.getGlobal()->getValueType()->isFunctionTy();
2895 auto AuthKey = IsFunctionTy ? AArch64PACKey::IA : AArch64PACKey::DA;
2896 emitAUT(Key: AuthKey, Pointer: AArch64::X16, Disc: AArch64::X17);
2897
2898 if (!STI->hasFPAC())
2899 emitPtrauthCheckAuthenticatedValue(TestedReg: AArch64::X16, ScratchReg: AArch64::X17, Key: AuthKey,
2900 Method: AArch64PAuth::AuthCheckMethod::XPAC);
2901 } else {
2902 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
2903 .addReg(Reg: AArch64::X16)
2904 .addReg(Reg: AArch64::X16)
2905 .addOperand(Op: GAMCLo));
2906 }
2907 } else {
2908 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
2909 .addReg(Reg: AArch64::X16)
2910 .addReg(Reg: AArch64::X16)
2911 .addOperand(Op: GAMCLo)
2912 .addImm(Val: 0));
2913 }
2914
2915 emitAddImm(Reg: AArch64::X16, Addend: Offset, Tmp: AArch64::X17);
2916 Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, ScratchReg: AArch64::X17);
2917
2918 emitPAC(Key, Pointer: AArch64::X16, Disc: DiscReg);
2919}
2920
2921void AArch64AsmPrinter::LowerLOADgotAUTH(const MachineInstr &MI) {
2922 Register DstReg = MI.getOperand(i: 0).getReg();
2923 Register AuthResultReg = STI->hasFPAC() ? DstReg : AArch64::X16;
2924 const MachineOperand &GAMO = MI.getOperand(i: 1);
2925 assert(GAMO.getOffset() == 0);
2926
2927 if (MI.getMF()->getTarget().getCodeModel() == CodeModel::Tiny) {
2928 MCOperand GAMC;
2929 MCInstLowering.lowerOperand(MO: GAMO, MCOp&: GAMC);
2930 EmitToStreamer(
2931 Inst: MCInstBuilder(AArch64::ADR).addReg(Reg: AArch64::X17).addOperand(Op: GAMC));
2932 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
2933 .addReg(Reg: AuthResultReg)
2934 .addReg(Reg: AArch64::X17)
2935 .addImm(Val: 0));
2936 } else {
2937 MachineOperand GAHiOp(GAMO);
2938 MachineOperand GALoOp(GAMO);
2939 GAHiOp.addTargetFlag(F: AArch64II::MO_PAGE);
2940 GALoOp.addTargetFlag(F: AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2941
2942 MCOperand GAMCHi, GAMCLo;
2943 MCInstLowering.lowerOperand(MO: GAHiOp, MCOp&: GAMCHi);
2944 MCInstLowering.lowerOperand(MO: GALoOp, MCOp&: GAMCLo);
2945
2946 EmitToStreamer(
2947 Inst: MCInstBuilder(AArch64::ADRP).addReg(Reg: AArch64::X17).addOperand(Op: GAMCHi));
2948
2949 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
2950 .addReg(Reg: AArch64::X17)
2951 .addReg(Reg: AArch64::X17)
2952 .addOperand(Op: GAMCLo)
2953 .addImm(Val: 0));
2954
2955 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
2956 .addReg(Reg: AuthResultReg)
2957 .addReg(Reg: AArch64::X17)
2958 .addImm(Val: 0));
2959 }
2960
2961 assert(GAMO.isGlobal());
2962 MCSymbol *UndefWeakSym;
2963 if (GAMO.getGlobal()->hasExternalWeakLinkage()) {
2964 UndefWeakSym = createTempSymbol(Name: "undef_weak");
2965 EmitToStreamer(
2966 Inst: MCInstBuilder(AArch64::CBZX)
2967 .addReg(Reg: AuthResultReg)
2968 .addExpr(Val: MCSymbolRefExpr::create(Symbol: UndefWeakSym, Ctx&: OutContext)));
2969 }
2970
2971 assert(GAMO.getGlobal()->getValueType() != nullptr);
2972
2973 bool IsFunctionTy = GAMO.getGlobal()->getValueType()->isFunctionTy();
2974 auto AuthKey = IsFunctionTy ? AArch64PACKey::IA : AArch64PACKey::DA;
2975 emitAUT(Key: AuthKey, Pointer: AuthResultReg, Disc: AArch64::X17);
2976
2977 if (GAMO.getGlobal()->hasExternalWeakLinkage())
2978 OutStreamer->emitLabel(Symbol: UndefWeakSym);
2979
2980 if (!STI->hasFPAC()) {
2981 emitPtrauthCheckAuthenticatedValue(TestedReg: AuthResultReg, ScratchReg: AArch64::X17, Key: AuthKey,
2982 Method: AArch64PAuth::AuthCheckMethod::XPAC);
2983
2984 emitMovXReg(Dest: DstReg, Src: AuthResultReg);
2985 }
2986}
2987
2988const MCExpr *
2989AArch64AsmPrinter::lowerBlockAddressConstant(const BlockAddress &BA) {
2990 const MCExpr *BAE = AsmPrinter::lowerBlockAddressConstant(BA);
2991 const Function &Fn = *BA.getFunction();
2992
2993 if (std::optional<uint16_t> BADisc =
2994 STI->getPtrAuthBlockAddressDiscriminatorIfEnabled(ParentFn: Fn))
2995 return AArch64AuthMCExpr::create(Expr: BAE, Discriminator: *BADisc, Key: AArch64PACKey::IA,
2996 /*HasAddressDiversity=*/false, Ctx&: OutContext);
2997
2998 return BAE;
2999}
3000
3001void AArch64AsmPrinter::emitCBPseudoExpansion(const MachineInstr *MI) {
3002 bool IsImm = false;
3003 unsigned Width = 0;
3004
3005 switch (MI->getOpcode()) {
3006 default:
3007 llvm_unreachable("This is not a CB pseudo instruction");
3008 case AArch64::CBBAssertExt:
3009 IsImm = false;
3010 Width = 8;
3011 break;
3012 case AArch64::CBHAssertExt:
3013 IsImm = false;
3014 Width = 16;
3015 break;
3016 case AArch64::CBWPrr:
3017 Width = 32;
3018 break;
3019 case AArch64::CBXPrr:
3020 Width = 64;
3021 break;
3022 case AArch64::CBWPri:
3023 IsImm = true;
3024 Width = 32;
3025 break;
3026 case AArch64::CBXPri:
3027 IsImm = true;
3028 Width = 64;
3029 break;
3030 }
3031
3032 AArch64CC::CondCode CC =
3033 static_cast<AArch64CC::CondCode>(MI->getOperand(i: 0).getImm());
3034 bool NeedsRegSwap = false;
3035 bool NeedsImmDec = false;
3036 bool NeedsImmInc = false;
3037
3038#define GET_CB_OPC(IsImm, Width, ImmCond, RegCond) \
3039 (IsImm \
3040 ? (Width == 32 ? AArch64::CB##ImmCond##Wri : AArch64::CB##ImmCond##Xri) \
3041 : (Width == 8 \
3042 ? AArch64::CBB##RegCond##Wrr \
3043 : (Width == 16 ? AArch64::CBH##RegCond##Wrr \
3044 : (Width == 32 ? AArch64::CB##RegCond##Wrr \
3045 : AArch64::CB##RegCond##Xrr))))
3046 unsigned MCOpC;
3047
3048 // Decide if we need to either swap register operands or increment/decrement
3049 // immediate operands
3050 switch (CC) {
3051 default:
3052 llvm_unreachable("Invalid CB condition code");
3053 case AArch64CC::EQ:
3054 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ EQ, /* Reg-Reg */ EQ);
3055 break;
3056 case AArch64CC::NE:
3057 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ NE, /* Reg-Reg */ NE);
3058 break;
3059 case AArch64CC::HS:
3060 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ HI, /* Reg-Reg */ HS);
3061 NeedsImmDec = IsImm;
3062 break;
3063 case AArch64CC::LO:
3064 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ LO, /* Reg-Reg */ HI);
3065 NeedsRegSwap = !IsImm;
3066 break;
3067 case AArch64CC::HI:
3068 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ HI, /* Reg-Reg */ HI);
3069 break;
3070 case AArch64CC::LS:
3071 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ LO, /* Reg-Reg */ HS);
3072 NeedsRegSwap = !IsImm;
3073 NeedsImmInc = IsImm;
3074 break;
3075 case AArch64CC::GE:
3076 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ GT, /* Reg-Reg */ GE);
3077 NeedsImmDec = IsImm;
3078 break;
3079 case AArch64CC::LT:
3080 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ LT, /* Reg-Reg */ GT);
3081 NeedsRegSwap = !IsImm;
3082 break;
3083 case AArch64CC::GT:
3084 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ GT, /* Reg-Reg */ GT);
3085 break;
3086 case AArch64CC::LE:
3087 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ LT, /* Reg-Reg */ GE);
3088 NeedsRegSwap = !IsImm;
3089 NeedsImmInc = IsImm;
3090 break;
3091 }
3092#undef GET_CB_OPC
3093
3094 MCInst Inst;
3095 Inst.setOpcode(MCOpC);
3096
3097 MCOperand Lhs, Rhs, Trgt;
3098 lowerOperand(MO: MI->getOperand(i: 1), MCOp&: Lhs);
3099 lowerOperand(MO: MI->getOperand(i: 2), MCOp&: Rhs);
3100 lowerOperand(MO: MI->getOperand(i: 3), MCOp&: Trgt);
3101
3102 // Now swap, increment or decrement
3103 if (NeedsRegSwap) {
3104 assert(Lhs.isReg() && "Expected register operand for CB");
3105 assert(Rhs.isReg() && "Expected register operand for CB");
3106 Inst.addOperand(Op: Rhs);
3107 Inst.addOperand(Op: Lhs);
3108 } else if (NeedsImmDec) {
3109 Rhs.setImm(Rhs.getImm() - 1);
3110 Inst.addOperand(Op: Lhs);
3111 Inst.addOperand(Op: Rhs);
3112 } else if (NeedsImmInc) {
3113 Rhs.setImm(Rhs.getImm() + 1);
3114 Inst.addOperand(Op: Lhs);
3115 Inst.addOperand(Op: Rhs);
3116 } else {
3117 Inst.addOperand(Op: Lhs);
3118 Inst.addOperand(Op: Rhs);
3119 }
3120
3121 assert((!IsImm || (Rhs.getImm() >= 0 && Rhs.getImm() < 64)) &&
3122 "CB immediate operand out-of-bounds");
3123
3124 Inst.addOperand(Op: Trgt);
3125 EmitToStreamer(S&: *OutStreamer, Inst);
3126}
3127
3128// Simple pseudo-instructions have their lowering (with expansion to real
3129// instructions) auto-generated.
3130#include "AArch64GenMCPseudoLowering.inc"
3131
3132void AArch64AsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) {
3133 S.emitInstruction(Inst, STI: *STI);
3134#ifndef NDEBUG
3135 ++InstsEmitted;
3136#endif
3137}
3138
3139void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
3140 AArch64_MC::verifyInstructionPredicates(Opcode: MI->getOpcode(), Features: STI->getFeatureBits());
3141
3142#ifndef NDEBUG
3143 InstsEmitted = 0;
3144 llvm::scope_exit CheckMISize([&]() {
3145 assert(STI->getInstrInfo()->getInstSizeInBytes(*MI) >= InstsEmitted * 4);
3146 });
3147#endif
3148
3149 // Do any auto-generated pseudo lowerings.
3150 if (MCInst OutInst; lowerPseudoInstExpansion(MI, Inst&: OutInst)) {
3151 EmitToStreamer(S&: *OutStreamer, Inst: OutInst);
3152 return;
3153 }
3154
3155 if (MI->getOpcode() == AArch64::ADRP) {
3156 for (auto &Opd : MI->operands()) {
3157 if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) ==
3158 "swift_async_extendedFramePointerFlags") {
3159 ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true;
3160 }
3161 }
3162 }
3163
3164 if (AArch64FI->getLOHRelated().count(Ptr: MI)) {
3165 // Generate a label for LOH related instruction
3166 MCSymbol *LOHLabel = createTempSymbol(Name: "loh");
3167 // Associate the instruction with the label
3168 LOHInstToLabel[MI] = LOHLabel;
3169 OutStreamer->emitLabel(Symbol: LOHLabel);
3170 }
3171
3172 AArch64TargetStreamer *TS =
3173 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
3174 // Do any manual lowerings.
3175 switch (MI->getOpcode()) {
3176 default:
3177 assert(!AArch64InstrInfo::isTailCallReturnInst(*MI) &&
3178 "Unhandled tail call instruction");
3179 break;
3180 case AArch64::HINT: {
3181 // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
3182 // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
3183 // non-empty. If MI is the initial BTI, place the
3184 // __patchable_function_entries label after BTI.
3185 if (CurrentPatchableFunctionEntrySym &&
3186 CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
3187 MI == &MF->front().front()) {
3188 int64_t Imm = MI->getOperand(i: 0).getImm();
3189 if ((Imm & 32) && (Imm & 6)) {
3190 MCInst Inst;
3191 MCInstLowering.Lower(MI, OutMI&: Inst);
3192 EmitToStreamer(S&: *OutStreamer, Inst);
3193 CurrentPatchableFunctionEntrySym = createTempSymbol(Name: "patch");
3194 OutStreamer->emitLabel(Symbol: CurrentPatchableFunctionEntrySym);
3195 return;
3196 }
3197 }
3198 break;
3199 }
3200 case AArch64::MOVMCSym: {
3201 Register DestReg = MI->getOperand(i: 0).getReg();
3202 const MachineOperand &MO_Sym = MI->getOperand(i: 1);
3203 MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
3204 MCOperand Hi_MCSym, Lo_MCSym;
3205
3206 Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
3207 Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
3208
3209 MCInstLowering.lowerOperand(MO: Hi_MOSym, MCOp&: Hi_MCSym);
3210 MCInstLowering.lowerOperand(MO: Lo_MOSym, MCOp&: Lo_MCSym);
3211
3212 MCInst MovZ;
3213 MovZ.setOpcode(AArch64::MOVZXi);
3214 MovZ.addOperand(Op: MCOperand::createReg(Reg: DestReg));
3215 MovZ.addOperand(Op: Hi_MCSym);
3216 MovZ.addOperand(Op: MCOperand::createImm(Val: 16));
3217 EmitToStreamer(S&: *OutStreamer, Inst: MovZ);
3218
3219 MCInst MovK;
3220 MovK.setOpcode(AArch64::MOVKXi);
3221 MovK.addOperand(Op: MCOperand::createReg(Reg: DestReg));
3222 MovK.addOperand(Op: MCOperand::createReg(Reg: DestReg));
3223 MovK.addOperand(Op: Lo_MCSym);
3224 MovK.addOperand(Op: MCOperand::createImm(Val: 0));
3225 EmitToStreamer(S&: *OutStreamer, Inst: MovK);
3226 return;
3227 }
3228 case AArch64::MOVIv2d_ns:
3229 // It is generally beneficial to rewrite "fmov s0, wzr" to "movi d0, #0".
3230 // as movi is more efficient across all cores. Newer cores can eliminate
3231 // fmovs early and there is no difference with movi, but this not true for
3232 // all implementations.
3233 //
3234 // The floating-point version doesn't quite work in rare cases on older
3235 // CPUs, so on those targets we lower this instruction to movi.16b instead.
3236 if (STI->hasZeroCycleZeroingFPWorkaround() &&
3237 MI->getOperand(i: 1).getImm() == 0) {
3238 MCInst TmpInst;
3239 TmpInst.setOpcode(AArch64::MOVIv16b_ns);
3240 TmpInst.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg()));
3241 TmpInst.addOperand(Op: MCOperand::createImm(Val: 0));
3242 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
3243 return;
3244 }
3245 break;
3246
3247 case AArch64::DBG_VALUE:
3248 case AArch64::DBG_VALUE_LIST:
3249 if (isVerbose() && OutStreamer->hasRawTextSupport()) {
3250 SmallString<128> TmpStr;
3251 raw_svector_ostream OS(TmpStr);
3252 PrintDebugValueComment(MI, OS);
3253 OutStreamer->emitRawText(String: StringRef(OS.str()));
3254 }
3255 return;
3256
3257 case AArch64::EMITBKEY: {
3258 ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
3259 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
3260 ExceptionHandlingType != ExceptionHandling::ARM)
3261 return;
3262
3263 if (getFunctionCFISectionType(MF: *MF) == CFISection::None)
3264 return;
3265
3266 OutStreamer->emitCFIBKeyFrame();
3267 return;
3268 }
3269
3270 case AArch64::EMITMTETAGGED: {
3271 ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
3272 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
3273 ExceptionHandlingType != ExceptionHandling::ARM)
3274 return;
3275
3276 if (getFunctionCFISectionType(MF: *MF) != CFISection::None)
3277 OutStreamer->emitCFIMTETaggedFrame();
3278 return;
3279 }
3280
3281 case AArch64::AUTx16x17: {
3282 const Register Pointer = AArch64::X16;
3283 const Register Scratch = AArch64::X17;
3284
3285 PtrAuthSchema AuthSchema((AArch64PACKey::ID)MI->getOperand(i: 0).getImm(),
3286 MI->getOperand(i: 1).getImm(), MI->getOperand(i: 2));
3287
3288 emitPtrauthAuthResign(Pointer, Scratch, AuthSchema, SignSchema: std::nullopt,
3289 OptAddend: std::nullopt, DS: MI->getDeactivationSymbol());
3290 return;
3291 }
3292
3293 case AArch64::AUTxMxN: {
3294 const Register Pointer = MI->getOperand(i: 0).getReg();
3295 const Register Scratch = MI->getOperand(i: 1).getReg();
3296
3297 PtrAuthSchema AuthSchema((AArch64PACKey::ID)MI->getOperand(i: 3).getImm(),
3298 MI->getOperand(i: 4).getImm(), MI->getOperand(i: 5));
3299
3300 emitPtrauthAuthResign(Pointer, Scratch, AuthSchema, SignSchema: std::nullopt,
3301 OptAddend: std::nullopt, DS: MI->getDeactivationSymbol());
3302 return;
3303 }
3304
3305 case AArch64::AUTPAC: {
3306 const Register Pointer = AArch64::X16;
3307 const Register Scratch = AArch64::X17;
3308
3309 PtrAuthSchema AuthSchema((AArch64PACKey::ID)MI->getOperand(i: 0).getImm(),
3310 MI->getOperand(i: 1).getImm(), MI->getOperand(i: 2));
3311
3312 PtrAuthSchema SignSchema((AArch64PACKey::ID)MI->getOperand(i: 3).getImm(),
3313 MI->getOperand(i: 4).getImm(), MI->getOperand(i: 5));
3314
3315 emitPtrauthAuthResign(Pointer, Scratch, AuthSchema, SignSchema,
3316 OptAddend: std::nullopt, DS: MI->getDeactivationSymbol());
3317 return;
3318 }
3319
3320 case AArch64::AUTRELLOADPAC: {
3321 const Register Pointer = AArch64::X16;
3322 const Register Scratch = AArch64::X17;
3323
3324 PtrAuthSchema AuthSchema((AArch64PACKey::ID)MI->getOperand(i: 0).getImm(),
3325 MI->getOperand(i: 1).getImm(), MI->getOperand(i: 2));
3326
3327 PtrAuthSchema SignSchema((AArch64PACKey::ID)MI->getOperand(i: 3).getImm(),
3328 MI->getOperand(i: 4).getImm(), MI->getOperand(i: 5));
3329
3330 emitPtrauthAuthResign(Pointer, Scratch, AuthSchema, SignSchema,
3331 OptAddend: MI->getOperand(i: 6).getImm(),
3332 DS: MI->getDeactivationSymbol());
3333
3334 return;
3335 }
3336
3337 case AArch64::PAC:
3338 emitPtrauthSign(MI);
3339 return;
3340
3341 case AArch64::LOADauthptrstatic:
3342 LowerLOADauthptrstatic(MI: *MI);
3343 return;
3344
3345 case AArch64::LOADgotPAC:
3346 case AArch64::MOVaddrPAC:
3347 LowerMOVaddrPAC(MI: *MI);
3348 return;
3349
3350 case AArch64::LOADgotAUTH:
3351 LowerLOADgotAUTH(MI: *MI);
3352 return;
3353
3354 case AArch64::BRA:
3355 case AArch64::BLRA:
3356 emitPtrauthBranch(MI);
3357 return;
3358
3359 // Tail calls use pseudo instructions so they have the proper code-gen
3360 // attributes (isCall, isReturn, etc.). We lower them to the real
3361 // instruction here.
3362 case AArch64::AUTH_TCRETURN:
3363 case AArch64::AUTH_TCRETURN_BTI: {
3364 Register Callee = MI->getOperand(i: 0).getReg();
3365 const auto Key = (AArch64PACKey::ID)MI->getOperand(i: 2).getImm();
3366 const uint64_t Disc = MI->getOperand(i: 3).getImm();
3367
3368 Register AddrDisc = MI->getOperand(i: 4).getReg();
3369
3370 Register ScratchReg = Callee == AArch64::X16 ? AArch64::X17 : AArch64::X16;
3371
3372 emitPtrauthTailCallHardening(TC: MI);
3373
3374 // See the comments in emitPtrauthBranch.
3375 if (Callee == AddrDisc)
3376 report_fatal_error(reason: "Call target is signed with its own value");
3377
3378 // After isX16X17Safer predicate was introduced, emitPtrauthDiscriminator is
3379 // no longer restricted to only reusing AddrDisc when it is X16 or X17
3380 // (which are implicit-def'ed by AUTH_TCRETURN pseudos), thus impose this
3381 // restriction manually not to clobber an unexpected register.
3382 bool AddrDiscIsImplicitDef =
3383 AddrDisc == AArch64::X16 || AddrDisc == AArch64::X17;
3384 Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, ScratchReg,
3385 MayClobberAddrDisc: AddrDiscIsImplicitDef);
3386 emitBLRA(/*IsCall*/ false, Key, Target: Callee, Disc: DiscReg);
3387 return;
3388 }
3389
3390 case AArch64::TCRETURNri:
3391 case AArch64::TCRETURNrix16x17:
3392 case AArch64::TCRETURNrix17:
3393 case AArch64::TCRETURNrinotx16:
3394 case AArch64::TCRETURNriALL: {
3395 emitPtrauthTailCallHardening(TC: MI);
3396
3397 recordIfImportCall(BranchInst: MI);
3398 MCInst TmpInst;
3399 TmpInst.setOpcode(AArch64::BR);
3400 TmpInst.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg()));
3401 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
3402 return;
3403 }
3404 case AArch64::TCRETURNdi: {
3405 emitPtrauthTailCallHardening(TC: MI);
3406
3407 MCOperand Dest;
3408 MCInstLowering.lowerOperand(MO: MI->getOperand(i: 0), MCOp&: Dest);
3409 recordIfImportCall(BranchInst: MI);
3410 MCInst TmpInst;
3411 TmpInst.setOpcode(AArch64::B);
3412 TmpInst.addOperand(Op: Dest);
3413 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
3414 return;
3415 }
3416 case AArch64::SpeculationBarrierISBDSBEndBB: {
3417 // Print DSB SYS + ISB
3418 MCInst TmpInstDSB;
3419 TmpInstDSB.setOpcode(AArch64::DSB);
3420 TmpInstDSB.addOperand(Op: MCOperand::createImm(Val: 0xf));
3421 EmitToStreamer(S&: *OutStreamer, Inst: TmpInstDSB);
3422 MCInst TmpInstISB;
3423 TmpInstISB.setOpcode(AArch64::ISB);
3424 TmpInstISB.addOperand(Op: MCOperand::createImm(Val: 0xf));
3425 EmitToStreamer(S&: *OutStreamer, Inst: TmpInstISB);
3426 return;
3427 }
3428 case AArch64::SpeculationBarrierSBEndBB: {
3429 // Print SB
3430 MCInst TmpInstSB;
3431 TmpInstSB.setOpcode(AArch64::SB);
3432 EmitToStreamer(S&: *OutStreamer, Inst: TmpInstSB);
3433 return;
3434 }
3435 case AArch64::TLSDESC_AUTH_CALLSEQ: {
3436 /// lower this to:
3437 /// adrp x0, :tlsdesc_auth:var
3438 /// ldr x16, [x0, #:tlsdesc_auth_lo12:var]
3439 /// add x0, x0, #:tlsdesc_auth_lo12:var
3440 /// blraa x16, x0
3441 /// (TPIDR_EL0 offset now in x0)
3442 const MachineOperand &MO_Sym = MI->getOperand(i: 0);
3443 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
3444 MCOperand SymTLSDescLo12, SymTLSDesc;
3445 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
3446 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
3447 MCInstLowering.lowerOperand(MO: MO_TLSDESC_LO12, MCOp&: SymTLSDescLo12);
3448 MCInstLowering.lowerOperand(MO: MO_TLSDESC, MCOp&: SymTLSDesc);
3449
3450 MCInst Adrp;
3451 Adrp.setOpcode(AArch64::ADRP);
3452 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3453 Adrp.addOperand(Op: SymTLSDesc);
3454 EmitToStreamer(S&: *OutStreamer, Inst: Adrp);
3455
3456 MCInst Ldr;
3457 Ldr.setOpcode(AArch64::LDRXui);
3458 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
3459 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3460 Ldr.addOperand(Op: SymTLSDescLo12);
3461 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
3462 EmitToStreamer(S&: *OutStreamer, Inst: Ldr);
3463
3464 MCInst Add;
3465 Add.setOpcode(AArch64::ADDXri);
3466 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3467 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3468 Add.addOperand(Op: SymTLSDescLo12);
3469 Add.addOperand(Op: MCOperand::createImm(Val: AArch64_AM::getShiftValue(Imm: 0)));
3470 EmitToStreamer(S&: *OutStreamer, Inst: Add);
3471
3472 // Authenticated TLSDESC accesses are not relaxed.
3473 // Thus, do not emit .tlsdesccall for AUTH TLSDESC.
3474
3475 MCInst Blraa;
3476 Blraa.setOpcode(AArch64::BLRAA);
3477 Blraa.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
3478 Blraa.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3479 EmitToStreamer(S&: *OutStreamer, Inst: Blraa);
3480
3481 return;
3482 }
3483 case AArch64::TLSDESC_CALLSEQ: {
3484 /// lower this to:
3485 /// adrp x0, :tlsdesc:var
3486 /// ldr x1, [x0, #:tlsdesc_lo12:var]
3487 /// add x0, x0, #:tlsdesc_lo12:var
3488 /// .tlsdesccall var
3489 /// blr x1
3490 /// (TPIDR_EL0 offset now in x0)
3491 const MachineOperand &MO_Sym = MI->getOperand(i: 0);
3492 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
3493 MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
3494 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
3495 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
3496 MCInstLowering.lowerOperand(MO: MO_Sym, MCOp&: Sym);
3497 MCInstLowering.lowerOperand(MO: MO_TLSDESC_LO12, MCOp&: SymTLSDescLo12);
3498 MCInstLowering.lowerOperand(MO: MO_TLSDESC, MCOp&: SymTLSDesc);
3499
3500 MCInst Adrp;
3501 Adrp.setOpcode(AArch64::ADRP);
3502 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3503 Adrp.addOperand(Op: SymTLSDesc);
3504 EmitToStreamer(S&: *OutStreamer, Inst: Adrp);
3505
3506 MCInst Ldr;
3507 if (STI->isTargetILP32()) {
3508 Ldr.setOpcode(AArch64::LDRWui);
3509 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::W1));
3510 } else {
3511 Ldr.setOpcode(AArch64::LDRXui);
3512 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X1));
3513 }
3514 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3515 Ldr.addOperand(Op: SymTLSDescLo12);
3516 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
3517 EmitToStreamer(S&: *OutStreamer, Inst: Ldr);
3518
3519 MCInst Add;
3520 if (STI->isTargetILP32()) {
3521 Add.setOpcode(AArch64::ADDWri);
3522 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::W0));
3523 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::W0));
3524 } else {
3525 Add.setOpcode(AArch64::ADDXri);
3526 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3527 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3528 }
3529 Add.addOperand(Op: SymTLSDescLo12);
3530 Add.addOperand(Op: MCOperand::createImm(Val: AArch64_AM::getShiftValue(Imm: 0)));
3531 EmitToStreamer(S&: *OutStreamer, Inst: Add);
3532
3533 // Emit a relocation-annotation. This expands to no code, but requests
3534 // the following instruction gets an R_AARCH64_TLSDESC_CALL.
3535 MCInst TLSDescCall;
3536 TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
3537 TLSDescCall.addOperand(Op: Sym);
3538 EmitToStreamer(S&: *OutStreamer, Inst: TLSDescCall);
3539#ifndef NDEBUG
3540 --InstsEmitted; // no code emitted
3541#endif
3542
3543 MCInst Blr;
3544 Blr.setOpcode(AArch64::BLR);
3545 Blr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X1));
3546 EmitToStreamer(S&: *OutStreamer, Inst: Blr);
3547
3548 return;
3549 }
3550
3551 case AArch64::JumpTableDest32:
3552 case AArch64::JumpTableDest16:
3553 case AArch64::JumpTableDest8:
3554 LowerJumpTableDest(OutStreamer&: *OutStreamer, MI: *MI);
3555 return;
3556
3557 case AArch64::BR_JumpTable:
3558 LowerHardenedBRJumpTable(MI: *MI);
3559 return;
3560
3561 case AArch64::FMOVH0:
3562 case AArch64::FMOVS0:
3563 case AArch64::FMOVD0:
3564 emitFMov0(MI: *MI);
3565 return;
3566
3567 case AArch64::MOPSMemoryCopyPseudo:
3568 case AArch64::MOPSMemoryMovePseudo:
3569 case AArch64::MOPSMemorySetPseudo:
3570 case AArch64::MOPSMemorySetTaggingPseudo:
3571 LowerMOPS(OutStreamer&: *OutStreamer, MI: *MI);
3572 return;
3573
3574 case TargetOpcode::STACKMAP:
3575 return LowerSTACKMAP(OutStreamer&: *OutStreamer, SM, MI: *MI);
3576
3577 case TargetOpcode::PATCHPOINT:
3578 return LowerPATCHPOINT(OutStreamer&: *OutStreamer, SM, MI: *MI);
3579
3580 case TargetOpcode::STATEPOINT:
3581 return LowerSTATEPOINT(OutStreamer&: *OutStreamer, SM, MI: *MI);
3582
3583 case TargetOpcode::FAULTING_OP:
3584 return LowerFAULTING_OP(FaultingMI: *MI);
3585
3586 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
3587 LowerPATCHABLE_FUNCTION_ENTER(MI: *MI);
3588 return;
3589
3590 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
3591 LowerPATCHABLE_FUNCTION_EXIT(MI: *MI);
3592 return;
3593
3594 case TargetOpcode::PATCHABLE_TAIL_CALL:
3595 LowerPATCHABLE_TAIL_CALL(MI: *MI);
3596 return;
3597 case TargetOpcode::PATCHABLE_EVENT_CALL:
3598 return LowerPATCHABLE_EVENT_CALL(MI: *MI, Typed: false);
3599 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
3600 return LowerPATCHABLE_EVENT_CALL(MI: *MI, Typed: true);
3601
3602 case AArch64::KCFI_CHECK:
3603 LowerKCFI_CHECK(MI: *MI);
3604 return;
3605
3606 case AArch64::HWASAN_CHECK_MEMACCESS:
3607 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
3608 case AArch64::HWASAN_CHECK_MEMACCESS_FIXEDSHADOW:
3609 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW:
3610 LowerHWASAN_CHECK_MEMACCESS(MI: *MI);
3611 return;
3612
3613 case AArch64::SEH_StackAlloc:
3614 TS->emitARM64WinCFIAllocStack(Size: MI->getOperand(i: 0).getImm());
3615 return;
3616
3617 case AArch64::SEH_SaveFPLR:
3618 TS->emitARM64WinCFISaveFPLR(Offset: MI->getOperand(i: 0).getImm());
3619 return;
3620
3621 case AArch64::SEH_SaveFPLR_X:
3622 assert(MI->getOperand(0).getImm() < 0 &&
3623 "Pre increment SEH opcode must have a negative offset");
3624 TS->emitARM64WinCFISaveFPLRX(Offset: -MI->getOperand(i: 0).getImm());
3625 return;
3626
3627 case AArch64::SEH_SaveReg:
3628 TS->emitARM64WinCFISaveReg(Reg: MI->getOperand(i: 0).getImm(),
3629 Offset: MI->getOperand(i: 1).getImm());
3630 return;
3631
3632 case AArch64::SEH_SaveReg_X:
3633 assert(MI->getOperand(1).getImm() < 0 &&
3634 "Pre increment SEH opcode must have a negative offset");
3635 TS->emitARM64WinCFISaveRegX(Reg: MI->getOperand(i: 0).getImm(),
3636 Offset: -MI->getOperand(i: 1).getImm());
3637 return;
3638
3639 case AArch64::SEH_SaveRegP:
3640 if (MI->getOperand(i: 1).getImm() == 30 && MI->getOperand(i: 0).getImm() >= 19 &&
3641 MI->getOperand(i: 0).getImm() <= 28) {
3642 assert((MI->getOperand(0).getImm() - 19) % 2 == 0 &&
3643 "Register paired with LR must be odd");
3644 TS->emitARM64WinCFISaveLRPair(Reg: MI->getOperand(i: 0).getImm(),
3645 Offset: MI->getOperand(i: 2).getImm());
3646 return;
3647 }
3648 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
3649 "Non-consecutive registers not allowed for save_regp");
3650 TS->emitARM64WinCFISaveRegP(Reg: MI->getOperand(i: 0).getImm(),
3651 Offset: MI->getOperand(i: 2).getImm());
3652 return;
3653
3654 case AArch64::SEH_SaveRegP_X:
3655 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
3656 "Non-consecutive registers not allowed for save_regp_x");
3657 assert(MI->getOperand(2).getImm() < 0 &&
3658 "Pre increment SEH opcode must have a negative offset");
3659 TS->emitARM64WinCFISaveRegPX(Reg: MI->getOperand(i: 0).getImm(),
3660 Offset: -MI->getOperand(i: 2).getImm());
3661 return;
3662
3663 case AArch64::SEH_SaveFReg:
3664 TS->emitARM64WinCFISaveFReg(Reg: MI->getOperand(i: 0).getImm(),
3665 Offset: MI->getOperand(i: 1).getImm());
3666 return;
3667
3668 case AArch64::SEH_SaveFReg_X:
3669 assert(MI->getOperand(1).getImm() < 0 &&
3670 "Pre increment SEH opcode must have a negative offset");
3671 TS->emitARM64WinCFISaveFRegX(Reg: MI->getOperand(i: 0).getImm(),
3672 Offset: -MI->getOperand(i: 1).getImm());
3673 return;
3674
3675 case AArch64::SEH_SaveFRegP:
3676 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
3677 "Non-consecutive registers not allowed for save_regp");
3678 TS->emitARM64WinCFISaveFRegP(Reg: MI->getOperand(i: 0).getImm(),
3679 Offset: MI->getOperand(i: 2).getImm());
3680 return;
3681
3682 case AArch64::SEH_SaveFRegP_X:
3683 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
3684 "Non-consecutive registers not allowed for save_regp_x");
3685 assert(MI->getOperand(2).getImm() < 0 &&
3686 "Pre increment SEH opcode must have a negative offset");
3687 TS->emitARM64WinCFISaveFRegPX(Reg: MI->getOperand(i: 0).getImm(),
3688 Offset: -MI->getOperand(i: 2).getImm());
3689 return;
3690
3691 case AArch64::SEH_SetFP:
3692 TS->emitARM64WinCFISetFP();
3693 return;
3694
3695 case AArch64::SEH_AddFP:
3696 TS->emitARM64WinCFIAddFP(Size: MI->getOperand(i: 0).getImm());
3697 return;
3698
3699 case AArch64::SEH_Nop:
3700 TS->emitARM64WinCFINop();
3701 return;
3702
3703 case AArch64::SEH_PrologEnd:
3704 TS->emitARM64WinCFIPrologEnd();
3705 return;
3706
3707 case AArch64::SEH_EpilogStart:
3708 TS->emitARM64WinCFIEpilogStart();
3709 return;
3710
3711 case AArch64::SEH_EpilogEnd:
3712 TS->emitARM64WinCFIEpilogEnd();
3713 return;
3714
3715 case AArch64::SEH_PACSignLR:
3716 TS->emitARM64WinCFIPACSignLR();
3717 return;
3718
3719 case AArch64::SEH_SaveAnyRegI:
3720 assert(MI->getOperand(1).getImm() <= 1008 &&
3721 "SaveAnyRegQP SEH opcode offset must fit into 6 bits");
3722 TS->emitARM64WinCFISaveAnyRegI(Reg: MI->getOperand(i: 0).getImm(),
3723 Offset: MI->getOperand(i: 1).getImm());
3724 return;
3725
3726 case AArch64::SEH_SaveAnyRegIP:
3727 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
3728 "Non-consecutive registers not allowed for save_any_reg");
3729 assert(MI->getOperand(2).getImm() <= 1008 &&
3730 "SaveAnyRegQP SEH opcode offset must fit into 6 bits");
3731 TS->emitARM64WinCFISaveAnyRegIP(Reg: MI->getOperand(i: 0).getImm(),
3732 Offset: MI->getOperand(i: 2).getImm());
3733 return;
3734
3735 case AArch64::SEH_SaveAnyRegQP:
3736 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
3737 "Non-consecutive registers not allowed for save_any_reg");
3738 assert(MI->getOperand(2).getImm() >= 0 &&
3739 "SaveAnyRegQP SEH opcode offset must be non-negative");
3740 assert(MI->getOperand(2).getImm() <= 1008 &&
3741 "SaveAnyRegQP SEH opcode offset must fit into 6 bits");
3742 TS->emitARM64WinCFISaveAnyRegQP(Reg: MI->getOperand(i: 0).getImm(),
3743 Offset: MI->getOperand(i: 2).getImm());
3744 return;
3745
3746 case AArch64::SEH_SaveAnyRegQPX:
3747 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
3748 "Non-consecutive registers not allowed for save_any_reg");
3749 assert(MI->getOperand(2).getImm() < 0 &&
3750 "SaveAnyRegQPX SEH opcode offset must be negative");
3751 assert(MI->getOperand(2).getImm() >= -1008 &&
3752 "SaveAnyRegQPX SEH opcode offset must fit into 6 bits");
3753 TS->emitARM64WinCFISaveAnyRegQPX(Reg: MI->getOperand(i: 0).getImm(),
3754 Offset: -MI->getOperand(i: 2).getImm());
3755 return;
3756
3757 case AArch64::SEH_AllocZ:
3758 assert(MI->getOperand(0).getImm() >= 0 &&
3759 "AllocZ SEH opcode offset must be non-negative");
3760 assert(MI->getOperand(0).getImm() <= 255 &&
3761 "AllocZ SEH opcode offset must fit into 8 bits");
3762 TS->emitARM64WinCFIAllocZ(Offset: MI->getOperand(i: 0).getImm());
3763 return;
3764
3765 case AArch64::SEH_SaveZReg:
3766 assert(MI->getOperand(1).getImm() >= 0 &&
3767 "SaveZReg SEH opcode offset must be non-negative");
3768 assert(MI->getOperand(1).getImm() <= 255 &&
3769 "SaveZReg SEH opcode offset must fit into 8 bits");
3770 TS->emitARM64WinCFISaveZReg(Reg: MI->getOperand(i: 0).getImm(),
3771 Offset: MI->getOperand(i: 1).getImm());
3772 return;
3773
3774 case AArch64::SEH_SavePReg:
3775 assert(MI->getOperand(1).getImm() >= 0 &&
3776 "SavePReg SEH opcode offset must be non-negative");
3777 assert(MI->getOperand(1).getImm() <= 255 &&
3778 "SavePReg SEH opcode offset must fit into 8 bits");
3779 TS->emitARM64WinCFISavePReg(Reg: MI->getOperand(i: 0).getImm(),
3780 Offset: MI->getOperand(i: 1).getImm());
3781 return;
3782
3783 case AArch64::BLR:
3784 case AArch64::BR: {
3785 recordIfImportCall(BranchInst: MI);
3786 MCInst TmpInst;
3787 MCInstLowering.Lower(MI, OutMI&: TmpInst);
3788 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
3789 return;
3790 }
3791 case AArch64::CBWPri:
3792 case AArch64::CBXPri:
3793 case AArch64::CBBAssertExt:
3794 case AArch64::CBHAssertExt:
3795 case AArch64::CBWPrr:
3796 case AArch64::CBXPrr:
3797 emitCBPseudoExpansion(MI);
3798 return;
3799 }
3800
3801 if (emitDeactivationSymbolRelocation(DS: MI->getDeactivationSymbol()))
3802 return;
3803
3804 // Finally, do the automated lowerings for everything else.
3805 MCInst TmpInst;
3806 MCInstLowering.Lower(MI, OutMI&: TmpInst);
3807 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
3808}
3809
3810void AArch64AsmPrinter::recordIfImportCall(
3811 const llvm::MachineInstr *BranchInst) {
3812 if (!EnableImportCallOptimization)
3813 return;
3814
3815 auto [GV, OpFlags] = BranchInst->getMF()->tryGetCalledGlobal(MI: BranchInst);
3816 if (GV && GV->hasDLLImportStorageClass()) {
3817 auto *CallSiteSymbol = MMI->getContext().createNamedTempSymbol(Name: "impcall");
3818 OutStreamer->emitLabel(Symbol: CallSiteSymbol);
3819
3820 auto *CalledSymbol = MCInstLowering.GetGlobalValueSymbol(GV, TargetFlags: OpFlags);
3821 SectionToImportedFunctionCalls[OutStreamer->getCurrentSectionOnly()]
3822 .push_back(x: {CallSiteSymbol, CalledSymbol});
3823 }
3824}
3825
3826void AArch64AsmPrinter::emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI,
3827 MCSymbol *LazyPointer) {
3828 // _ifunc:
3829 // adrp x16, lazy_pointer@GOTPAGE
3830 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF]
3831 // ldr x16, [x16]
3832 // br x16
3833
3834 {
3835 MCInst Adrp;
3836 Adrp.setOpcode(AArch64::ADRP);
3837 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
3838 MCOperand SymPage;
3839 MCInstLowering.lowerOperand(
3840 MO: MachineOperand::CreateMCSymbol(Sym: LazyPointer,
3841 TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGE),
3842 MCOp&: SymPage);
3843 Adrp.addOperand(Op: SymPage);
3844 EmitToStreamer(Inst: Adrp);
3845 }
3846
3847 {
3848 MCInst Ldr;
3849 Ldr.setOpcode(AArch64::LDRXui);
3850 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
3851 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
3852 MCOperand SymPageOff;
3853 MCInstLowering.lowerOperand(
3854 MO: MachineOperand::CreateMCSymbol(Sym: LazyPointer, TargetFlags: AArch64II::MO_GOT |
3855 AArch64II::MO_PAGEOFF),
3856 MCOp&: SymPageOff);
3857 Ldr.addOperand(Op: SymPageOff);
3858 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
3859 EmitToStreamer(Inst: Ldr);
3860 }
3861
3862 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
3863 .addReg(Reg: AArch64::X16)
3864 .addReg(Reg: AArch64::X16)
3865 .addImm(Val: 0));
3866
3867 EmitToStreamer(Inst: MCInstBuilder(TM.getTargetTriple().isArm64e() ? AArch64::BRAAZ
3868 : AArch64::BR)
3869 .addReg(Reg: AArch64::X16));
3870}
3871
3872void AArch64AsmPrinter::emitMachOIFuncStubHelperBody(Module &M,
3873 const GlobalIFunc &GI,
3874 MCSymbol *LazyPointer) {
3875 // These stub helpers are only ever called once, so here we're optimizing for
3876 // minimum size by using the pre-indexed store variants, which saves a few
3877 // bytes of instructions to bump & restore sp.
3878
3879 // _ifunc.stub_helper:
3880 // stp fp, lr, [sp, #-16]!
3881 // mov fp, sp
3882 // stp x1, x0, [sp, #-16]!
3883 // stp x3, x2, [sp, #-16]!
3884 // stp x5, x4, [sp, #-16]!
3885 // stp x7, x6, [sp, #-16]!
3886 // stp d1, d0, [sp, #-16]!
3887 // stp d3, d2, [sp, #-16]!
3888 // stp d5, d4, [sp, #-16]!
3889 // stp d7, d6, [sp, #-16]!
3890 // bl _resolver
3891 // adrp x16, lazy_pointer@GOTPAGE
3892 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF]
3893 // str x0, [x16]
3894 // mov x16, x0
3895 // ldp d7, d6, [sp], #16
3896 // ldp d5, d4, [sp], #16
3897 // ldp d3, d2, [sp], #16
3898 // ldp d1, d0, [sp], #16
3899 // ldp x7, x6, [sp], #16
3900 // ldp x5, x4, [sp], #16
3901 // ldp x3, x2, [sp], #16
3902 // ldp x1, x0, [sp], #16
3903 // ldp fp, lr, [sp], #16
3904 // br x16
3905
3906 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPXpre)
3907 .addReg(Reg: AArch64::SP)
3908 .addReg(Reg: AArch64::FP)
3909 .addReg(Reg: AArch64::LR)
3910 .addReg(Reg: AArch64::SP)
3911 .addImm(Val: -2));
3912
3913 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
3914 .addReg(Reg: AArch64::FP)
3915 .addReg(Reg: AArch64::SP)
3916 .addImm(Val: 0)
3917 .addImm(Val: 0));
3918
3919 for (int I = 0; I != 4; ++I)
3920 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPXpre)
3921 .addReg(Reg: AArch64::SP)
3922 .addReg(Reg: AArch64::X1 + 2 * I)
3923 .addReg(Reg: AArch64::X0 + 2 * I)
3924 .addReg(Reg: AArch64::SP)
3925 .addImm(Val: -2));
3926
3927 for (int I = 0; I != 4; ++I)
3928 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPDpre)
3929 .addReg(Reg: AArch64::SP)
3930 .addReg(Reg: AArch64::D1 + 2 * I)
3931 .addReg(Reg: AArch64::D0 + 2 * I)
3932 .addReg(Reg: AArch64::SP)
3933 .addImm(Val: -2));
3934
3935 EmitToStreamer(
3936 Inst: MCInstBuilder(AArch64::BL)
3937 .addOperand(Op: MCOperand::createExpr(Val: lowerConstant(CV: GI.getResolver()))));
3938
3939 {
3940 MCInst Adrp;
3941 Adrp.setOpcode(AArch64::ADRP);
3942 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
3943 MCOperand SymPage;
3944 MCInstLowering.lowerOperand(
3945 MO: MachineOperand::CreateES(SymName: LazyPointer->getName().data() + 1,
3946 TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGE),
3947 MCOp&: SymPage);
3948 Adrp.addOperand(Op: SymPage);
3949 EmitToStreamer(Inst: Adrp);
3950 }
3951
3952 {
3953 MCInst Ldr;
3954 Ldr.setOpcode(AArch64::LDRXui);
3955 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
3956 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
3957 MCOperand SymPageOff;
3958 MCInstLowering.lowerOperand(
3959 MO: MachineOperand::CreateES(SymName: LazyPointer->getName().data() + 1,
3960 TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGEOFF),
3961 MCOp&: SymPageOff);
3962 Ldr.addOperand(Op: SymPageOff);
3963 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
3964 EmitToStreamer(Inst: Ldr);
3965 }
3966
3967 EmitToStreamer(Inst: MCInstBuilder(AArch64::STRXui)
3968 .addReg(Reg: AArch64::X0)
3969 .addReg(Reg: AArch64::X16)
3970 .addImm(Val: 0));
3971
3972 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
3973 .addReg(Reg: AArch64::X16)
3974 .addReg(Reg: AArch64::X0)
3975 .addImm(Val: 0)
3976 .addImm(Val: 0));
3977
3978 for (int I = 3; I != -1; --I)
3979 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDPDpost)
3980 .addReg(Reg: AArch64::SP)
3981 .addReg(Reg: AArch64::D1 + 2 * I)
3982 .addReg(Reg: AArch64::D0 + 2 * I)
3983 .addReg(Reg: AArch64::SP)
3984 .addImm(Val: 2));
3985
3986 for (int I = 3; I != -1; --I)
3987 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDPXpost)
3988 .addReg(Reg: AArch64::SP)
3989 .addReg(Reg: AArch64::X1 + 2 * I)
3990 .addReg(Reg: AArch64::X0 + 2 * I)
3991 .addReg(Reg: AArch64::SP)
3992 .addImm(Val: 2));
3993
3994 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDPXpost)
3995 .addReg(Reg: AArch64::SP)
3996 .addReg(Reg: AArch64::FP)
3997 .addReg(Reg: AArch64::LR)
3998 .addReg(Reg: AArch64::SP)
3999 .addImm(Val: 2));
4000
4001 EmitToStreamer(Inst: MCInstBuilder(TM.getTargetTriple().isArm64e() ? AArch64::BRAAZ
4002 : AArch64::BR)
4003 .addReg(Reg: AArch64::X16));
4004}
4005
4006const MCExpr *AArch64AsmPrinter::lowerConstant(const Constant *CV,
4007 const Constant *BaseCV,
4008 uint64_t Offset) {
4009 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: CV)) {
4010 return MCSymbolRefExpr::create(Symbol: MCInstLowering.GetGlobalValueSymbol(GV, TargetFlags: 0),
4011 Ctx&: OutContext);
4012 }
4013
4014 return AsmPrinter::lowerConstant(CV, BaseCV, Offset);
4015}
4016
4017char AArch64AsmPrinter::ID = 0;
4018
4019INITIALIZE_PASS(AArch64AsmPrinter, "aarch64-asm-printer",
4020 "AArch64 Assembly Printer", false, false)
4021
4022// Force static initialization.
4023extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
4024LLVMInitializeAArch64AsmPrinter() {
4025 RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
4026 RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
4027 RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
4028 RegisterAsmPrinter<AArch64AsmPrinter> W(getTheARM64_32Target());
4029 RegisterAsmPrinter<AArch64AsmPrinter> V(getTheAArch64_32Target());
4030}
4031