1//===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a printer that converts from our internal representation
10// of machine-dependent LLVM code to the AArch64 assembly language.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AArch64.h"
15#include "AArch64MCInstLower.h"
16#include "AArch64MachineFunctionInfo.h"
17#include "AArch64RegisterInfo.h"
18#include "AArch64Subtarget.h"
19#include "AArch64TargetObjectFile.h"
20#include "MCTargetDesc/AArch64AddressingModes.h"
21#include "MCTargetDesc/AArch64InstPrinter.h"
22#include "MCTargetDesc/AArch64MCAsmInfo.h"
23#include "MCTargetDesc/AArch64MCTargetDesc.h"
24#include "MCTargetDesc/AArch64TargetStreamer.h"
25#include "TargetInfo/AArch64TargetInfo.h"
26#include "Utils/AArch64BaseInfo.h"
27#include "llvm/ADT/DenseMap.h"
28#include "llvm/ADT/ScopeExit.h"
29#include "llvm/ADT/SmallString.h"
30#include "llvm/ADT/SmallVector.h"
31#include "llvm/ADT/StringRef.h"
32#include "llvm/ADT/Twine.h"
33#include "llvm/BinaryFormat/COFF.h"
34#include "llvm/BinaryFormat/ELF.h"
35#include "llvm/BinaryFormat/MachO.h"
36#include "llvm/CodeGen/AsmPrinter.h"
37#include "llvm/CodeGen/FaultMaps.h"
38#include "llvm/CodeGen/MachineBasicBlock.h"
39#include "llvm/CodeGen/MachineFunction.h"
40#include "llvm/CodeGen/MachineInstr.h"
41#include "llvm/CodeGen/MachineJumpTableInfo.h"
42#include "llvm/CodeGen/MachineModuleInfoImpls.h"
43#include "llvm/CodeGen/MachineOperand.h"
44#include "llvm/CodeGen/StackMaps.h"
45#include "llvm/CodeGen/TargetRegisterInfo.h"
46#include "llvm/IR/DataLayout.h"
47#include "llvm/IR/DebugInfoMetadata.h"
48#include "llvm/IR/Mangler.h"
49#include "llvm/IR/Module.h"
50#include "llvm/MC/MCAsmInfo.h"
51#include "llvm/MC/MCContext.h"
52#include "llvm/MC/MCInst.h"
53#include "llvm/MC/MCInstBuilder.h"
54#include "llvm/MC/MCSectionELF.h"
55#include "llvm/MC/MCSectionMachO.h"
56#include "llvm/MC/MCStreamer.h"
57#include "llvm/MC/MCSymbol.h"
58#include "llvm/MC/TargetRegistry.h"
59#include "llvm/Support/Casting.h"
60#include "llvm/Support/CommandLine.h"
61#include "llvm/Support/Compiler.h"
62#include "llvm/Support/ErrorHandling.h"
63#include "llvm/Support/raw_ostream.h"
64#include "llvm/Target/TargetMachine.h"
65#include "llvm/TargetParser/Triple.h"
66#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
67#include <cassert>
68#include <cstdint>
69#include <map>
70#include <memory>
71
72using namespace llvm;
73
74enum PtrauthCheckMode { Default, Unchecked, Poison, Trap };
75static cl::opt<PtrauthCheckMode> PtrauthAuthChecks(
76 "aarch64-ptrauth-auth-checks", cl::Hidden,
77 cl::values(clEnumValN(Unchecked, "none", "don't test for failure"),
78 clEnumValN(Poison, "poison", "poison on failure"),
79 clEnumValN(Trap, "trap", "trap on failure")),
80 cl::desc("Check pointer authentication auth/resign failures"),
81 cl::init(Val: Default));
82
83#define DEBUG_TYPE "asm-printer"
84
85namespace {
86
87class AArch64AsmPrinter : public AsmPrinter {
88 AArch64MCInstLower MCInstLowering;
89 FaultMaps FM;
90 const AArch64Subtarget *STI;
91 bool ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = false;
92#ifndef NDEBUG
93 unsigned InstsEmitted;
94#endif
95 bool EnableImportCallOptimization = false;
96 DenseMap<MCSection *, std::vector<std::pair<MCSymbol *, MCSymbol *>>>
97 SectionToImportedFunctionCalls;
98
99public:
100 static char ID;
101
102 AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
103 : AsmPrinter(TM, std::move(Streamer), ID),
104 MCInstLowering(OutContext, *this), FM(*this) {}
105
106 StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
107
108 /// Wrapper for MCInstLowering.lowerOperand() for the
109 /// tblgen'erated pseudo lowering.
110 bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
111 return MCInstLowering.lowerOperand(MO, MCOp);
112 }
113
114 const MCExpr *lowerConstantPtrAuth(const ConstantPtrAuth &CPA) override;
115
116 const MCExpr *lowerBlockAddressConstant(const BlockAddress &BA) override;
117
118 void emitStartOfAsmFile(Module &M) override;
119 void emitJumpTableImpl(const MachineJumpTableInfo &MJTI,
120 ArrayRef<unsigned> JumpTableIndices) override;
121 std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
122 codeview::JumpTableEntrySize>
123 getCodeViewJumpTableInfo(int JTI, const MachineInstr *BranchInstr,
124 const MCSymbol *BranchLabel) const override;
125
126 void emitFunctionEntryLabel() override;
127
128 void emitXXStructor(const DataLayout &DL, const Constant *CV) override;
129
130 void LowerJumpTableDest(MCStreamer &OutStreamer, const MachineInstr &MI);
131
132 void LowerHardenedBRJumpTable(const MachineInstr &MI);
133
134 void LowerMOPS(MCStreamer &OutStreamer, const MachineInstr &MI);
135
136 void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
137 const MachineInstr &MI);
138 void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
139 const MachineInstr &MI);
140 void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
141 const MachineInstr &MI);
142 void LowerFAULTING_OP(const MachineInstr &MI);
143
144 void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
145 void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
146 void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
147 void LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI, bool Typed);
148
149 typedef std::tuple<unsigned, bool, uint32_t, bool, uint64_t>
150 HwasanMemaccessTuple;
151 std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
152 void LowerKCFI_CHECK(const MachineInstr &MI);
153 void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
154 void emitHwasanMemaccessSymbols(Module &M);
155
156 void emitSled(const MachineInstr &MI, SledKind Kind);
157
158 // Emit the sequence for BRA/BLRA (authenticate + branch/call).
159 void emitPtrauthBranch(const MachineInstr *MI);
160
161 void emitPtrauthCheckAuthenticatedValue(Register TestedReg,
162 Register ScratchReg,
163 AArch64PACKey::ID Key,
164 AArch64PAuth::AuthCheckMethod Method,
165 bool ShouldTrap,
166 const MCSymbol *OnFailure);
167
168 // Check authenticated LR before tail calling.
169 void emitPtrauthTailCallHardening(const MachineInstr *TC);
170
171 // Emit the sequence for AUT or AUTPAC.
172 void emitPtrauthAuthResign(const MachineInstr *MI);
173
174 // Emit the sequence to compute the discriminator.
175 //
176 // ScratchReg should be x16/x17.
177 //
178 // The returned register is either unmodified AddrDisc or x16/x17.
179 //
180 // If the expanded pseudo is allowed to clobber AddrDisc register, setting
181 // MayUseAddrAsScratch may save one MOV instruction, provided the address
182 // is already in x16/x17 (i.e. return x16/x17 which is the *modified* AddrDisc
183 // register at the same time):
184 //
185 // mov x17, x16
186 // movk x17, #1234, lsl #48
187 // ; x16 is not used anymore
188 //
189 // can be replaced by
190 //
191 // movk x16, #1234, lsl #48
192 Register emitPtrauthDiscriminator(uint16_t Disc, Register AddrDisc,
193 Register ScratchReg,
194 bool MayUseAddrAsScratch = false);
195
196 // Emit the sequence for LOADauthptrstatic
197 void LowerLOADauthptrstatic(const MachineInstr &MI);
198
199 // Emit the sequence for LOADgotPAC/MOVaddrPAC (either GOT adrp-ldr or
200 // adrp-add followed by PAC sign)
201 void LowerMOVaddrPAC(const MachineInstr &MI);
202
203 // Emit the sequence for LOADgotAUTH (load signed pointer from signed ELF GOT
204 // and authenticate it with, if FPAC bit is not set, check+trap sequence after
205 // authenticating)
206 void LowerLOADgotAUTH(const MachineInstr &MI);
207
208 /// tblgen'erated driver function for lowering simple MI->MC
209 /// pseudo instructions.
210 bool lowerPseudoInstExpansion(const MachineInstr *MI, MCInst &Inst);
211
212 // Emit Build Attributes
213 void emitAttributes(unsigned Flags, uint64_t PAuthABIPlatform,
214 uint64_t PAuthABIVersion, AArch64TargetStreamer *TS);
215
216 // Emit expansion of Compare-and-branch pseudo instructions
217 void emitCBPseudoExpansion(const MachineInstr *MI);
218
219 void EmitToStreamer(MCStreamer &S, const MCInst &Inst);
220 void EmitToStreamer(const MCInst &Inst) {
221 EmitToStreamer(S&: *OutStreamer, Inst);
222 }
223
224 void emitInstruction(const MachineInstr *MI) override;
225
226 void emitFunctionHeaderComment() override;
227
228 void getAnalysisUsage(AnalysisUsage &AU) const override {
229 AsmPrinter::getAnalysisUsage(AU);
230 AU.setPreservesAll();
231 }
232
233 bool runOnMachineFunction(MachineFunction &MF) override {
234 if (auto *PSIW = getAnalysisIfAvailable<ProfileSummaryInfoWrapperPass>())
235 PSI = &PSIW->getPSI();
236 if (auto *SDPIW =
237 getAnalysisIfAvailable<StaticDataProfileInfoWrapperPass>())
238 SDPI = &SDPIW->getStaticDataProfileInfo();
239
240 AArch64FI = MF.getInfo<AArch64FunctionInfo>();
241 STI = &MF.getSubtarget<AArch64Subtarget>();
242
243 SetupMachineFunction(MF);
244
245 if (STI->isTargetCOFF()) {
246 bool Local = MF.getFunction().hasLocalLinkage();
247 COFF::SymbolStorageClass Scl =
248 Local ? COFF::IMAGE_SYM_CLASS_STATIC : COFF::IMAGE_SYM_CLASS_EXTERNAL;
249 int Type =
250 COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
251
252 OutStreamer->beginCOFFSymbolDef(Symbol: CurrentFnSym);
253 OutStreamer->emitCOFFSymbolStorageClass(StorageClass: Scl);
254 OutStreamer->emitCOFFSymbolType(Type);
255 OutStreamer->endCOFFSymbolDef();
256 }
257
258 // Emit the rest of the function body.
259 emitFunctionBody();
260
261 // Emit the XRay table for this function.
262 emitXRayTable();
263
264 // We didn't modify anything.
265 return false;
266 }
267
268 const MCExpr *lowerConstant(const Constant *CV,
269 const Constant *BaseCV = nullptr,
270 uint64_t Offset = 0) override;
271
272private:
273 void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
274 bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
275 bool printAsmRegInClass(const MachineOperand &MO,
276 const TargetRegisterClass *RC, unsigned AltName,
277 raw_ostream &O);
278
279 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
280 const char *ExtraCode, raw_ostream &O) override;
281 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
282 const char *ExtraCode, raw_ostream &O) override;
283
284 void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
285
286 void emitFunctionBodyEnd() override;
287 void emitGlobalAlias(const Module &M, const GlobalAlias &GA) override;
288
289 MCSymbol *GetCPISymbol(unsigned CPID) const override;
290 void emitEndOfAsmFile(Module &M) override;
291
292 AArch64FunctionInfo *AArch64FI = nullptr;
293
294 /// Emit the LOHs contained in AArch64FI.
295 void emitLOHs();
296
297 void emitMovXReg(Register Dest, Register Src);
298 void emitMOVZ(Register Dest, uint64_t Imm, unsigned Shift);
299 void emitMOVK(Register Dest, uint64_t Imm, unsigned Shift);
300
301 /// Emit instruction to set float register to zero.
302 void emitFMov0(const MachineInstr &MI);
303
304 using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
305
306 MInstToMCSymbol LOHInstToLabel;
307
308 bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const override {
309 return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags;
310 }
311
312 const MCSubtargetInfo *getIFuncMCSubtargetInfo() const override {
313 assert(STI);
314 return STI;
315 }
316 void emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI,
317 MCSymbol *LazyPointer) override;
318 void emitMachOIFuncStubHelperBody(Module &M, const GlobalIFunc &GI,
319 MCSymbol *LazyPointer) override;
320
321 /// Checks if this instruction is part of a sequence that is eligle for import
322 /// call optimization and, if so, records it to be emitted in the import call
323 /// section.
324 void recordIfImportCall(const MachineInstr *BranchInst);
325};
326
327} // end anonymous namespace
328
329void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
330 const Triple &TT = TM.getTargetTriple();
331
332 if (TT.isOSBinFormatCOFF()) {
333 emitCOFFFeatureSymbol(M);
334 emitCOFFReplaceableFunctionData(M);
335
336 if (M.getModuleFlag(Key: "import-call-optimization"))
337 EnableImportCallOptimization = true;
338 }
339
340 if (!TT.isOSBinFormatELF())
341 return;
342
343 // For emitting build attributes and .note.gnu.property section
344 auto *TS =
345 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
346 // Assemble feature flags that may require creation of build attributes and a
347 // note section.
348 unsigned BAFlags = 0;
349 unsigned GNUFlags = 0;
350 if (const auto *BTE = mdconst::extract_or_null<ConstantInt>(
351 MD: M.getModuleFlag(Key: "branch-target-enforcement"))) {
352 if (!BTE->isZero()) {
353 BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_BTI_Flag;
354 GNUFlags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
355 }
356 }
357
358 if (const auto *GCS = mdconst::extract_or_null<ConstantInt>(
359 MD: M.getModuleFlag(Key: "guarded-control-stack"))) {
360 if (!GCS->isZero()) {
361 BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_GCS_Flag;
362 GNUFlags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_GCS;
363 }
364 }
365
366 if (const auto *Sign = mdconst::extract_or_null<ConstantInt>(
367 MD: M.getModuleFlag(Key: "sign-return-address"))) {
368 if (!Sign->isZero()) {
369 BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_PAC_Flag;
370 GNUFlags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
371 }
372 }
373
374 uint64_t PAuthABIPlatform = -1;
375 if (const auto *PAP = mdconst::extract_or_null<ConstantInt>(
376 MD: M.getModuleFlag(Key: "aarch64-elf-pauthabi-platform"))) {
377 PAuthABIPlatform = PAP->getZExtValue();
378 }
379
380 uint64_t PAuthABIVersion = -1;
381 if (const auto *PAV = mdconst::extract_or_null<ConstantInt>(
382 MD: M.getModuleFlag(Key: "aarch64-elf-pauthabi-version"))) {
383 PAuthABIVersion = PAV->getZExtValue();
384 }
385
386 // Emit AArch64 Build Attributes
387 emitAttributes(Flags: BAFlags, PAuthABIPlatform, PAuthABIVersion, TS);
388 // Emit a .note.gnu.property section with the flags.
389 TS->emitNoteSection(Flags: GNUFlags, PAuthABIPlatform, PAuthABIVersion);
390}
391
392void AArch64AsmPrinter::emitFunctionHeaderComment() {
393 const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>();
394 std::optional<std::string> OutlinerString = FI->getOutliningStyle();
395 if (OutlinerString != std::nullopt)
396 OutStreamer->getCommentOS() << ' ' << OutlinerString;
397}
398
399void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
400{
401 const Function &F = MF->getFunction();
402 if (F.hasFnAttribute(Kind: "patchable-function-entry")) {
403 unsigned Num;
404 if (F.getFnAttribute(Kind: "patchable-function-entry")
405 .getValueAsString()
406 .getAsInteger(Radix: 10, Result&: Num))
407 return;
408 emitNops(N: Num);
409 return;
410 }
411
412 emitSled(MI, Kind: SledKind::FUNCTION_ENTER);
413}
414
415void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI) {
416 emitSled(MI, Kind: SledKind::FUNCTION_EXIT);
417}
418
419void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) {
420 emitSled(MI, Kind: SledKind::TAIL_CALL);
421}
422
423void AArch64AsmPrinter::emitSled(const MachineInstr &MI, SledKind Kind) {
424 static const int8_t NoopsInSledCount = 7;
425 // We want to emit the following pattern:
426 //
427 // .Lxray_sled_N:
428 // ALIGN
429 // B #32
430 // ; 7 NOP instructions (28 bytes)
431 // .tmpN
432 //
433 // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
434 // over the full 32 bytes (8 instructions) with the following pattern:
435 //
436 // STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
437 // LDR W17, #12 ; W17 := function ID
438 // LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
439 // BLR X16 ; call the tracing trampoline
440 // ;DATA: 32 bits of function ID
441 // ;DATA: lower 32 bits of the address of the trampoline
442 // ;DATA: higher 32 bits of the address of the trampoline
443 // LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
444 //
445 OutStreamer->emitCodeAlignment(Alignment: Align(4), STI: &getSubtargetInfo());
446 auto CurSled = OutContext.createTempSymbol(Name: "xray_sled_", AlwaysAddSuffix: true);
447 OutStreamer->emitLabel(Symbol: CurSled);
448 auto Target = OutContext.createTempSymbol();
449
450 // Emit "B #32" instruction, which jumps over the next 28 bytes.
451 // The operand has to be the number of 4-byte instructions to jump over,
452 // including the current instruction.
453 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::B).addImm(Val: 8));
454
455 for (int8_t I = 0; I < NoopsInSledCount; I++)
456 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::HINT).addImm(Val: 0));
457
458 OutStreamer->emitLabel(Symbol: Target);
459 recordSled(Sled: CurSled, MI, Kind, Version: 2);
460}
461
462void AArch64AsmPrinter::emitAttributes(unsigned Flags,
463 uint64_t PAuthABIPlatform,
464 uint64_t PAuthABIVersion,
465 AArch64TargetStreamer *TS) {
466
467 PAuthABIPlatform = (uint64_t(-1) == PAuthABIPlatform) ? 0 : PAuthABIPlatform;
468 PAuthABIVersion = (uint64_t(-1) == PAuthABIVersion) ? 0 : PAuthABIVersion;
469
470 if (PAuthABIPlatform || PAuthABIVersion) {
471 TS->emitAttributesSubsection(
472 VendorName: AArch64BuildAttributes::getVendorName(
473 Vendor: AArch64BuildAttributes::AEABI_PAUTHABI),
474 IsOptional: AArch64BuildAttributes::SubsectionOptional::REQUIRED,
475 ParameterType: AArch64BuildAttributes::SubsectionType::ULEB128);
476 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
477 Vendor: AArch64BuildAttributes::AEABI_PAUTHABI),
478 Tag: AArch64BuildAttributes::TAG_PAUTH_PLATFORM,
479 Value: PAuthABIPlatform, String: "");
480 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
481 Vendor: AArch64BuildAttributes::AEABI_PAUTHABI),
482 Tag: AArch64BuildAttributes::TAG_PAUTH_SCHEMA, Value: PAuthABIVersion,
483 String: "");
484 }
485
486 unsigned BTIValue =
487 (Flags & AArch64BuildAttributes::Feature_BTI_Flag) ? 1 : 0;
488 unsigned PACValue =
489 (Flags & AArch64BuildAttributes::Feature_PAC_Flag) ? 1 : 0;
490 unsigned GCSValue =
491 (Flags & AArch64BuildAttributes::Feature_GCS_Flag) ? 1 : 0;
492
493 if (BTIValue || PACValue || GCSValue) {
494 TS->emitAttributesSubsection(
495 VendorName: AArch64BuildAttributes::getVendorName(
496 Vendor: AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
497 IsOptional: AArch64BuildAttributes::SubsectionOptional::OPTIONAL,
498 ParameterType: AArch64BuildAttributes::SubsectionType::ULEB128);
499 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
500 Vendor: AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
501 Tag: AArch64BuildAttributes::TAG_FEATURE_BTI, Value: BTIValue, String: "");
502 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
503 Vendor: AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
504 Tag: AArch64BuildAttributes::TAG_FEATURE_PAC, Value: PACValue, String: "");
505 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
506 Vendor: AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
507 Tag: AArch64BuildAttributes::TAG_FEATURE_GCS, Value: GCSValue, String: "");
508 }
509}
510
511// Emit the following code for Intrinsic::{xray_customevent,xray_typedevent}
512// (built-in functions __xray_customevent/__xray_typedevent).
513//
514// .Lxray_event_sled_N:
515// b 1f
516// save x0 and x1 (and also x2 for TYPED_EVENT_CALL)
517// set up x0 and x1 (and also x2 for TYPED_EVENT_CALL)
518// bl __xray_CustomEvent or __xray_TypedEvent
519// restore x0 and x1 (and also x2 for TYPED_EVENT_CALL)
520// 1:
521//
522// There are 6 instructions for EVENT_CALL and 9 for TYPED_EVENT_CALL.
523//
524// Then record a sled of kind CUSTOM_EVENT or TYPED_EVENT.
525// After patching, b .+N will become a nop.
526void AArch64AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI,
527 bool Typed) {
528 auto &O = *OutStreamer;
529 MCSymbol *CurSled = OutContext.createTempSymbol(Name: "xray_sled_", AlwaysAddSuffix: true);
530 O.emitLabel(Symbol: CurSled);
531 bool MachO = TM.getTargetTriple().isOSBinFormatMachO();
532 auto *Sym = MCSymbolRefExpr::create(
533 Symbol: OutContext.getOrCreateSymbol(
534 Name: Twine(MachO ? "_" : "") +
535 (Typed ? "__xray_TypedEvent" : "__xray_CustomEvent")),
536 Ctx&: OutContext);
537 if (Typed) {
538 O.AddComment(T: "Begin XRay typed event");
539 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::B).addImm(Val: 9));
540 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::STPXpre)
541 .addReg(Reg: AArch64::SP)
542 .addReg(Reg: AArch64::X0)
543 .addReg(Reg: AArch64::X1)
544 .addReg(Reg: AArch64::SP)
545 .addImm(Val: -4));
546 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::STRXui)
547 .addReg(Reg: AArch64::X2)
548 .addReg(Reg: AArch64::SP)
549 .addImm(Val: 2));
550 emitMovXReg(Dest: AArch64::X0, Src: MI.getOperand(i: 0).getReg());
551 emitMovXReg(Dest: AArch64::X1, Src: MI.getOperand(i: 1).getReg());
552 emitMovXReg(Dest: AArch64::X2, Src: MI.getOperand(i: 2).getReg());
553 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::BL).addExpr(Val: Sym));
554 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::LDRXui)
555 .addReg(Reg: AArch64::X2)
556 .addReg(Reg: AArch64::SP)
557 .addImm(Val: 2));
558 O.AddComment(T: "End XRay typed event");
559 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::LDPXpost)
560 .addReg(Reg: AArch64::SP)
561 .addReg(Reg: AArch64::X0)
562 .addReg(Reg: AArch64::X1)
563 .addReg(Reg: AArch64::SP)
564 .addImm(Val: 4));
565
566 recordSled(Sled: CurSled, MI, Kind: SledKind::TYPED_EVENT, Version: 2);
567 } else {
568 O.AddComment(T: "Begin XRay custom event");
569 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::B).addImm(Val: 6));
570 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::STPXpre)
571 .addReg(Reg: AArch64::SP)
572 .addReg(Reg: AArch64::X0)
573 .addReg(Reg: AArch64::X1)
574 .addReg(Reg: AArch64::SP)
575 .addImm(Val: -2));
576 emitMovXReg(Dest: AArch64::X0, Src: MI.getOperand(i: 0).getReg());
577 emitMovXReg(Dest: AArch64::X1, Src: MI.getOperand(i: 1).getReg());
578 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::BL).addExpr(Val: Sym));
579 O.AddComment(T: "End XRay custom event");
580 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::LDPXpost)
581 .addReg(Reg: AArch64::SP)
582 .addReg(Reg: AArch64::X0)
583 .addReg(Reg: AArch64::X1)
584 .addReg(Reg: AArch64::SP)
585 .addImm(Val: 2));
586
587 recordSled(Sled: CurSled, MI, Kind: SledKind::CUSTOM_EVENT, Version: 2);
588 }
589}
590
591void AArch64AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
592 Register AddrReg = MI.getOperand(i: 0).getReg();
593 assert(std::next(MI.getIterator())->isCall() &&
594 "KCFI_CHECK not followed by a call instruction");
595 assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg &&
596 "KCFI_CHECK call target doesn't match call operand");
597
598 // Default to using the intra-procedure-call temporary registers for
599 // comparing the hashes.
600 unsigned ScratchRegs[] = {AArch64::W16, AArch64::W17};
601 if (AddrReg == AArch64::XZR) {
602 // Checking XZR makes no sense. Instead of emitting a load, zero
603 // ScratchRegs[0] and use it for the ESR AddrIndex below.
604 AddrReg = getXRegFromWReg(Reg: ScratchRegs[0]);
605 emitMovXReg(Dest: AddrReg, Src: AArch64::XZR);
606 } else {
607 // If one of the scratch registers is used for the call target (e.g.
608 // with AArch64::TCRETURNriBTI), we can clobber another caller-saved
609 // temporary register instead (in this case, AArch64::W9) as the check
610 // is immediately followed by the call instruction.
611 for (auto &Reg : ScratchRegs) {
612 if (Reg == getWRegFromXReg(Reg: AddrReg)) {
613 Reg = AArch64::W9;
614 break;
615 }
616 }
617 assert(ScratchRegs[0] != AddrReg && ScratchRegs[1] != AddrReg &&
618 "Invalid scratch registers for KCFI_CHECK");
619
620 // Adjust the offset for patchable-function-prefix. This assumes that
621 // patchable-function-prefix is the same for all functions.
622 int64_t PrefixNops = 0;
623 (void)MI.getMF()
624 ->getFunction()
625 .getFnAttribute(Kind: "patchable-function-prefix")
626 .getValueAsString()
627 .getAsInteger(Radix: 10, Result&: PrefixNops);
628
629 // Load the target function type hash.
630 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::LDURWi)
631 .addReg(Reg: ScratchRegs[0])
632 .addReg(Reg: AddrReg)
633 .addImm(Val: -(PrefixNops * 4 + 4)));
634 }
635
636 // Load the expected type hash.
637 const int64_t Type = MI.getOperand(i: 1).getImm();
638 emitMOVK(Dest: ScratchRegs[1], Imm: Type & 0xFFFF, Shift: 0);
639 emitMOVK(Dest: ScratchRegs[1], Imm: (Type >> 16) & 0xFFFF, Shift: 16);
640
641 // Compare the hashes and trap if there's a mismatch.
642 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::SUBSWrs)
643 .addReg(Reg: AArch64::WZR)
644 .addReg(Reg: ScratchRegs[0])
645 .addReg(Reg: ScratchRegs[1])
646 .addImm(Val: 0));
647
648 MCSymbol *Pass = OutContext.createTempSymbol();
649 EmitToStreamer(S&: *OutStreamer,
650 Inst: MCInstBuilder(AArch64::Bcc)
651 .addImm(Val: AArch64CC::EQ)
652 .addExpr(Val: MCSymbolRefExpr::create(Symbol: Pass, Ctx&: OutContext)));
653
654 // The base ESR is 0x8000 and the register information is encoded in bits
655 // 0-9 as follows:
656 // - 0-4: n, where the register Xn contains the target address
657 // - 5-9: m, where the register Wm contains the expected type hash
658 // Where n, m are in [0, 30].
659 unsigned TypeIndex = ScratchRegs[1] - AArch64::W0;
660 unsigned AddrIndex;
661 switch (AddrReg) {
662 default:
663 AddrIndex = AddrReg - AArch64::X0;
664 break;
665 case AArch64::FP:
666 AddrIndex = 29;
667 break;
668 case AArch64::LR:
669 AddrIndex = 30;
670 break;
671 }
672
673 assert(AddrIndex < 31 && TypeIndex < 31);
674
675 unsigned ESR = 0x8000 | ((TypeIndex & 31) << 5) | (AddrIndex & 31);
676 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::BRK).addImm(Val: ESR));
677 OutStreamer->emitLabel(Symbol: Pass);
678}
679
680void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
681 Register Reg = MI.getOperand(i: 0).getReg();
682
683 // The HWASan pass won't emit a CHECK_MEMACCESS intrinsic with a pointer
684 // statically known to be zero. However, conceivably, the HWASan pass may
685 // encounter a "cannot currently statically prove to be null" pointer (and is
686 // therefore unable to omit the intrinsic) that later optimization passes
687 // convert into a statically known-null pointer.
688 if (Reg == AArch64::XZR)
689 return;
690
691 bool IsShort =
692 ((MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES) ||
693 (MI.getOpcode() ==
694 AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW));
695 uint32_t AccessInfo = MI.getOperand(i: 1).getImm();
696 bool IsFixedShadow =
697 ((MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_FIXEDSHADOW) ||
698 (MI.getOpcode() ==
699 AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW));
700 uint64_t FixedShadowOffset = IsFixedShadow ? MI.getOperand(i: 2).getImm() : 0;
701
702 MCSymbol *&Sym = HwasanMemaccessSymbols[HwasanMemaccessTuple(
703 Reg, IsShort, AccessInfo, IsFixedShadow, FixedShadowOffset)];
704 if (!Sym) {
705 // FIXME: Make this work on non-ELF.
706 if (!TM.getTargetTriple().isOSBinFormatELF())
707 report_fatal_error(reason: "llvm.hwasan.check.memaccess only supported on ELF");
708
709 std::string SymName = "__hwasan_check_x" + utostr(X: Reg - AArch64::X0) + "_" +
710 utostr(X: AccessInfo);
711 if (IsFixedShadow)
712 SymName += "_fixed_" + utostr(X: FixedShadowOffset);
713 if (IsShort)
714 SymName += "_short_v2";
715 Sym = OutContext.getOrCreateSymbol(Name: SymName);
716 }
717
718 EmitToStreamer(S&: *OutStreamer,
719 Inst: MCInstBuilder(AArch64::BL)
720 .addExpr(Val: MCSymbolRefExpr::create(Symbol: Sym, Ctx&: OutContext)));
721}
722
723void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) {
724 if (HwasanMemaccessSymbols.empty())
725 return;
726
727 const Triple &TT = TM.getTargetTriple();
728 assert(TT.isOSBinFormatELF());
729 std::unique_ptr<MCSubtargetInfo> STI(
730 TM.getTarget().createMCSubtargetInfo(TheTriple: TT.str(), CPU: "", Features: ""));
731 assert(STI && "Unable to create subtarget info");
732 this->STI = static_cast<const AArch64Subtarget *>(&*STI);
733
734 MCSymbol *HwasanTagMismatchV1Sym =
735 OutContext.getOrCreateSymbol(Name: "__hwasan_tag_mismatch");
736 MCSymbol *HwasanTagMismatchV2Sym =
737 OutContext.getOrCreateSymbol(Name: "__hwasan_tag_mismatch_v2");
738
739 const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
740 MCSymbolRefExpr::create(Symbol: HwasanTagMismatchV1Sym, Ctx&: OutContext);
741 const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
742 MCSymbolRefExpr::create(Symbol: HwasanTagMismatchV2Sym, Ctx&: OutContext);
743
744 for (auto &P : HwasanMemaccessSymbols) {
745 unsigned Reg = std::get<0>(t: P.first);
746 bool IsShort = std::get<1>(t: P.first);
747 uint32_t AccessInfo = std::get<2>(t: P.first);
748 bool IsFixedShadow = std::get<3>(t: P.first);
749 uint64_t FixedShadowOffset = std::get<4>(t: P.first);
750 const MCSymbolRefExpr *HwasanTagMismatchRef =
751 IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
752 MCSymbol *Sym = P.second;
753
754 bool HasMatchAllTag =
755 (AccessInfo >> HWASanAccessInfo::HasMatchAllShift) & 1;
756 uint8_t MatchAllTag =
757 (AccessInfo >> HWASanAccessInfo::MatchAllShift) & 0xff;
758 unsigned Size =
759 1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf);
760 bool CompileKernel =
761 (AccessInfo >> HWASanAccessInfo::CompileKernelShift) & 1;
762
763 OutStreamer->switchSection(Section: OutContext.getELFSection(
764 Section: ".text.hot", Type: ELF::SHT_PROGBITS,
765 Flags: ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, EntrySize: 0, Group: Sym->getName(),
766 /*IsComdat=*/true));
767
768 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_ELF_TypeFunction);
769 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Weak);
770 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Hidden);
771 OutStreamer->emitLabel(Symbol: Sym);
772
773 EmitToStreamer(Inst: MCInstBuilder(AArch64::SBFMXri)
774 .addReg(Reg: AArch64::X16)
775 .addReg(Reg)
776 .addImm(Val: 4)
777 .addImm(Val: 55));
778
779 if (IsFixedShadow) {
780 // Aarch64 makes it difficult to embed large constants in the code.
781 // Fortuitously, kShadowBaseAlignment == 32, so we use the 32-bit
782 // left-shift option in the MOV instruction. Combined with the 16-bit
783 // immediate, this is enough to represent any offset up to 2**48.
784 emitMOVZ(Dest: AArch64::X17, Imm: FixedShadowOffset >> 32, Shift: 32);
785 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRBBroX)
786 .addReg(Reg: AArch64::W16)
787 .addReg(Reg: AArch64::X17)
788 .addReg(Reg: AArch64::X16)
789 .addImm(Val: 0)
790 .addImm(Val: 0));
791 } else {
792 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRBBroX)
793 .addReg(Reg: AArch64::W16)
794 .addReg(Reg: IsShort ? AArch64::X20 : AArch64::X9)
795 .addReg(Reg: AArch64::X16)
796 .addImm(Val: 0)
797 .addImm(Val: 0));
798 }
799
800 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSXrs)
801 .addReg(Reg: AArch64::XZR)
802 .addReg(Reg: AArch64::X16)
803 .addReg(Reg)
804 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSR, Imm: 56)));
805 MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
806 EmitToStreamer(Inst: MCInstBuilder(AArch64::Bcc)
807 .addImm(Val: AArch64CC::NE)
808 .addExpr(Val: MCSymbolRefExpr::create(
809 Symbol: HandleMismatchOrPartialSym, Ctx&: OutContext)));
810 MCSymbol *ReturnSym = OutContext.createTempSymbol();
811 OutStreamer->emitLabel(Symbol: ReturnSym);
812 EmitToStreamer(Inst: MCInstBuilder(AArch64::RET).addReg(Reg: AArch64::LR));
813 OutStreamer->emitLabel(Symbol: HandleMismatchOrPartialSym);
814
815 if (HasMatchAllTag) {
816 EmitToStreamer(Inst: MCInstBuilder(AArch64::UBFMXri)
817 .addReg(Reg: AArch64::X17)
818 .addReg(Reg)
819 .addImm(Val: 56)
820 .addImm(Val: 63));
821 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSXri)
822 .addReg(Reg: AArch64::XZR)
823 .addReg(Reg: AArch64::X17)
824 .addImm(Val: MatchAllTag)
825 .addImm(Val: 0));
826 EmitToStreamer(
827 Inst: MCInstBuilder(AArch64::Bcc)
828 .addImm(Val: AArch64CC::EQ)
829 .addExpr(Val: MCSymbolRefExpr::create(Symbol: ReturnSym, Ctx&: OutContext)));
830 }
831
832 if (IsShort) {
833 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSWri)
834 .addReg(Reg: AArch64::WZR)
835 .addReg(Reg: AArch64::W16)
836 .addImm(Val: 15)
837 .addImm(Val: 0));
838 MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
839 EmitToStreamer(
840 Inst: MCInstBuilder(AArch64::Bcc)
841 .addImm(Val: AArch64CC::HI)
842 .addExpr(Val: MCSymbolRefExpr::create(Symbol: HandleMismatchSym, Ctx&: OutContext)));
843
844 EmitToStreamer(Inst: MCInstBuilder(AArch64::ANDXri)
845 .addReg(Reg: AArch64::X17)
846 .addReg(Reg)
847 .addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: 0xf, regSize: 64)));
848 if (Size != 1)
849 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
850 .addReg(Reg: AArch64::X17)
851 .addReg(Reg: AArch64::X17)
852 .addImm(Val: Size - 1)
853 .addImm(Val: 0));
854 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSWrs)
855 .addReg(Reg: AArch64::WZR)
856 .addReg(Reg: AArch64::W16)
857 .addReg(Reg: AArch64::W17)
858 .addImm(Val: 0));
859 EmitToStreamer(
860 Inst: MCInstBuilder(AArch64::Bcc)
861 .addImm(Val: AArch64CC::LS)
862 .addExpr(Val: MCSymbolRefExpr::create(Symbol: HandleMismatchSym, Ctx&: OutContext)));
863
864 EmitToStreamer(Inst: MCInstBuilder(AArch64::ORRXri)
865 .addReg(Reg: AArch64::X16)
866 .addReg(Reg)
867 .addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: 0xf, regSize: 64)));
868 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRBBui)
869 .addReg(Reg: AArch64::W16)
870 .addReg(Reg: AArch64::X16)
871 .addImm(Val: 0));
872 EmitToStreamer(
873 Inst: MCInstBuilder(AArch64::SUBSXrs)
874 .addReg(Reg: AArch64::XZR)
875 .addReg(Reg: AArch64::X16)
876 .addReg(Reg)
877 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSR, Imm: 56)));
878 EmitToStreamer(
879 Inst: MCInstBuilder(AArch64::Bcc)
880 .addImm(Val: AArch64CC::EQ)
881 .addExpr(Val: MCSymbolRefExpr::create(Symbol: ReturnSym, Ctx&: OutContext)));
882
883 OutStreamer->emitLabel(Symbol: HandleMismatchSym);
884 }
885
886 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPXpre)
887 .addReg(Reg: AArch64::SP)
888 .addReg(Reg: AArch64::X0)
889 .addReg(Reg: AArch64::X1)
890 .addReg(Reg: AArch64::SP)
891 .addImm(Val: -32));
892 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPXi)
893 .addReg(Reg: AArch64::FP)
894 .addReg(Reg: AArch64::LR)
895 .addReg(Reg: AArch64::SP)
896 .addImm(Val: 29));
897
898 if (Reg != AArch64::X0)
899 emitMovXReg(Dest: AArch64::X0, Src: Reg);
900 emitMOVZ(Dest: AArch64::X1, Imm: AccessInfo & HWASanAccessInfo::RuntimeMask, Shift: 0);
901
902 if (CompileKernel) {
903 // The Linux kernel's dynamic loader doesn't support GOT relative
904 // relocations, but it doesn't support late binding either, so just call
905 // the function directly.
906 EmitToStreamer(Inst: MCInstBuilder(AArch64::B).addExpr(Val: HwasanTagMismatchRef));
907 } else {
908 // Intentionally load the GOT entry and branch to it, rather than possibly
909 // late binding the function, which may clobber the registers before we
910 // have a chance to save them.
911 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADRP)
912 .addReg(Reg: AArch64::X16)
913 .addExpr(Val: MCSpecifierExpr::create(Expr: HwasanTagMismatchRef,
914 S: AArch64::S_GOT_PAGE,
915 Ctx&: OutContext)));
916 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
917 .addReg(Reg: AArch64::X16)
918 .addReg(Reg: AArch64::X16)
919 .addExpr(Val: MCSpecifierExpr::create(Expr: HwasanTagMismatchRef,
920 S: AArch64::S_GOT_LO12,
921 Ctx&: OutContext)));
922 EmitToStreamer(Inst: MCInstBuilder(AArch64::BR).addReg(Reg: AArch64::X16));
923 }
924 }
925 this->STI = nullptr;
926}
927
928static void emitAuthenticatedPointer(MCStreamer &OutStreamer,
929 MCSymbol *StubLabel,
930 const MCExpr *StubAuthPtrRef) {
931 // sym$auth_ptr$key$disc:
932 OutStreamer.emitLabel(Symbol: StubLabel);
933 OutStreamer.emitValue(Value: StubAuthPtrRef, /*size=*/Size: 8);
934}
935
936void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
937 emitHwasanMemaccessSymbols(M);
938
939 const Triple &TT = TM.getTargetTriple();
940 if (TT.isOSBinFormatMachO()) {
941 // Output authenticated pointers as indirect symbols, if we have any.
942 MachineModuleInfoMachO &MMIMacho =
943 MMI->getObjFileInfo<MachineModuleInfoMachO>();
944
945 auto Stubs = MMIMacho.getAuthGVStubList();
946
947 if (!Stubs.empty()) {
948 // Switch to the "__auth_ptr" section.
949 OutStreamer->switchSection(
950 Section: OutContext.getMachOSection(Segment: "__DATA", Section: "__auth_ptr", TypeAndAttributes: MachO::S_REGULAR,
951 K: SectionKind::getMetadata()));
952 emitAlignment(Alignment: Align(8));
953
954 for (const auto &Stub : Stubs)
955 emitAuthenticatedPointer(OutStreamer&: *OutStreamer, StubLabel: Stub.first, StubAuthPtrRef: Stub.second);
956
957 OutStreamer->addBlankLine();
958 }
959
960 // Funny Darwin hack: This flag tells the linker that no global symbols
961 // contain code that falls through to other global symbols (e.g. the obvious
962 // implementation of multiple entry points). If this doesn't occur, the
963 // linker can safely perform dead code stripping. Since LLVM never
964 // generates code that does this, it is always safe to set.
965 OutStreamer->emitSubsectionsViaSymbols();
966 }
967
968 if (TT.isOSBinFormatELF()) {
969 // Output authenticated pointers as indirect symbols, if we have any.
970 MachineModuleInfoELF &MMIELF = MMI->getObjFileInfo<MachineModuleInfoELF>();
971
972 auto Stubs = MMIELF.getAuthGVStubList();
973
974 if (!Stubs.empty()) {
975 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
976 OutStreamer->switchSection(Section: TLOF.getDataSection());
977 emitAlignment(Alignment: Align(8));
978
979 for (const auto &Stub : Stubs)
980 emitAuthenticatedPointer(OutStreamer&: *OutStreamer, StubLabel: Stub.first, StubAuthPtrRef: Stub.second);
981
982 OutStreamer->addBlankLine();
983 }
984
985 // With signed ELF GOT enabled, the linker looks at the symbol type to
986 // choose between keys IA (for STT_FUNC) and DA (for other types). Symbols
987 // for functions not defined in the module have STT_NOTYPE type by default.
988 // This makes linker to emit signing schema with DA key (instead of IA) for
989 // corresponding R_AARCH64_AUTH_GLOB_DAT dynamic reloc. To avoid that, force
990 // all function symbols used in the module to have STT_FUNC type. See
991 // https://github.com/ARM-software/abi-aa/blob/main/pauthabielf64/pauthabielf64.rst#default-signing-schema
992 const auto *PtrAuthELFGOTFlag = mdconst::extract_or_null<ConstantInt>(
993 MD: M.getModuleFlag(Key: "ptrauth-elf-got"));
994 if (PtrAuthELFGOTFlag && PtrAuthELFGOTFlag->getZExtValue() == 1)
995 for (const GlobalValue &GV : M.global_values())
996 if (!GV.use_empty() && isa<Function>(Val: GV) &&
997 !GV.getName().starts_with(Prefix: "llvm."))
998 OutStreamer->emitSymbolAttribute(Symbol: getSymbol(GV: &GV),
999 Attribute: MCSA_ELF_TypeFunction);
1000 }
1001
1002 // Emit stack and fault map information.
1003 FM.serializeToFaultMapSection();
1004
1005 // If import call optimization is enabled, emit the appropriate section.
1006 // We do this whether or not we recorded any import calls.
1007 if (EnableImportCallOptimization && TT.isOSBinFormatCOFF()) {
1008 OutStreamer->switchSection(Section: getObjFileLowering().getImportCallSection());
1009
1010 // Section always starts with some magic.
1011 constexpr char ImpCallMagic[12] = "Imp_Call_V1";
1012 OutStreamer->emitBytes(Data: StringRef{ImpCallMagic, sizeof(ImpCallMagic)});
1013
1014 // Layout of this section is:
1015 // Per section that contains calls to imported functions:
1016 // uint32_t SectionSize: Size in bytes for information in this section.
1017 // uint32_t Section Number
1018 // Per call to imported function in section:
1019 // uint32_t Kind: the kind of imported function.
1020 // uint32_t BranchOffset: the offset of the branch instruction in its
1021 // parent section.
1022 // uint32_t TargetSymbolId: the symbol id of the called function.
1023 for (auto &[Section, CallsToImportedFuncs] :
1024 SectionToImportedFunctionCalls) {
1025 unsigned SectionSize =
1026 sizeof(uint32_t) * (2 + 3 * CallsToImportedFuncs.size());
1027 OutStreamer->emitInt32(Value: SectionSize);
1028 OutStreamer->emitCOFFSecNumber(Symbol: Section->getBeginSymbol());
1029 for (auto &[CallsiteSymbol, CalledSymbol] : CallsToImportedFuncs) {
1030 // Kind is always IMAGE_REL_ARM64_DYNAMIC_IMPORT_CALL (0x13).
1031 OutStreamer->emitInt32(Value: 0x13);
1032 OutStreamer->emitCOFFSecOffset(Symbol: CallsiteSymbol);
1033 OutStreamer->emitCOFFSymbolIndex(Symbol: CalledSymbol);
1034 }
1035 }
1036 }
1037}
1038
1039void AArch64AsmPrinter::emitLOHs() {
1040 SmallVector<MCSymbol *, 3> MCArgs;
1041
1042 for (const auto &D : AArch64FI->getLOHContainer()) {
1043 for (const MachineInstr *MI : D.getArgs()) {
1044 MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(x: MI);
1045 assert(LabelIt != LOHInstToLabel.end() &&
1046 "Label hasn't been inserted for LOH related instruction");
1047 MCArgs.push_back(Elt: LabelIt->second);
1048 }
1049 OutStreamer->emitLOHDirective(Kind: D.getKind(), Args: MCArgs);
1050 MCArgs.clear();
1051 }
1052}
1053
1054void AArch64AsmPrinter::emitFunctionBodyEnd() {
1055 if (!AArch64FI->getLOHRelated().empty())
1056 emitLOHs();
1057}
1058
1059/// GetCPISymbol - Return the symbol for the specified constant pool entry.
1060MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
1061 // Darwin uses a linker-private symbol name for constant-pools (to
1062 // avoid addends on the relocation?), ELF has no such concept and
1063 // uses a normal private symbol.
1064 if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
1065 return OutContext.getOrCreateSymbol(
1066 Name: Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
1067 Twine(getFunctionNumber()) + "_" + Twine(CPID));
1068
1069 return AsmPrinter::GetCPISymbol(CPID);
1070}
1071
1072void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
1073 raw_ostream &O) {
1074 const MachineOperand &MO = MI->getOperand(i: OpNum);
1075 switch (MO.getType()) {
1076 default:
1077 llvm_unreachable("<unknown operand type>");
1078 case MachineOperand::MO_Register: {
1079 Register Reg = MO.getReg();
1080 assert(Reg.isPhysical());
1081 assert(!MO.getSubReg() && "Subregs should be eliminated!");
1082 O << AArch64InstPrinter::getRegisterName(Reg);
1083 break;
1084 }
1085 case MachineOperand::MO_Immediate: {
1086 O << MO.getImm();
1087 break;
1088 }
1089 case MachineOperand::MO_GlobalAddress: {
1090 PrintSymbolOperand(MO, OS&: O);
1091 break;
1092 }
1093 case MachineOperand::MO_BlockAddress: {
1094 MCSymbol *Sym = GetBlockAddressSymbol(BA: MO.getBlockAddress());
1095 Sym->print(OS&: O, MAI);
1096 break;
1097 }
1098 }
1099}
1100
1101bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
1102 raw_ostream &O) {
1103 Register Reg = MO.getReg();
1104 switch (Mode) {
1105 default:
1106 return true; // Unknown mode.
1107 case 'w':
1108 Reg = getWRegFromXReg(Reg);
1109 break;
1110 case 'x':
1111 Reg = getXRegFromWReg(Reg);
1112 break;
1113 case 't':
1114 Reg = getXRegFromXRegTuple(RegTuple: Reg);
1115 break;
1116 }
1117
1118 O << AArch64InstPrinter::getRegisterName(Reg);
1119 return false;
1120}
1121
1122// Prints the register in MO using class RC using the offset in the
1123// new register class. This should not be used for cross class
1124// printing.
1125bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
1126 const TargetRegisterClass *RC,
1127 unsigned AltName, raw_ostream &O) {
1128 assert(MO.isReg() && "Should only get here with a register!");
1129 const TargetRegisterInfo *RI = STI->getRegisterInfo();
1130 Register Reg = MO.getReg();
1131 MCRegister RegToPrint = RC->getRegister(i: RI->getEncodingValue(Reg));
1132 if (!RI->regsOverlap(RegA: RegToPrint, RegB: Reg))
1133 return true;
1134 O << AArch64InstPrinter::getRegisterName(Reg: RegToPrint, AltIdx: AltName);
1135 return false;
1136}
1137
1138bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
1139 const char *ExtraCode, raw_ostream &O) {
1140 const MachineOperand &MO = MI->getOperand(i: OpNum);
1141
1142 // First try the generic code, which knows about modifiers like 'c' and 'n'.
1143 if (!AsmPrinter::PrintAsmOperand(MI, OpNo: OpNum, ExtraCode, OS&: O))
1144 return false;
1145
1146 // Does this asm operand have a single letter operand modifier?
1147 if (ExtraCode && ExtraCode[0]) {
1148 if (ExtraCode[1] != 0)
1149 return true; // Unknown modifier.
1150
1151 switch (ExtraCode[0]) {
1152 default:
1153 return true; // Unknown modifier.
1154 case 'w': // Print W register
1155 case 'x': // Print X register
1156 if (MO.isReg())
1157 return printAsmMRegister(MO, Mode: ExtraCode[0], O);
1158 if (MO.isImm() && MO.getImm() == 0) {
1159 unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
1160 O << AArch64InstPrinter::getRegisterName(Reg);
1161 return false;
1162 }
1163 printOperand(MI, OpNum, O);
1164 return false;
1165 case 'b': // Print B register.
1166 case 'h': // Print H register.
1167 case 's': // Print S register.
1168 case 'd': // Print D register.
1169 case 'q': // Print Q register.
1170 case 'z': // Print Z register.
1171 if (MO.isReg()) {
1172 const TargetRegisterClass *RC;
1173 switch (ExtraCode[0]) {
1174 case 'b':
1175 RC = &AArch64::FPR8RegClass;
1176 break;
1177 case 'h':
1178 RC = &AArch64::FPR16RegClass;
1179 break;
1180 case 's':
1181 RC = &AArch64::FPR32RegClass;
1182 break;
1183 case 'd':
1184 RC = &AArch64::FPR64RegClass;
1185 break;
1186 case 'q':
1187 RC = &AArch64::FPR128RegClass;
1188 break;
1189 case 'z':
1190 RC = &AArch64::ZPRRegClass;
1191 break;
1192 default:
1193 return true;
1194 }
1195 return printAsmRegInClass(MO, RC, AltName: AArch64::NoRegAltName, O);
1196 }
1197 printOperand(MI, OpNum, O);
1198 return false;
1199 }
1200 }
1201
1202 // According to ARM, we should emit x and v registers unless we have a
1203 // modifier.
1204 if (MO.isReg()) {
1205 Register Reg = MO.getReg();
1206
1207 // If this is a w or x register, print an x register.
1208 if (AArch64::GPR32allRegClass.contains(Reg) ||
1209 AArch64::GPR64allRegClass.contains(Reg))
1210 return printAsmMRegister(MO, Mode: 'x', O);
1211
1212 // If this is an x register tuple, print an x register.
1213 if (AArch64::GPR64x8ClassRegClass.contains(Reg))
1214 return printAsmMRegister(MO, Mode: 't', O);
1215
1216 unsigned AltName = AArch64::NoRegAltName;
1217 const TargetRegisterClass *RegClass;
1218 if (AArch64::ZPRRegClass.contains(Reg)) {
1219 RegClass = &AArch64::ZPRRegClass;
1220 } else if (AArch64::PPRRegClass.contains(Reg)) {
1221 RegClass = &AArch64::PPRRegClass;
1222 } else if (AArch64::PNRRegClass.contains(Reg)) {
1223 RegClass = &AArch64::PNRRegClass;
1224 } else {
1225 RegClass = &AArch64::FPR128RegClass;
1226 AltName = AArch64::vreg;
1227 }
1228
1229 // If this is a b, h, s, d, or q register, print it as a v register.
1230 return printAsmRegInClass(MO, RC: RegClass, AltName, O);
1231 }
1232
1233 printOperand(MI, OpNum, O);
1234 return false;
1235}
1236
1237bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
1238 unsigned OpNum,
1239 const char *ExtraCode,
1240 raw_ostream &O) {
1241 if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
1242 return true; // Unknown modifier.
1243
1244 const MachineOperand &MO = MI->getOperand(i: OpNum);
1245 assert(MO.isReg() && "unexpected inline asm memory operand");
1246 O << "[" << AArch64InstPrinter::getRegisterName(Reg: MO.getReg()) << "]";
1247 return false;
1248}
1249
1250void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
1251 raw_ostream &OS) {
1252 unsigned NOps = MI->getNumOperands();
1253 assert(NOps == 4);
1254 OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
1255 // cast away const; DIetc do not take const operands for some reason.
1256 OS << MI->getDebugVariable()->getName();
1257 OS << " <- ";
1258 // Frame address. Currently handles register +- offset only.
1259 assert(MI->isIndirectDebugValue());
1260 OS << '[';
1261 for (unsigned I = 0, E = std::distance(first: MI->debug_operands().begin(),
1262 last: MI->debug_operands().end());
1263 I < E; ++I) {
1264 if (I != 0)
1265 OS << ", ";
1266 printOperand(MI, OpNum: I, O&: OS);
1267 }
1268 OS << ']';
1269 OS << "+";
1270 printOperand(MI, OpNum: NOps - 2, O&: OS);
1271}
1272
1273void AArch64AsmPrinter::emitJumpTableImpl(const MachineJumpTableInfo &MJTI,
1274 ArrayRef<unsigned> JumpTableIndices) {
1275 // Fast return if there is nothing to emit to avoid creating empty sections.
1276 if (JumpTableIndices.empty())
1277 return;
1278 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
1279 const auto &F = MF->getFunction();
1280 ArrayRef<MachineJumpTableEntry> JT = MJTI.getJumpTables();
1281
1282 MCSection *ReadOnlySec = nullptr;
1283 if (TM.Options.EnableStaticDataPartitioning) {
1284 ReadOnlySec =
1285 TLOF.getSectionForJumpTable(F, TM, JTE: &JT[JumpTableIndices.front()]);
1286 } else {
1287 ReadOnlySec = TLOF.getSectionForJumpTable(F, TM);
1288 }
1289 OutStreamer->switchSection(Section: ReadOnlySec);
1290
1291 auto AFI = MF->getInfo<AArch64FunctionInfo>();
1292 for (unsigned JTI : JumpTableIndices) {
1293 const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
1294
1295 // If this jump table was deleted, ignore it.
1296 if (JTBBs.empty()) continue;
1297
1298 unsigned Size = AFI->getJumpTableEntrySize(Idx: JTI);
1299 emitAlignment(Alignment: Align(Size));
1300 OutStreamer->emitLabel(Symbol: GetJTISymbol(JTID: JTI));
1301
1302 const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(Idx: JTI);
1303 const MCExpr *Base = MCSymbolRefExpr::create(Symbol: BaseSym, Ctx&: OutContext);
1304
1305 for (auto *JTBB : JTBBs) {
1306 const MCExpr *Value =
1307 MCSymbolRefExpr::create(Symbol: JTBB->getSymbol(), Ctx&: OutContext);
1308
1309 // Each entry is:
1310 // .byte/.hword (LBB - Lbase)>>2
1311 // or plain:
1312 // .word LBB - Lbase
1313 Value = MCBinaryExpr::createSub(LHS: Value, RHS: Base, Ctx&: OutContext);
1314 if (Size != 4)
1315 Value = MCBinaryExpr::createLShr(
1316 LHS: Value, RHS: MCConstantExpr::create(Value: 2, Ctx&: OutContext), Ctx&: OutContext);
1317
1318 OutStreamer->emitValue(Value, Size);
1319 }
1320 }
1321}
1322
1323std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
1324 codeview::JumpTableEntrySize>
1325AArch64AsmPrinter::getCodeViewJumpTableInfo(int JTI,
1326 const MachineInstr *BranchInstr,
1327 const MCSymbol *BranchLabel) const {
1328 const auto AFI = MF->getInfo<AArch64FunctionInfo>();
1329 const auto Base = AArch64FI->getJumpTableEntryPCRelSymbol(Idx: JTI);
1330 codeview::JumpTableEntrySize EntrySize;
1331 switch (AFI->getJumpTableEntrySize(Idx: JTI)) {
1332 case 1:
1333 EntrySize = codeview::JumpTableEntrySize::UInt8ShiftLeft;
1334 break;
1335 case 2:
1336 EntrySize = codeview::JumpTableEntrySize::UInt16ShiftLeft;
1337 break;
1338 case 4:
1339 EntrySize = codeview::JumpTableEntrySize::Int32;
1340 break;
1341 default:
1342 llvm_unreachable("Unexpected jump table entry size");
1343 }
1344 return std::make_tuple(args: Base, args: 0, args&: BranchLabel, args&: EntrySize);
1345}
1346
1347void AArch64AsmPrinter::emitFunctionEntryLabel() {
1348 const Triple &TT = TM.getTargetTriple();
1349 if (TT.isOSBinFormatELF() &&
1350 (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall ||
1351 MF->getFunction().getCallingConv() ==
1352 CallingConv::AArch64_SVE_VectorCall ||
1353 MF->getInfo<AArch64FunctionInfo>()->isSVECC())) {
1354 auto *TS =
1355 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
1356 TS->emitDirectiveVariantPCS(Symbol: CurrentFnSym);
1357 }
1358
1359 AsmPrinter::emitFunctionEntryLabel();
1360
1361 if (TT.isWindowsArm64EC() && !MF->getFunction().hasLocalLinkage()) {
1362 // For ARM64EC targets, a function definition's name is mangled differently
1363 // from the normal symbol, emit required aliases here.
1364 auto emitFunctionAlias = [&](MCSymbol *Src, MCSymbol *Dst) {
1365 OutStreamer->emitSymbolAttribute(Symbol: Src, Attribute: MCSA_WeakAntiDep);
1366 OutStreamer->emitAssignment(
1367 Symbol: Src, Value: MCSymbolRefExpr::create(Symbol: Dst, Ctx&: MMI->getContext()));
1368 };
1369
1370 auto getSymbolFromMetadata = [&](StringRef Name) {
1371 MCSymbol *Sym = nullptr;
1372 if (MDNode *Node = MF->getFunction().getMetadata(Kind: Name)) {
1373 StringRef NameStr = cast<MDString>(Val: Node->getOperand(I: 0))->getString();
1374 Sym = MMI->getContext().getOrCreateSymbol(Name: NameStr);
1375 }
1376 return Sym;
1377 };
1378
1379 SmallVector<MDNode *> UnmangledNames;
1380 MF->getFunction().getMetadata(Kind: "arm64ec_unmangled_name", MDs&: UnmangledNames);
1381 for (MDNode *Node : UnmangledNames) {
1382 StringRef NameStr = cast<MDString>(Val: Node->getOperand(I: 0))->getString();
1383 MCSymbol *UnmangledSym = MMI->getContext().getOrCreateSymbol(Name: NameStr);
1384 if (std::optional<std::string> MangledName =
1385 getArm64ECMangledFunctionName(Name: UnmangledSym->getName())) {
1386 MCSymbol *ECMangledSym =
1387 MMI->getContext().getOrCreateSymbol(Name: *MangledName);
1388 emitFunctionAlias(UnmangledSym, ECMangledSym);
1389 }
1390 }
1391 if (MCSymbol *ECMangledSym =
1392 getSymbolFromMetadata("arm64ec_ecmangled_name"))
1393 emitFunctionAlias(ECMangledSym, CurrentFnSym);
1394 }
1395}
1396
1397void AArch64AsmPrinter::emitXXStructor(const DataLayout &DL,
1398 const Constant *CV) {
1399 if (const auto *CPA = dyn_cast<ConstantPtrAuth>(Val: CV))
1400 if (CPA->hasAddressDiscriminator() &&
1401 !CPA->hasSpecialAddressDiscriminator(
1402 Value: ConstantPtrAuth::AddrDiscriminator_CtorsDtors))
1403 report_fatal_error(
1404 reason: "unexpected address discrimination value for ctors/dtors entry, only "
1405 "'ptr inttoptr (i64 1 to ptr)' is allowed");
1406 // If we have signed pointers in xxstructors list, they'll be lowered to @AUTH
1407 // MCExpr's via AArch64AsmPrinter::lowerConstantPtrAuth. It does not look at
1408 // actual address discrimination value and only checks
1409 // hasAddressDiscriminator(), so it's OK to leave special address
1410 // discrimination value here.
1411 AsmPrinter::emitXXStructor(DL, CV);
1412}
1413
1414void AArch64AsmPrinter::emitGlobalAlias(const Module &M,
1415 const GlobalAlias &GA) {
1416 if (auto F = dyn_cast_or_null<Function>(Val: GA.getAliasee())) {
1417 // Global aliases must point to a definition, but unmangled patchable
1418 // symbols are special and need to point to an undefined symbol with "EXP+"
1419 // prefix. Such undefined symbol is resolved by the linker by creating
1420 // x86 thunk that jumps back to the actual EC target.
1421 if (MDNode *Node = F->getMetadata(Kind: "arm64ec_exp_name")) {
1422 StringRef ExpStr = cast<MDString>(Val: Node->getOperand(I: 0))->getString();
1423 MCSymbol *ExpSym = MMI->getContext().getOrCreateSymbol(Name: ExpStr);
1424 MCSymbol *Sym = MMI->getContext().getOrCreateSymbol(Name: GA.getName());
1425
1426 OutStreamer->beginCOFFSymbolDef(Symbol: ExpSym);
1427 OutStreamer->emitCOFFSymbolStorageClass(StorageClass: COFF::IMAGE_SYM_CLASS_EXTERNAL);
1428 OutStreamer->emitCOFFSymbolType(Type: COFF::IMAGE_SYM_DTYPE_FUNCTION
1429 << COFF::SCT_COMPLEX_TYPE_SHIFT);
1430 OutStreamer->endCOFFSymbolDef();
1431
1432 OutStreamer->beginCOFFSymbolDef(Symbol: Sym);
1433 OutStreamer->emitCOFFSymbolStorageClass(StorageClass: COFF::IMAGE_SYM_CLASS_EXTERNAL);
1434 OutStreamer->emitCOFFSymbolType(Type: COFF::IMAGE_SYM_DTYPE_FUNCTION
1435 << COFF::SCT_COMPLEX_TYPE_SHIFT);
1436 OutStreamer->endCOFFSymbolDef();
1437 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Weak);
1438 OutStreamer->emitAssignment(
1439 Symbol: Sym, Value: MCSymbolRefExpr::create(Symbol: ExpSym, Ctx&: MMI->getContext()));
1440 return;
1441 }
1442 }
1443 AsmPrinter::emitGlobalAlias(M, GA);
1444}
1445
1446/// Small jump tables contain an unsigned byte or half, representing the offset
1447/// from the lowest-addressed possible destination to the desired basic
1448/// block. Since all instructions are 4-byte aligned, this is further compressed
1449/// by counting in instructions rather than bytes (i.e. divided by 4). So, to
1450/// materialize the correct destination we need:
1451///
1452/// adr xDest, .LBB0_0
1453/// ldrb wScratch, [xTable, xEntry] (with "lsl #1" for ldrh).
1454/// add xDest, xDest, xScratch (with "lsl #2" for smaller entries)
1455void AArch64AsmPrinter::LowerJumpTableDest(llvm::MCStreamer &OutStreamer,
1456 const llvm::MachineInstr &MI) {
1457 Register DestReg = MI.getOperand(i: 0).getReg();
1458 Register ScratchReg = MI.getOperand(i: 1).getReg();
1459 Register ScratchRegW =
1460 STI->getRegisterInfo()->getSubReg(Reg: ScratchReg, Idx: AArch64::sub_32);
1461 Register TableReg = MI.getOperand(i: 2).getReg();
1462 Register EntryReg = MI.getOperand(i: 3).getReg();
1463 int JTIdx = MI.getOperand(i: 4).getIndex();
1464 int Size = AArch64FI->getJumpTableEntrySize(Idx: JTIdx);
1465
1466 // This has to be first because the compression pass based its reachability
1467 // calculations on the start of the JumpTableDest instruction.
1468 auto Label =
1469 MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(Idx: JTIdx);
1470
1471 // If we don't already have a symbol to use as the base, use the ADR
1472 // instruction itself.
1473 if (!Label) {
1474 Label = MF->getContext().createTempSymbol();
1475 AArch64FI->setJumpTableEntryInfo(Idx: JTIdx, Size, PCRelSym: Label);
1476 OutStreamer.emitLabel(Symbol: Label);
1477 }
1478
1479 auto LabelExpr = MCSymbolRefExpr::create(Symbol: Label, Ctx&: MF->getContext());
1480 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::ADR)
1481 .addReg(Reg: DestReg)
1482 .addExpr(Val: LabelExpr));
1483
1484 // Load the number of instruction-steps to offset from the label.
1485 unsigned LdrOpcode;
1486 switch (Size) {
1487 case 1: LdrOpcode = AArch64::LDRBBroX; break;
1488 case 2: LdrOpcode = AArch64::LDRHHroX; break;
1489 case 4: LdrOpcode = AArch64::LDRSWroX; break;
1490 default:
1491 llvm_unreachable("Unknown jump table size");
1492 }
1493
1494 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(LdrOpcode)
1495 .addReg(Reg: Size == 4 ? ScratchReg : ScratchRegW)
1496 .addReg(Reg: TableReg)
1497 .addReg(Reg: EntryReg)
1498 .addImm(Val: 0)
1499 .addImm(Val: Size == 1 ? 0 : 1));
1500
1501 // Add to the already materialized base label address, multiplying by 4 if
1502 // compressed.
1503 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::ADDXrs)
1504 .addReg(Reg: DestReg)
1505 .addReg(Reg: DestReg)
1506 .addReg(Reg: ScratchReg)
1507 .addImm(Val: Size == 4 ? 0 : 2));
1508}
1509
1510void AArch64AsmPrinter::LowerHardenedBRJumpTable(const MachineInstr &MI) {
1511 const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
1512 assert(MJTI && "Can't lower jump-table dispatch without JTI");
1513
1514 const std::vector<MachineJumpTableEntry> &JTs = MJTI->getJumpTables();
1515 assert(!JTs.empty() && "Invalid JT index for jump-table dispatch");
1516
1517 // Emit:
1518 // mov x17, #<size of table> ; depending on table size, with MOVKs
1519 // cmp x16, x17 ; or #imm if table size fits in 12-bit
1520 // csel x16, x16, xzr, ls ; check for index overflow
1521 //
1522 // adrp x17, Ltable@PAGE ; materialize table address
1523 // add x17, Ltable@PAGEOFF
1524 // ldrsw x16, [x17, x16, lsl #2] ; load table entry
1525 //
1526 // Lanchor:
1527 // adr x17, Lanchor ; compute target address
1528 // add x16, x17, x16
1529 // br x16 ; branch to target
1530
1531 MachineOperand JTOp = MI.getOperand(i: 0);
1532
1533 unsigned JTI = JTOp.getIndex();
1534 assert(!AArch64FI->getJumpTableEntryPCRelSymbol(JTI) &&
1535 "unsupported compressed jump table");
1536
1537 const uint64_t NumTableEntries = JTs[JTI].MBBs.size();
1538
1539 // cmp only supports a 12-bit immediate. If we need more, materialize the
1540 // immediate, using x17 as a scratch register.
1541 uint64_t MaxTableEntry = NumTableEntries - 1;
1542 if (isUInt<12>(x: MaxTableEntry)) {
1543 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::SUBSXri)
1544 .addReg(Reg: AArch64::XZR)
1545 .addReg(Reg: AArch64::X16)
1546 .addImm(Val: MaxTableEntry)
1547 .addImm(Val: 0));
1548 } else {
1549 emitMOVZ(Dest: AArch64::X17, Imm: static_cast<uint16_t>(MaxTableEntry), Shift: 0);
1550 // It's sad that we have to manually materialize instructions, but we can't
1551 // trivially reuse the main pseudo expansion logic.
1552 // A MOVK sequence is easy enough to generate and handles the general case.
1553 for (int Offset = 16; Offset < 64; Offset += 16) {
1554 if ((MaxTableEntry >> Offset) == 0)
1555 break;
1556 emitMOVK(Dest: AArch64::X17, Imm: static_cast<uint16_t>(MaxTableEntry >> Offset),
1557 Shift: Offset);
1558 }
1559 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::SUBSXrs)
1560 .addReg(Reg: AArch64::XZR)
1561 .addReg(Reg: AArch64::X16)
1562 .addReg(Reg: AArch64::X17)
1563 .addImm(Val: 0));
1564 }
1565
1566 // This picks entry #0 on failure.
1567 // We might want to trap instead.
1568 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::CSELXr)
1569 .addReg(Reg: AArch64::X16)
1570 .addReg(Reg: AArch64::X16)
1571 .addReg(Reg: AArch64::XZR)
1572 .addImm(Val: AArch64CC::LS));
1573
1574 // Prepare the @PAGE/@PAGEOFF low/high operands.
1575 MachineOperand JTMOHi(JTOp), JTMOLo(JTOp);
1576 MCOperand JTMCHi, JTMCLo;
1577
1578 JTMOHi.setTargetFlags(AArch64II::MO_PAGE);
1579 JTMOLo.setTargetFlags(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
1580
1581 MCInstLowering.lowerOperand(MO: JTMOHi, MCOp&: JTMCHi);
1582 MCInstLowering.lowerOperand(MO: JTMOLo, MCOp&: JTMCLo);
1583
1584 EmitToStreamer(
1585 S&: *OutStreamer,
1586 Inst: MCInstBuilder(AArch64::ADRP).addReg(Reg: AArch64::X17).addOperand(Op: JTMCHi));
1587
1588 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ADDXri)
1589 .addReg(Reg: AArch64::X17)
1590 .addReg(Reg: AArch64::X17)
1591 .addOperand(Op: JTMCLo)
1592 .addImm(Val: 0));
1593
1594 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::LDRSWroX)
1595 .addReg(Reg: AArch64::X16)
1596 .addReg(Reg: AArch64::X17)
1597 .addReg(Reg: AArch64::X16)
1598 .addImm(Val: 0)
1599 .addImm(Val: 1));
1600
1601 MCSymbol *AdrLabel = MF->getContext().createTempSymbol();
1602 const auto *AdrLabelE = MCSymbolRefExpr::create(Symbol: AdrLabel, Ctx&: MF->getContext());
1603 AArch64FI->setJumpTableEntryInfo(Idx: JTI, Size: 4, PCRelSym: AdrLabel);
1604
1605 OutStreamer->emitLabel(Symbol: AdrLabel);
1606 EmitToStreamer(
1607 S&: *OutStreamer,
1608 Inst: MCInstBuilder(AArch64::ADR).addReg(Reg: AArch64::X17).addExpr(Val: AdrLabelE));
1609
1610 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ADDXrs)
1611 .addReg(Reg: AArch64::X16)
1612 .addReg(Reg: AArch64::X17)
1613 .addReg(Reg: AArch64::X16)
1614 .addImm(Val: 0));
1615
1616 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::BR).addReg(Reg: AArch64::X16));
1617}
1618
1619void AArch64AsmPrinter::LowerMOPS(llvm::MCStreamer &OutStreamer,
1620 const llvm::MachineInstr &MI) {
1621 unsigned Opcode = MI.getOpcode();
1622 assert(STI->hasMOPS());
1623 assert(STI->hasMTE() || Opcode != AArch64::MOPSMemorySetTaggingPseudo);
1624
1625 const auto Ops = [Opcode]() -> std::array<unsigned, 3> {
1626 if (Opcode == AArch64::MOPSMemoryCopyPseudo)
1627 return {AArch64::CPYFP, AArch64::CPYFM, AArch64::CPYFE};
1628 if (Opcode == AArch64::MOPSMemoryMovePseudo)
1629 return {AArch64::CPYP, AArch64::CPYM, AArch64::CPYE};
1630 if (Opcode == AArch64::MOPSMemorySetPseudo)
1631 return {AArch64::SETP, AArch64::SETM, AArch64::SETE};
1632 if (Opcode == AArch64::MOPSMemorySetTaggingPseudo)
1633 return {AArch64::SETGP, AArch64::SETGM, AArch64::MOPSSETGE};
1634 llvm_unreachable("Unhandled memory operation pseudo");
1635 }();
1636 const bool IsSet = Opcode == AArch64::MOPSMemorySetPseudo ||
1637 Opcode == AArch64::MOPSMemorySetTaggingPseudo;
1638
1639 for (auto Op : Ops) {
1640 int i = 0;
1641 auto MCIB = MCInstBuilder(Op);
1642 // Destination registers
1643 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1644 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1645 if (!IsSet)
1646 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1647 // Input registers
1648 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1649 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1650 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1651
1652 EmitToStreamer(S&: OutStreamer, Inst: MCIB);
1653 }
1654}
1655
1656void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
1657 const MachineInstr &MI) {
1658 unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
1659
1660 auto &Ctx = OutStreamer.getContext();
1661 MCSymbol *MILabel = Ctx.createTempSymbol();
1662 OutStreamer.emitLabel(Symbol: MILabel);
1663
1664 SM.recordStackMap(L: *MILabel, MI);
1665 assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1666
1667 // Scan ahead to trim the shadow.
1668 const MachineBasicBlock &MBB = *MI.getParent();
1669 MachineBasicBlock::const_iterator MII(MI);
1670 ++MII;
1671 while (NumNOPBytes > 0) {
1672 if (MII == MBB.end() || MII->isCall() ||
1673 MII->getOpcode() == AArch64::DBG_VALUE ||
1674 MII->getOpcode() == TargetOpcode::PATCHPOINT ||
1675 MII->getOpcode() == TargetOpcode::STACKMAP)
1676 break;
1677 ++MII;
1678 NumNOPBytes -= 4;
1679 }
1680
1681 // Emit nops.
1682 for (unsigned i = 0; i < NumNOPBytes; i += 4)
1683 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::HINT).addImm(Val: 0));
1684}
1685
1686// Lower a patchpoint of the form:
1687// [<def>], <id>, <numBytes>, <target>, <numArgs>
1688void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1689 const MachineInstr &MI) {
1690 auto &Ctx = OutStreamer.getContext();
1691 MCSymbol *MILabel = Ctx.createTempSymbol();
1692 OutStreamer.emitLabel(Symbol: MILabel);
1693 SM.recordPatchPoint(L: *MILabel, MI);
1694
1695 PatchPointOpers Opers(&MI);
1696
1697 int64_t CallTarget = Opers.getCallTarget().getImm();
1698 unsigned EncodedBytes = 0;
1699 if (CallTarget) {
1700 assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
1701 "High 16 bits of call target should be zero.");
1702 Register ScratchReg = MI.getOperand(i: Opers.getNextScratchIdx()).getReg();
1703 EncodedBytes = 16;
1704 // Materialize the jump address:
1705 emitMOVZ(Dest: ScratchReg, Imm: (CallTarget >> 32) & 0xFFFF, Shift: 32);
1706 emitMOVK(Dest: ScratchReg, Imm: (CallTarget >> 16) & 0xFFFF, Shift: 16);
1707 emitMOVK(Dest: ScratchReg, Imm: CallTarget & 0xFFFF, Shift: 0);
1708 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::BLR).addReg(Reg: ScratchReg));
1709 }
1710 // Emit padding.
1711 unsigned NumBytes = Opers.getNumPatchBytes();
1712 assert(NumBytes >= EncodedBytes &&
1713 "Patchpoint can't request size less than the length of a call.");
1714 assert((NumBytes - EncodedBytes) % 4 == 0 &&
1715 "Invalid number of NOP bytes requested!");
1716 for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
1717 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::HINT).addImm(Val: 0));
1718}
1719
1720void AArch64AsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1721 const MachineInstr &MI) {
1722 StatepointOpers SOpers(&MI);
1723 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
1724 assert(PatchBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1725 for (unsigned i = 0; i < PatchBytes; i += 4)
1726 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::HINT).addImm(Val: 0));
1727 } else {
1728 // Lower call target and choose correct opcode
1729 const MachineOperand &CallTarget = SOpers.getCallTarget();
1730 MCOperand CallTargetMCOp;
1731 unsigned CallOpcode;
1732 switch (CallTarget.getType()) {
1733 case MachineOperand::MO_GlobalAddress:
1734 case MachineOperand::MO_ExternalSymbol:
1735 MCInstLowering.lowerOperand(MO: CallTarget, MCOp&: CallTargetMCOp);
1736 CallOpcode = AArch64::BL;
1737 break;
1738 case MachineOperand::MO_Immediate:
1739 CallTargetMCOp = MCOperand::createImm(Val: CallTarget.getImm());
1740 CallOpcode = AArch64::BL;
1741 break;
1742 case MachineOperand::MO_Register:
1743 CallTargetMCOp = MCOperand::createReg(Reg: CallTarget.getReg());
1744 CallOpcode = AArch64::BLR;
1745 break;
1746 default:
1747 llvm_unreachable("Unsupported operand type in statepoint call target");
1748 break;
1749 }
1750
1751 EmitToStreamer(S&: OutStreamer,
1752 Inst: MCInstBuilder(CallOpcode).addOperand(Op: CallTargetMCOp));
1753 }
1754
1755 auto &Ctx = OutStreamer.getContext();
1756 MCSymbol *MILabel = Ctx.createTempSymbol();
1757 OutStreamer.emitLabel(Symbol: MILabel);
1758 SM.recordStatepoint(L: *MILabel, MI);
1759}
1760
1761void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) {
1762 // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
1763 // <opcode>, <operands>
1764
1765 Register DefRegister = FaultingMI.getOperand(i: 0).getReg();
1766 FaultMaps::FaultKind FK =
1767 static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(i: 1).getImm());
1768 MCSymbol *HandlerLabel = FaultingMI.getOperand(i: 2).getMBB()->getSymbol();
1769 unsigned Opcode = FaultingMI.getOperand(i: 3).getImm();
1770 unsigned OperandsBeginIdx = 4;
1771
1772 auto &Ctx = OutStreamer->getContext();
1773 MCSymbol *FaultingLabel = Ctx.createTempSymbol();
1774 OutStreamer->emitLabel(Symbol: FaultingLabel);
1775
1776 assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
1777 FM.recordFaultingOp(FaultTy: FK, FaultingLabel, HandlerLabel);
1778
1779 MCInst MI;
1780 MI.setOpcode(Opcode);
1781
1782 if (DefRegister != (Register)0)
1783 MI.addOperand(Op: MCOperand::createReg(Reg: DefRegister));
1784
1785 for (const MachineOperand &MO :
1786 llvm::drop_begin(RangeOrContainer: FaultingMI.operands(), N: OperandsBeginIdx)) {
1787 MCOperand Dest;
1788 lowerOperand(MO, MCOp&: Dest);
1789 MI.addOperand(Op: Dest);
1790 }
1791
1792 OutStreamer->AddComment(T: "on-fault: " + HandlerLabel->getName());
1793 EmitToStreamer(Inst: MI);
1794}
1795
1796void AArch64AsmPrinter::emitMovXReg(Register Dest, Register Src) {
1797 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ORRXrs)
1798 .addReg(Reg: Dest)
1799 .addReg(Reg: AArch64::XZR)
1800 .addReg(Reg: Src)
1801 .addImm(Val: 0));
1802}
1803
1804void AArch64AsmPrinter::emitMOVZ(Register Dest, uint64_t Imm, unsigned Shift) {
1805 bool Is64Bit = AArch64::GPR64RegClass.contains(Reg: Dest);
1806 EmitToStreamer(S&: *OutStreamer,
1807 Inst: MCInstBuilder(Is64Bit ? AArch64::MOVZXi : AArch64::MOVZWi)
1808 .addReg(Reg: Dest)
1809 .addImm(Val: Imm)
1810 .addImm(Val: Shift));
1811}
1812
1813void AArch64AsmPrinter::emitMOVK(Register Dest, uint64_t Imm, unsigned Shift) {
1814 bool Is64Bit = AArch64::GPR64RegClass.contains(Reg: Dest);
1815 EmitToStreamer(S&: *OutStreamer,
1816 Inst: MCInstBuilder(Is64Bit ? AArch64::MOVKXi : AArch64::MOVKWi)
1817 .addReg(Reg: Dest)
1818 .addReg(Reg: Dest)
1819 .addImm(Val: Imm)
1820 .addImm(Val: Shift));
1821}
1822
1823void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) {
1824 Register DestReg = MI.getOperand(i: 0).getReg();
1825 if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround() &&
1826 STI->isNeonAvailable()) {
1827 // Convert H/S register to corresponding D register
1828 if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31)
1829 DestReg = AArch64::D0 + (DestReg - AArch64::H0);
1830 else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31)
1831 DestReg = AArch64::D0 + (DestReg - AArch64::S0);
1832 else
1833 assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
1834
1835 MCInst MOVI;
1836 MOVI.setOpcode(AArch64::MOVID);
1837 MOVI.addOperand(Op: MCOperand::createReg(Reg: DestReg));
1838 MOVI.addOperand(Op: MCOperand::createImm(Val: 0));
1839 EmitToStreamer(S&: *OutStreamer, Inst: MOVI);
1840 } else {
1841 MCInst FMov;
1842 switch (MI.getOpcode()) {
1843 default: llvm_unreachable("Unexpected opcode");
1844 case AArch64::FMOVH0:
1845 FMov.setOpcode(STI->hasFullFP16() ? AArch64::FMOVWHr : AArch64::FMOVWSr);
1846 if (!STI->hasFullFP16())
1847 DestReg = (AArch64::S0 + (DestReg - AArch64::H0));
1848 FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg));
1849 FMov.addOperand(Op: MCOperand::createReg(Reg: AArch64::WZR));
1850 break;
1851 case AArch64::FMOVS0:
1852 FMov.setOpcode(AArch64::FMOVWSr);
1853 FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg));
1854 FMov.addOperand(Op: MCOperand::createReg(Reg: AArch64::WZR));
1855 break;
1856 case AArch64::FMOVD0:
1857 FMov.setOpcode(AArch64::FMOVXDr);
1858 FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg));
1859 FMov.addOperand(Op: MCOperand::createReg(Reg: AArch64::XZR));
1860 break;
1861 }
1862 EmitToStreamer(S&: *OutStreamer, Inst: FMov);
1863 }
1864}
1865
1866Register AArch64AsmPrinter::emitPtrauthDiscriminator(uint16_t Disc,
1867 Register AddrDisc,
1868 Register ScratchReg,
1869 bool MayUseAddrAsScratch) {
1870 assert(ScratchReg == AArch64::X16 || ScratchReg == AArch64::X17);
1871 // So far we've used NoRegister in pseudos. Now we need real encodings.
1872 if (AddrDisc == AArch64::NoRegister)
1873 AddrDisc = AArch64::XZR;
1874
1875 // If there is no constant discriminator, there's no blend involved:
1876 // just use the address discriminator register as-is (XZR or not).
1877 if (!Disc)
1878 return AddrDisc;
1879
1880 // If there's only a constant discriminator, MOV it into the scratch register.
1881 if (AddrDisc == AArch64::XZR) {
1882 emitMOVZ(Dest: ScratchReg, Imm: Disc, Shift: 0);
1883 return ScratchReg;
1884 }
1885
1886 // If there are both, emit a blend into the scratch register.
1887
1888 // Check if we can save one MOV instruction.
1889 assert(MayUseAddrAsScratch || ScratchReg != AddrDisc);
1890 bool AddrDiscIsSafe = AddrDisc == AArch64::X16 || AddrDisc == AArch64::X17;
1891 if (MayUseAddrAsScratch && AddrDiscIsSafe)
1892 ScratchReg = AddrDisc;
1893 else
1894 emitMovXReg(Dest: ScratchReg, Src: AddrDisc);
1895
1896 emitMOVK(Dest: ScratchReg, Imm: Disc, Shift: 48);
1897 return ScratchReg;
1898}
1899
1900/// Emits a code sequence to check an authenticated pointer value.
1901///
1902/// If OnFailure argument is passed, jump there on check failure instead
1903/// of proceeding to the next instruction (only if ShouldTrap is false).
1904void AArch64AsmPrinter::emitPtrauthCheckAuthenticatedValue(
1905 Register TestedReg, Register ScratchReg, AArch64PACKey::ID Key,
1906 AArch64PAuth::AuthCheckMethod Method, bool ShouldTrap,
1907 const MCSymbol *OnFailure) {
1908 // Insert a sequence to check if authentication of TestedReg succeeded,
1909 // such as:
1910 //
1911 // - checked and clearing:
1912 // ; x16 is TestedReg, x17 is ScratchReg
1913 // mov x17, x16
1914 // xpaci x17
1915 // cmp x16, x17
1916 // b.eq Lsuccess
1917 // mov x16, x17
1918 // b Lend
1919 // Lsuccess:
1920 // ; skipped if authentication failed
1921 // Lend:
1922 // ...
1923 //
1924 // - checked and trapping:
1925 // mov x17, x16
1926 // xpaci x17
1927 // cmp x16, x17
1928 // b.eq Lsuccess
1929 // brk #<0xc470 + aut key>
1930 // Lsuccess:
1931 // ...
1932 //
1933 // See the documentation on AuthCheckMethod enumeration constants for
1934 // the specific code sequences that can be used to perform the check.
1935 using AArch64PAuth::AuthCheckMethod;
1936
1937 if (Method == AuthCheckMethod::None)
1938 return;
1939 if (Method == AuthCheckMethod::DummyLoad) {
1940 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRWui)
1941 .addReg(Reg: getWRegFromXReg(Reg: ScratchReg))
1942 .addReg(Reg: TestedReg)
1943 .addImm(Val: 0));
1944 assert(ShouldTrap && !OnFailure && "DummyLoad always traps on error");
1945 return;
1946 }
1947
1948 MCSymbol *SuccessSym = createTempSymbol(Name: "auth_success_");
1949 if (Method == AuthCheckMethod::XPAC || Method == AuthCheckMethod::XPACHint) {
1950 // mov Xscratch, Xtested
1951 emitMovXReg(Dest: ScratchReg, Src: TestedReg);
1952
1953 if (Method == AuthCheckMethod::XPAC) {
1954 // xpac(i|d) Xscratch
1955 unsigned XPACOpc = getXPACOpcodeForKey(K: Key);
1956 EmitToStreamer(
1957 Inst: MCInstBuilder(XPACOpc).addReg(Reg: ScratchReg).addReg(Reg: ScratchReg));
1958 } else {
1959 // xpaclri
1960
1961 // Note that this method applies XPAC to TestedReg instead of ScratchReg.
1962 assert(TestedReg == AArch64::LR &&
1963 "XPACHint mode is only compatible with checking the LR register");
1964 assert((Key == AArch64PACKey::IA || Key == AArch64PACKey::IB) &&
1965 "XPACHint mode is only compatible with I-keys");
1966 EmitToStreamer(Inst: MCInstBuilder(AArch64::XPACLRI));
1967 }
1968
1969 // cmp Xtested, Xscratch
1970 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSXrs)
1971 .addReg(Reg: AArch64::XZR)
1972 .addReg(Reg: TestedReg)
1973 .addReg(Reg: ScratchReg)
1974 .addImm(Val: 0));
1975
1976 // b.eq Lsuccess
1977 EmitToStreamer(
1978 Inst: MCInstBuilder(AArch64::Bcc)
1979 .addImm(Val: AArch64CC::EQ)
1980 .addExpr(Val: MCSymbolRefExpr::create(Symbol: SuccessSym, Ctx&: OutContext)));
1981 } else if (Method == AuthCheckMethod::HighBitsNoTBI) {
1982 // eor Xscratch, Xtested, Xtested, lsl #1
1983 EmitToStreamer(Inst: MCInstBuilder(AArch64::EORXrs)
1984 .addReg(Reg: ScratchReg)
1985 .addReg(Reg: TestedReg)
1986 .addReg(Reg: TestedReg)
1987 .addImm(Val: 1));
1988 // tbz Xscratch, #62, Lsuccess
1989 EmitToStreamer(
1990 Inst: MCInstBuilder(AArch64::TBZX)
1991 .addReg(Reg: ScratchReg)
1992 .addImm(Val: 62)
1993 .addExpr(Val: MCSymbolRefExpr::create(Symbol: SuccessSym, Ctx&: OutContext)));
1994 } else {
1995 llvm_unreachable("Unsupported check method");
1996 }
1997
1998 if (ShouldTrap) {
1999 assert(!OnFailure && "Cannot specify OnFailure with ShouldTrap");
2000 // Trapping sequences do a 'brk'.
2001 // brk #<0xc470 + aut key>
2002 EmitToStreamer(Inst: MCInstBuilder(AArch64::BRK).addImm(Val: 0xc470 | Key));
2003 } else {
2004 // Non-trapping checked sequences return the stripped result in TestedReg,
2005 // skipping over success-only code (such as re-signing the pointer) if
2006 // there is one.
2007 // Note that this can introduce an authentication oracle (such as based on
2008 // the high bits of the re-signed value).
2009
2010 // FIXME: The XPAC method can be optimized by applying XPAC to TestedReg
2011 // instead of ScratchReg, thus eliminating one `mov` instruction.
2012 // Both XPAC and XPACHint can be further optimized by not using a
2013 // conditional branch jumping over an unconditional one.
2014
2015 switch (Method) {
2016 case AuthCheckMethod::XPACHint:
2017 // LR is already XPAC-ed at this point.
2018 break;
2019 case AuthCheckMethod::XPAC:
2020 // mov Xtested, Xscratch
2021 emitMovXReg(Dest: TestedReg, Src: ScratchReg);
2022 break;
2023 default:
2024 // If Xtested was not XPAC-ed so far, emit XPAC here.
2025 // xpac(i|d) Xtested
2026 unsigned XPACOpc = getXPACOpcodeForKey(K: Key);
2027 EmitToStreamer(
2028 Inst: MCInstBuilder(XPACOpc).addReg(Reg: TestedReg).addReg(Reg: TestedReg));
2029 }
2030
2031 if (OnFailure) {
2032 // b Lend
2033 EmitToStreamer(
2034 Inst: MCInstBuilder(AArch64::B)
2035 .addExpr(Val: MCSymbolRefExpr::create(Symbol: OnFailure, Ctx&: OutContext)));
2036 }
2037 }
2038
2039 // If the auth check succeeds, we can continue.
2040 // Lsuccess:
2041 OutStreamer->emitLabel(Symbol: SuccessSym);
2042}
2043
2044// With Pointer Authentication, it may be needed to explicitly check the
2045// authenticated value in LR before performing a tail call.
2046// Otherwise, the callee may re-sign the invalid return address,
2047// introducing a signing oracle.
2048void AArch64AsmPrinter::emitPtrauthTailCallHardening(const MachineInstr *TC) {
2049 if (!AArch64FI->shouldSignReturnAddress(MF: *MF))
2050 return;
2051
2052 auto LRCheckMethod = STI->getAuthenticatedLRCheckMethod(MF: *MF);
2053 if (LRCheckMethod == AArch64PAuth::AuthCheckMethod::None)
2054 return;
2055
2056 const AArch64RegisterInfo *TRI = STI->getRegisterInfo();
2057 Register ScratchReg =
2058 TC->readsRegister(Reg: AArch64::X16, TRI) ? AArch64::X17 : AArch64::X16;
2059 assert(!TC->readsRegister(ScratchReg, TRI) &&
2060 "Neither x16 nor x17 is available as a scratch register");
2061 AArch64PACKey::ID Key =
2062 AArch64FI->shouldSignWithBKey() ? AArch64PACKey::IB : AArch64PACKey::IA;
2063 emitPtrauthCheckAuthenticatedValue(
2064 TestedReg: AArch64::LR, ScratchReg, Key, Method: LRCheckMethod,
2065 /*ShouldTrap=*/true, /*OnFailure=*/nullptr);
2066}
2067
2068void AArch64AsmPrinter::emitPtrauthAuthResign(const MachineInstr *MI) {
2069 const bool IsAUTPAC = MI->getOpcode() == AArch64::AUTPAC;
2070
2071 // We expand AUT/AUTPAC into a sequence of the form
2072 //
2073 // ; authenticate x16
2074 // ; check pointer in x16
2075 // Lsuccess:
2076 // ; sign x16 (if AUTPAC)
2077 // Lend: ; if not trapping on failure
2078 //
2079 // with the checking sequence chosen depending on whether/how we should check
2080 // the pointer and whether we should trap on failure.
2081
2082 // By default, auth/resign sequences check for auth failures.
2083 bool ShouldCheck = true;
2084 // In the checked sequence, we only trap if explicitly requested.
2085 bool ShouldTrap = MF->getFunction().hasFnAttribute(Kind: "ptrauth-auth-traps");
2086
2087 // On an FPAC CPU, you get traps whether you want them or not: there's
2088 // no point in emitting checks or traps.
2089 if (STI->hasFPAC())
2090 ShouldCheck = ShouldTrap = false;
2091
2092 // However, command-line flags can override this, for experimentation.
2093 switch (PtrauthAuthChecks) {
2094 case PtrauthCheckMode::Default:
2095 break;
2096 case PtrauthCheckMode::Unchecked:
2097 ShouldCheck = ShouldTrap = false;
2098 break;
2099 case PtrauthCheckMode::Poison:
2100 ShouldCheck = true;
2101 ShouldTrap = false;
2102 break;
2103 case PtrauthCheckMode::Trap:
2104 ShouldCheck = ShouldTrap = true;
2105 break;
2106 }
2107
2108 auto AUTKey = (AArch64PACKey::ID)MI->getOperand(i: 0).getImm();
2109 uint64_t AUTDisc = MI->getOperand(i: 1).getImm();
2110 unsigned AUTAddrDisc = MI->getOperand(i: 2).getReg();
2111
2112 // Compute aut discriminator into x17
2113 assert(isUInt<16>(AUTDisc));
2114 Register AUTDiscReg =
2115 emitPtrauthDiscriminator(Disc: AUTDisc, AddrDisc: AUTAddrDisc, ScratchReg: AArch64::X17);
2116 bool AUTZero = AUTDiscReg == AArch64::XZR;
2117 unsigned AUTOpc = getAUTOpcodeForKey(K: AUTKey, Zero: AUTZero);
2118
2119 // autiza x16 ; if AUTZero
2120 // autia x16, x17 ; if !AUTZero
2121 MCInst AUTInst;
2122 AUTInst.setOpcode(AUTOpc);
2123 AUTInst.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
2124 AUTInst.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
2125 if (!AUTZero)
2126 AUTInst.addOperand(Op: MCOperand::createReg(Reg: AUTDiscReg));
2127 EmitToStreamer(S&: *OutStreamer, Inst: AUTInst);
2128
2129 // Unchecked or checked-but-non-trapping AUT is just an "AUT": we're done.
2130 if (!IsAUTPAC && (!ShouldCheck || !ShouldTrap))
2131 return;
2132
2133 MCSymbol *EndSym = nullptr;
2134
2135 if (ShouldCheck) {
2136 if (IsAUTPAC && !ShouldTrap)
2137 EndSym = createTempSymbol(Name: "resign_end_");
2138
2139 emitPtrauthCheckAuthenticatedValue(TestedReg: AArch64::X16, ScratchReg: AArch64::X17, Key: AUTKey,
2140 Method: AArch64PAuth::AuthCheckMethod::XPAC,
2141 ShouldTrap, OnFailure: EndSym);
2142 }
2143
2144 // We already emitted unchecked and checked-but-non-trapping AUTs.
2145 // That left us with trapping AUTs, and AUTPACs.
2146 // Trapping AUTs don't need PAC: we're done.
2147 if (!IsAUTPAC)
2148 return;
2149
2150 auto PACKey = (AArch64PACKey::ID)MI->getOperand(i: 3).getImm();
2151 uint64_t PACDisc = MI->getOperand(i: 4).getImm();
2152 unsigned PACAddrDisc = MI->getOperand(i: 5).getReg();
2153
2154 // Compute pac discriminator into x17
2155 assert(isUInt<16>(PACDisc));
2156 Register PACDiscReg =
2157 emitPtrauthDiscriminator(Disc: PACDisc, AddrDisc: PACAddrDisc, ScratchReg: AArch64::X17);
2158 bool PACZero = PACDiscReg == AArch64::XZR;
2159 unsigned PACOpc = getPACOpcodeForKey(K: PACKey, Zero: PACZero);
2160
2161 // pacizb x16 ; if PACZero
2162 // pacib x16, x17 ; if !PACZero
2163 MCInst PACInst;
2164 PACInst.setOpcode(PACOpc);
2165 PACInst.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
2166 PACInst.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
2167 if (!PACZero)
2168 PACInst.addOperand(Op: MCOperand::createReg(Reg: PACDiscReg));
2169 EmitToStreamer(S&: *OutStreamer, Inst: PACInst);
2170
2171 // Lend:
2172 if (EndSym)
2173 OutStreamer->emitLabel(Symbol: EndSym);
2174}
2175
2176void AArch64AsmPrinter::emitPtrauthBranch(const MachineInstr *MI) {
2177 bool IsCall = MI->getOpcode() == AArch64::BLRA;
2178 unsigned BrTarget = MI->getOperand(i: 0).getReg();
2179
2180 auto Key = (AArch64PACKey::ID)MI->getOperand(i: 1).getImm();
2181 assert((Key == AArch64PACKey::IA || Key == AArch64PACKey::IB) &&
2182 "Invalid auth call key");
2183
2184 uint64_t Disc = MI->getOperand(i: 2).getImm();
2185 assert(isUInt<16>(Disc));
2186
2187 unsigned AddrDisc = MI->getOperand(i: 3).getReg();
2188
2189 // Make sure AddrDisc is solely used to compute the discriminator.
2190 // While hardly meaningful, it is still possible to describe an authentication
2191 // of a pointer against its own value (instead of storage address) with
2192 // intrinsics, so use report_fatal_error instead of assert.
2193 if (BrTarget == AddrDisc)
2194 report_fatal_error(reason: "Branch target is signed with its own value");
2195
2196 // If we are printing BLRA pseudo instruction, then x16 and x17 are
2197 // implicit-def'ed by the MI and AddrDisc is not used as any other input, so
2198 // try to save one MOV by setting MayUseAddrAsScratch.
2199 // Unlike BLRA, BRA pseudo is used to perform computed goto, and thus not
2200 // declared as clobbering x16/x17.
2201 Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, ScratchReg: AArch64::X17,
2202 /*MayUseAddrAsScratch=*/IsCall);
2203 bool IsZeroDisc = DiscReg == AArch64::XZR;
2204
2205 unsigned Opc;
2206 if (IsCall) {
2207 if (Key == AArch64PACKey::IA)
2208 Opc = IsZeroDisc ? AArch64::BLRAAZ : AArch64::BLRAA;
2209 else
2210 Opc = IsZeroDisc ? AArch64::BLRABZ : AArch64::BLRAB;
2211 } else {
2212 if (Key == AArch64PACKey::IA)
2213 Opc = IsZeroDisc ? AArch64::BRAAZ : AArch64::BRAA;
2214 else
2215 Opc = IsZeroDisc ? AArch64::BRABZ : AArch64::BRAB;
2216 }
2217
2218 MCInst BRInst;
2219 BRInst.setOpcode(Opc);
2220 BRInst.addOperand(Op: MCOperand::createReg(Reg: BrTarget));
2221 if (!IsZeroDisc)
2222 BRInst.addOperand(Op: MCOperand::createReg(Reg: DiscReg));
2223 EmitToStreamer(S&: *OutStreamer, Inst: BRInst);
2224}
2225
2226const MCExpr *
2227AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) {
2228 MCContext &Ctx = OutContext;
2229
2230 // Figure out the base symbol and the addend, if any.
2231 APInt Offset(64, 0);
2232 const Value *BaseGV = CPA.getPointer()->stripAndAccumulateConstantOffsets(
2233 DL: getDataLayout(), Offset, /*AllowNonInbounds=*/true);
2234
2235 auto *BaseGVB = dyn_cast<GlobalValue>(Val: BaseGV);
2236
2237 // If we can't understand the referenced ConstantExpr, there's nothing
2238 // else we can do: emit an error.
2239 if (!BaseGVB) {
2240 BaseGV->getContext().emitError(
2241 ErrorStr: "cannot resolve target base/addend of ptrauth constant");
2242 return nullptr;
2243 }
2244
2245 // If there is an addend, turn that into the appropriate MCExpr.
2246 const MCExpr *Sym = MCSymbolRefExpr::create(Symbol: getSymbol(GV: BaseGVB), Ctx);
2247 if (Offset.sgt(RHS: 0))
2248 Sym = MCBinaryExpr::createAdd(
2249 LHS: Sym, RHS: MCConstantExpr::create(Value: Offset.getSExtValue(), Ctx), Ctx);
2250 else if (Offset.slt(RHS: 0))
2251 Sym = MCBinaryExpr::createSub(
2252 LHS: Sym, RHS: MCConstantExpr::create(Value: (-Offset).getSExtValue(), Ctx), Ctx);
2253
2254 uint64_t KeyID = CPA.getKey()->getZExtValue();
2255 // We later rely on valid KeyID value in AArch64PACKeyIDToString call from
2256 // AArch64AuthMCExpr::printImpl, so fail fast.
2257 if (KeyID > AArch64PACKey::LAST)
2258 report_fatal_error(reason: "AArch64 PAC Key ID '" + Twine(KeyID) +
2259 "' out of range [0, " +
2260 Twine((unsigned)AArch64PACKey::LAST) + "]");
2261
2262 uint64_t Disc = CPA.getDiscriminator()->getZExtValue();
2263 if (!isUInt<16>(x: Disc))
2264 report_fatal_error(reason: "AArch64 PAC Discriminator '" + Twine(Disc) +
2265 "' out of range [0, 0xFFFF]");
2266
2267 // Finally build the complete @AUTH expr.
2268 return AArch64AuthMCExpr::create(Expr: Sym, Discriminator: Disc, Key: AArch64PACKey::ID(KeyID),
2269 HasAddressDiversity: CPA.hasAddressDiscriminator(), Ctx);
2270}
2271
2272void AArch64AsmPrinter::LowerLOADauthptrstatic(const MachineInstr &MI) {
2273 unsigned DstReg = MI.getOperand(i: 0).getReg();
2274 const MachineOperand &GAOp = MI.getOperand(i: 1);
2275 const uint64_t KeyC = MI.getOperand(i: 2).getImm();
2276 assert(KeyC <= AArch64PACKey::LAST &&
2277 "key is out of range [0, AArch64PACKey::LAST]");
2278 const auto Key = (AArch64PACKey::ID)KeyC;
2279 const uint64_t Disc = MI.getOperand(i: 3).getImm();
2280 assert(isUInt<16>(Disc) &&
2281 "constant discriminator is out of range [0, 0xffff]");
2282
2283 // Emit instruction sequence like the following:
2284 // ADRP x16, symbol$auth_ptr$key$disc
2285 // LDR x16, [x16, :lo12:symbol$auth_ptr$key$disc]
2286 //
2287 // Where the $auth_ptr$ symbol is the stub slot containing the signed pointer
2288 // to symbol.
2289 MCSymbol *AuthPtrStubSym;
2290 if (TM.getTargetTriple().isOSBinFormatELF()) {
2291 const auto &TLOF =
2292 static_cast<const AArch64_ELFTargetObjectFile &>(getObjFileLowering());
2293
2294 assert(GAOp.getOffset() == 0 &&
2295 "non-zero offset for $auth_ptr$ stub slots is not supported");
2296 const MCSymbol *GASym = TM.getSymbol(GV: GAOp.getGlobal());
2297 AuthPtrStubSym = TLOF.getAuthPtrSlotSymbol(TM, MMI, RawSym: GASym, Key, Discriminator: Disc);
2298 } else {
2299 assert(TM.getTargetTriple().isOSBinFormatMachO() &&
2300 "LOADauthptrstatic is implemented only for MachO/ELF");
2301
2302 const auto &TLOF = static_cast<const AArch64_MachoTargetObjectFile &>(
2303 getObjFileLowering());
2304
2305 assert(GAOp.getOffset() == 0 &&
2306 "non-zero offset for $auth_ptr$ stub slots is not supported");
2307 const MCSymbol *GASym = TM.getSymbol(GV: GAOp.getGlobal());
2308 AuthPtrStubSym = TLOF.getAuthPtrSlotSymbol(TM, MMI, RawSym: GASym, Key, Discriminator: Disc);
2309 }
2310
2311 MachineOperand StubMOHi =
2312 MachineOperand::CreateMCSymbol(Sym: AuthPtrStubSym, TargetFlags: AArch64II::MO_PAGE);
2313 MachineOperand StubMOLo = MachineOperand::CreateMCSymbol(
2314 Sym: AuthPtrStubSym, TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2315 MCOperand StubMCHi, StubMCLo;
2316
2317 MCInstLowering.lowerOperand(MO: StubMOHi, MCOp&: StubMCHi);
2318 MCInstLowering.lowerOperand(MO: StubMOLo, MCOp&: StubMCLo);
2319
2320 EmitToStreamer(
2321 S&: *OutStreamer,
2322 Inst: MCInstBuilder(AArch64::ADRP).addReg(Reg: DstReg).addOperand(Op: StubMCHi));
2323
2324 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::LDRXui)
2325 .addReg(Reg: DstReg)
2326 .addReg(Reg: DstReg)
2327 .addOperand(Op: StubMCLo));
2328}
2329
2330void AArch64AsmPrinter::LowerMOVaddrPAC(const MachineInstr &MI) {
2331 const bool IsGOTLoad = MI.getOpcode() == AArch64::LOADgotPAC;
2332 const bool IsELFSignedGOT = MI.getParent()
2333 ->getParent()
2334 ->getInfo<AArch64FunctionInfo>()
2335 ->hasELFSignedGOT();
2336 MachineOperand GAOp = MI.getOperand(i: 0);
2337 const uint64_t KeyC = MI.getOperand(i: 1).getImm();
2338 assert(KeyC <= AArch64PACKey::LAST &&
2339 "key is out of range [0, AArch64PACKey::LAST]");
2340 const auto Key = (AArch64PACKey::ID)KeyC;
2341 const unsigned AddrDisc = MI.getOperand(i: 2).getReg();
2342 const uint64_t Disc = MI.getOperand(i: 3).getImm();
2343 assert(isUInt<16>(Disc) &&
2344 "constant discriminator is out of range [0, 0xffff]");
2345
2346 const int64_t Offset = GAOp.getOffset();
2347 GAOp.setOffset(0);
2348
2349 // Emit:
2350 // target materialization:
2351 // - via GOT:
2352 // - unsigned GOT:
2353 // adrp x16, :got:target
2354 // ldr x16, [x16, :got_lo12:target]
2355 // add offset to x16 if offset != 0
2356 // - ELF signed GOT:
2357 // adrp x17, :got:target
2358 // add x17, x17, :got_auth_lo12:target
2359 // ldr x16, [x17]
2360 // aut{i|d}a x16, x17
2361 // check+trap sequence (if no FPAC)
2362 // add offset to x16 if offset != 0
2363 //
2364 // - direct:
2365 // adrp x16, target
2366 // add x16, x16, :lo12:target
2367 // add offset to x16 if offset != 0
2368 //
2369 // add offset to x16:
2370 // - abs(offset) fits 24 bits:
2371 // add/sub x16, x16, #<offset>[, #lsl 12] (up to 2 instructions)
2372 // - abs(offset) does not fit 24 bits:
2373 // - offset < 0:
2374 // movn+movk sequence filling x17 register with the offset (up to 4
2375 // instructions)
2376 // add x16, x16, x17
2377 // - offset > 0:
2378 // movz+movk sequence filling x17 register with the offset (up to 4
2379 // instructions)
2380 // add x16, x16, x17
2381 //
2382 // signing:
2383 // - 0 discriminator:
2384 // paciza x16
2385 // - Non-0 discriminator, no address discriminator:
2386 // mov x17, #Disc
2387 // pacia x16, x17
2388 // - address discriminator (with potentially folded immediate discriminator):
2389 // pacia x16, xAddrDisc
2390
2391 MachineOperand GAMOHi(GAOp), GAMOLo(GAOp);
2392 MCOperand GAMCHi, GAMCLo;
2393
2394 GAMOHi.setTargetFlags(AArch64II::MO_PAGE);
2395 GAMOLo.setTargetFlags(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2396 if (IsGOTLoad) {
2397 GAMOHi.addTargetFlag(F: AArch64II::MO_GOT);
2398 GAMOLo.addTargetFlag(F: AArch64II::MO_GOT);
2399 }
2400
2401 MCInstLowering.lowerOperand(MO: GAMOHi, MCOp&: GAMCHi);
2402 MCInstLowering.lowerOperand(MO: GAMOLo, MCOp&: GAMCLo);
2403
2404 EmitToStreamer(
2405 Inst: MCInstBuilder(AArch64::ADRP)
2406 .addReg(Reg: IsGOTLoad && IsELFSignedGOT ? AArch64::X17 : AArch64::X16)
2407 .addOperand(Op: GAMCHi));
2408
2409 if (IsGOTLoad) {
2410 if (IsELFSignedGOT) {
2411 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
2412 .addReg(Reg: AArch64::X17)
2413 .addReg(Reg: AArch64::X17)
2414 .addOperand(Op: GAMCLo)
2415 .addImm(Val: 0));
2416
2417 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
2418 .addReg(Reg: AArch64::X16)
2419 .addReg(Reg: AArch64::X17)
2420 .addImm(Val: 0));
2421
2422 assert(GAOp.isGlobal());
2423 assert(GAOp.getGlobal()->getValueType() != nullptr);
2424 unsigned AuthOpcode = GAOp.getGlobal()->getValueType()->isFunctionTy()
2425 ? AArch64::AUTIA
2426 : AArch64::AUTDA;
2427
2428 EmitToStreamer(Inst: MCInstBuilder(AuthOpcode)
2429 .addReg(Reg: AArch64::X16)
2430 .addReg(Reg: AArch64::X16)
2431 .addReg(Reg: AArch64::X17));
2432
2433 if (!STI->hasFPAC()) {
2434 auto AuthKey = (AuthOpcode == AArch64::AUTIA ? AArch64PACKey::IA
2435 : AArch64PACKey::DA);
2436
2437 emitPtrauthCheckAuthenticatedValue(TestedReg: AArch64::X16, ScratchReg: AArch64::X17, Key: AuthKey,
2438 Method: AArch64PAuth::AuthCheckMethod::XPAC,
2439 /*ShouldTrap=*/true,
2440 /*OnFailure=*/nullptr);
2441 }
2442 } else {
2443 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
2444 .addReg(Reg: AArch64::X16)
2445 .addReg(Reg: AArch64::X16)
2446 .addOperand(Op: GAMCLo));
2447 }
2448 } else {
2449 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
2450 .addReg(Reg: AArch64::X16)
2451 .addReg(Reg: AArch64::X16)
2452 .addOperand(Op: GAMCLo)
2453 .addImm(Val: 0));
2454 }
2455
2456 if (Offset != 0) {
2457 const uint64_t AbsOffset = (Offset > 0 ? Offset : -((uint64_t)Offset));
2458 const bool IsNeg = Offset < 0;
2459 if (isUInt<24>(x: AbsOffset)) {
2460 for (int BitPos = 0; BitPos != 24 && (AbsOffset >> BitPos);
2461 BitPos += 12) {
2462 EmitToStreamer(
2463 Inst: MCInstBuilder(IsNeg ? AArch64::SUBXri : AArch64::ADDXri)
2464 .addReg(Reg: AArch64::X16)
2465 .addReg(Reg: AArch64::X16)
2466 .addImm(Val: (AbsOffset >> BitPos) & 0xfff)
2467 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: BitPos)));
2468 }
2469 } else {
2470 const uint64_t UOffset = Offset;
2471 EmitToStreamer(Inst: MCInstBuilder(IsNeg ? AArch64::MOVNXi : AArch64::MOVZXi)
2472 .addReg(Reg: AArch64::X17)
2473 .addImm(Val: (IsNeg ? ~UOffset : UOffset) & 0xffff)
2474 .addImm(/*shift=*/Val: 0));
2475 auto NeedMovk = [IsNeg, UOffset](int BitPos) -> bool {
2476 assert(BitPos == 16 || BitPos == 32 || BitPos == 48);
2477 uint64_t Shifted = UOffset >> BitPos;
2478 if (!IsNeg)
2479 return Shifted != 0;
2480 for (int I = 0; I != 64 - BitPos; I += 16)
2481 if (((Shifted >> I) & 0xffff) != 0xffff)
2482 return true;
2483 return false;
2484 };
2485 for (int BitPos = 16; BitPos != 64 && NeedMovk(BitPos); BitPos += 16)
2486 emitMOVK(Dest: AArch64::X17, Imm: (UOffset >> BitPos) & 0xffff, Shift: BitPos);
2487
2488 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXrs)
2489 .addReg(Reg: AArch64::X16)
2490 .addReg(Reg: AArch64::X16)
2491 .addReg(Reg: AArch64::X17)
2492 .addImm(/*shift=*/Val: 0));
2493 }
2494 }
2495
2496 Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, ScratchReg: AArch64::X17);
2497
2498 auto MIB = MCInstBuilder(getPACOpcodeForKey(K: Key, Zero: DiscReg == AArch64::XZR))
2499 .addReg(Reg: AArch64::X16)
2500 .addReg(Reg: AArch64::X16);
2501 if (DiscReg != AArch64::XZR)
2502 MIB.addReg(Reg: DiscReg);
2503 EmitToStreamer(Inst: MIB);
2504}
2505
2506void AArch64AsmPrinter::LowerLOADgotAUTH(const MachineInstr &MI) {
2507 Register DstReg = MI.getOperand(i: 0).getReg();
2508 Register AuthResultReg = STI->hasFPAC() ? DstReg : AArch64::X16;
2509 const MachineOperand &GAMO = MI.getOperand(i: 1);
2510 assert(GAMO.getOffset() == 0);
2511
2512 if (MI.getMF()->getTarget().getCodeModel() == CodeModel::Tiny) {
2513 MCOperand GAMC;
2514 MCInstLowering.lowerOperand(MO: GAMO, MCOp&: GAMC);
2515 EmitToStreamer(
2516 Inst: MCInstBuilder(AArch64::ADR).addReg(Reg: AArch64::X17).addOperand(Op: GAMC));
2517 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
2518 .addReg(Reg: AuthResultReg)
2519 .addReg(Reg: AArch64::X17)
2520 .addImm(Val: 0));
2521 } else {
2522 MachineOperand GAHiOp(GAMO);
2523 MachineOperand GALoOp(GAMO);
2524 GAHiOp.addTargetFlag(F: AArch64II::MO_PAGE);
2525 GALoOp.addTargetFlag(F: AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2526
2527 MCOperand GAMCHi, GAMCLo;
2528 MCInstLowering.lowerOperand(MO: GAHiOp, MCOp&: GAMCHi);
2529 MCInstLowering.lowerOperand(MO: GALoOp, MCOp&: GAMCLo);
2530
2531 EmitToStreamer(
2532 Inst: MCInstBuilder(AArch64::ADRP).addReg(Reg: AArch64::X17).addOperand(Op: GAMCHi));
2533
2534 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
2535 .addReg(Reg: AArch64::X17)
2536 .addReg(Reg: AArch64::X17)
2537 .addOperand(Op: GAMCLo)
2538 .addImm(Val: 0));
2539
2540 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
2541 .addReg(Reg: AuthResultReg)
2542 .addReg(Reg: AArch64::X17)
2543 .addImm(Val: 0));
2544 }
2545
2546 assert(GAMO.isGlobal());
2547 MCSymbol *UndefWeakSym;
2548 if (GAMO.getGlobal()->hasExternalWeakLinkage()) {
2549 UndefWeakSym = createTempSymbol(Name: "undef_weak");
2550 EmitToStreamer(
2551 Inst: MCInstBuilder(AArch64::CBZX)
2552 .addReg(Reg: AuthResultReg)
2553 .addExpr(Val: MCSymbolRefExpr::create(Symbol: UndefWeakSym, Ctx&: OutContext)));
2554 }
2555
2556 assert(GAMO.getGlobal()->getValueType() != nullptr);
2557 unsigned AuthOpcode = GAMO.getGlobal()->getValueType()->isFunctionTy()
2558 ? AArch64::AUTIA
2559 : AArch64::AUTDA;
2560 EmitToStreamer(Inst: MCInstBuilder(AuthOpcode)
2561 .addReg(Reg: AuthResultReg)
2562 .addReg(Reg: AuthResultReg)
2563 .addReg(Reg: AArch64::X17));
2564
2565 if (GAMO.getGlobal()->hasExternalWeakLinkage())
2566 OutStreamer->emitLabel(Symbol: UndefWeakSym);
2567
2568 if (!STI->hasFPAC()) {
2569 auto AuthKey =
2570 (AuthOpcode == AArch64::AUTIA ? AArch64PACKey::IA : AArch64PACKey::DA);
2571
2572 emitPtrauthCheckAuthenticatedValue(TestedReg: AuthResultReg, ScratchReg: AArch64::X17, Key: AuthKey,
2573 Method: AArch64PAuth::AuthCheckMethod::XPAC,
2574 /*ShouldTrap=*/true,
2575 /*OnFailure=*/nullptr);
2576
2577 emitMovXReg(Dest: DstReg, Src: AuthResultReg);
2578 }
2579}
2580
2581const MCExpr *
2582AArch64AsmPrinter::lowerBlockAddressConstant(const BlockAddress &BA) {
2583 const MCExpr *BAE = AsmPrinter::lowerBlockAddressConstant(BA);
2584 const Function &Fn = *BA.getFunction();
2585
2586 if (std::optional<uint16_t> BADisc =
2587 STI->getPtrAuthBlockAddressDiscriminatorIfEnabled(ParentFn: Fn))
2588 return AArch64AuthMCExpr::create(Expr: BAE, Discriminator: *BADisc, Key: AArch64PACKey::IA,
2589 /*HasAddressDiversity=*/false, Ctx&: OutContext);
2590
2591 return BAE;
2592}
2593
2594void AArch64AsmPrinter::emitCBPseudoExpansion(const MachineInstr *MI) {
2595 bool IsImm = false;
2596 bool Is32Bit = false;
2597
2598 switch (MI->getOpcode()) {
2599 default:
2600 llvm_unreachable("This is not a CB pseudo instruction");
2601 case AArch64::CBWPrr:
2602 Is32Bit = true;
2603 break;
2604 case AArch64::CBXPrr:
2605 Is32Bit = false;
2606 break;
2607 case AArch64::CBWPri:
2608 IsImm = true;
2609 Is32Bit = true;
2610 break;
2611 case AArch64::CBXPri:
2612 IsImm = true;
2613 break;
2614 }
2615
2616 AArch64CC::CondCode CC =
2617 static_cast<AArch64CC::CondCode>(MI->getOperand(i: 0).getImm());
2618 bool NeedsRegSwap = false;
2619 bool NeedsImmDec = false;
2620 bool NeedsImmInc = false;
2621
2622 // Decide if we need to either swap register operands or increment/decrement
2623 // immediate operands
2624 unsigned MCOpC;
2625 switch (CC) {
2626 default:
2627 llvm_unreachable("Invalid CB condition code");
2628 case AArch64CC::EQ:
2629 MCOpC = IsImm ? (Is32Bit ? AArch64::CBEQWri : AArch64::CBEQXri)
2630 : (Is32Bit ? AArch64::CBEQWrr : AArch64::CBEQXrr);
2631 break;
2632 case AArch64CC::NE:
2633 MCOpC = IsImm ? (Is32Bit ? AArch64::CBNEWri : AArch64::CBNEXri)
2634 : (Is32Bit ? AArch64::CBNEWrr : AArch64::CBNEXrr);
2635 break;
2636 case AArch64CC::HS:
2637 MCOpC = IsImm ? (Is32Bit ? AArch64::CBHIWri : AArch64::CBHIXri)
2638 : (Is32Bit ? AArch64::CBHSWrr : AArch64::CBHSXrr);
2639 NeedsImmDec = IsImm;
2640 break;
2641 case AArch64CC::LO:
2642 MCOpC = IsImm ? (Is32Bit ? AArch64::CBLOWri : AArch64::CBLOXri)
2643 : (Is32Bit ? AArch64::CBHIWrr : AArch64::CBHIXrr);
2644 NeedsRegSwap = !IsImm;
2645 break;
2646 case AArch64CC::HI:
2647 MCOpC = IsImm ? (Is32Bit ? AArch64::CBHIWri : AArch64::CBHIXri)
2648 : (Is32Bit ? AArch64::CBHIWrr : AArch64::CBHIXrr);
2649 break;
2650 case AArch64CC::LS:
2651 MCOpC = IsImm ? (Is32Bit ? AArch64::CBLOWri : AArch64::CBLOXri)
2652 : (Is32Bit ? AArch64::CBHSWrr : AArch64::CBHSXrr);
2653 NeedsRegSwap = !IsImm;
2654 NeedsImmInc = IsImm;
2655 break;
2656 case AArch64CC::GE:
2657 MCOpC = IsImm ? (Is32Bit ? AArch64::CBGTWri : AArch64::CBGTXri)
2658 : (Is32Bit ? AArch64::CBGEWrr : AArch64::CBGEXrr);
2659 NeedsImmDec = IsImm;
2660 break;
2661 case AArch64CC::LT:
2662 MCOpC = IsImm ? (Is32Bit ? AArch64::CBLTWri : AArch64::CBLTXri)
2663 : (Is32Bit ? AArch64::CBGTWrr : AArch64::CBGTXrr);
2664 NeedsRegSwap = !IsImm;
2665 break;
2666 case AArch64CC::GT:
2667 MCOpC = IsImm ? (Is32Bit ? AArch64::CBGTWri : AArch64::CBGTXri)
2668 : (Is32Bit ? AArch64::CBGTWrr : AArch64::CBGTXrr);
2669 break;
2670 case AArch64CC::LE:
2671 MCOpC = IsImm ? (Is32Bit ? AArch64::CBLTWri : AArch64::CBLTXri)
2672 : (Is32Bit ? AArch64::CBGEWrr : AArch64::CBGEXrr);
2673 NeedsRegSwap = !IsImm;
2674 NeedsImmInc = IsImm;
2675 break;
2676 }
2677
2678 MCInst Inst;
2679 Inst.setOpcode(MCOpC);
2680
2681 MCOperand Lhs, Rhs, Trgt;
2682 lowerOperand(MO: MI->getOperand(i: 1), MCOp&: Lhs);
2683 lowerOperand(MO: MI->getOperand(i: 2), MCOp&: Rhs);
2684 lowerOperand(MO: MI->getOperand(i: 3), MCOp&: Trgt);
2685
2686 // Now swap, increment or decrement
2687 if (NeedsRegSwap) {
2688 assert(Lhs.isReg() && "Expected register operand for CB");
2689 assert(Rhs.isReg() && "Expected register operand for CB");
2690 Inst.addOperand(Op: Rhs);
2691 Inst.addOperand(Op: Lhs);
2692 } else if (NeedsImmDec) {
2693 Rhs.setImm(Rhs.getImm() - 1);
2694 Inst.addOperand(Op: Lhs);
2695 Inst.addOperand(Op: Rhs);
2696 } else if (NeedsImmInc) {
2697 Rhs.setImm(Rhs.getImm() + 1);
2698 Inst.addOperand(Op: Lhs);
2699 Inst.addOperand(Op: Rhs);
2700 } else {
2701 Inst.addOperand(Op: Lhs);
2702 Inst.addOperand(Op: Rhs);
2703 }
2704
2705 assert((!IsImm || (Rhs.getImm() >= 0 && Rhs.getImm() < 64)) &&
2706 "CB immediate operand out-of-bounds");
2707
2708 Inst.addOperand(Op: Trgt);
2709 EmitToStreamer(S&: *OutStreamer, Inst);
2710}
2711
2712// Simple pseudo-instructions have their lowering (with expansion to real
2713// instructions) auto-generated.
2714#include "AArch64GenMCPseudoLowering.inc"
2715
2716void AArch64AsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) {
2717 S.emitInstruction(Inst, STI: *STI);
2718#ifndef NDEBUG
2719 ++InstsEmitted;
2720#endif
2721}
2722
2723void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
2724 AArch64_MC::verifyInstructionPredicates(Opcode: MI->getOpcode(), Features: STI->getFeatureBits());
2725
2726#ifndef NDEBUG
2727 InstsEmitted = 0;
2728 auto CheckMISize = make_scope_exit([&]() {
2729 assert(STI->getInstrInfo()->getInstSizeInBytes(*MI) >= InstsEmitted * 4);
2730 });
2731#endif
2732
2733 // Do any auto-generated pseudo lowerings.
2734 if (MCInst OutInst; lowerPseudoInstExpansion(MI, Inst&: OutInst)) {
2735 EmitToStreamer(S&: *OutStreamer, Inst: OutInst);
2736 return;
2737 }
2738
2739 if (MI->getOpcode() == AArch64::ADRP) {
2740 for (auto &Opd : MI->operands()) {
2741 if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) ==
2742 "swift_async_extendedFramePointerFlags") {
2743 ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true;
2744 }
2745 }
2746 }
2747
2748 if (AArch64FI->getLOHRelated().count(Ptr: MI)) {
2749 // Generate a label for LOH related instruction
2750 MCSymbol *LOHLabel = createTempSymbol(Name: "loh");
2751 // Associate the instruction with the label
2752 LOHInstToLabel[MI] = LOHLabel;
2753 OutStreamer->emitLabel(Symbol: LOHLabel);
2754 }
2755
2756 AArch64TargetStreamer *TS =
2757 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
2758 // Do any manual lowerings.
2759 switch (MI->getOpcode()) {
2760 default:
2761 assert(!AArch64InstrInfo::isTailCallReturnInst(*MI) &&
2762 "Unhandled tail call instruction");
2763 break;
2764 case AArch64::HINT: {
2765 // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
2766 // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
2767 // non-empty. If MI is the initial BTI, place the
2768 // __patchable_function_entries label after BTI.
2769 if (CurrentPatchableFunctionEntrySym &&
2770 CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
2771 MI == &MF->front().front()) {
2772 int64_t Imm = MI->getOperand(i: 0).getImm();
2773 if ((Imm & 32) && (Imm & 6)) {
2774 MCInst Inst;
2775 MCInstLowering.Lower(MI, OutMI&: Inst);
2776 EmitToStreamer(S&: *OutStreamer, Inst);
2777 CurrentPatchableFunctionEntrySym = createTempSymbol(Name: "patch");
2778 OutStreamer->emitLabel(Symbol: CurrentPatchableFunctionEntrySym);
2779 return;
2780 }
2781 }
2782 break;
2783 }
2784 case AArch64::MOVMCSym: {
2785 Register DestReg = MI->getOperand(i: 0).getReg();
2786 const MachineOperand &MO_Sym = MI->getOperand(i: 1);
2787 MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
2788 MCOperand Hi_MCSym, Lo_MCSym;
2789
2790 Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
2791 Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
2792
2793 MCInstLowering.lowerOperand(MO: Hi_MOSym, MCOp&: Hi_MCSym);
2794 MCInstLowering.lowerOperand(MO: Lo_MOSym, MCOp&: Lo_MCSym);
2795
2796 MCInst MovZ;
2797 MovZ.setOpcode(AArch64::MOVZXi);
2798 MovZ.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2799 MovZ.addOperand(Op: Hi_MCSym);
2800 MovZ.addOperand(Op: MCOperand::createImm(Val: 16));
2801 EmitToStreamer(S&: *OutStreamer, Inst: MovZ);
2802
2803 MCInst MovK;
2804 MovK.setOpcode(AArch64::MOVKXi);
2805 MovK.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2806 MovK.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2807 MovK.addOperand(Op: Lo_MCSym);
2808 MovK.addOperand(Op: MCOperand::createImm(Val: 0));
2809 EmitToStreamer(S&: *OutStreamer, Inst: MovK);
2810 return;
2811 }
2812 case AArch64::MOVIv2d_ns:
2813 // It is generally beneficial to rewrite "fmov s0, wzr" to "movi d0, #0".
2814 // as movi is more efficient across all cores. Newer cores can eliminate
2815 // fmovs early and there is no difference with movi, but this not true for
2816 // all implementations.
2817 //
2818 // The floating-point version doesn't quite work in rare cases on older
2819 // CPUs, so on those targets we lower this instruction to movi.16b instead.
2820 if (STI->hasZeroCycleZeroingFPWorkaround() &&
2821 MI->getOperand(i: 1).getImm() == 0) {
2822 MCInst TmpInst;
2823 TmpInst.setOpcode(AArch64::MOVIv16b_ns);
2824 TmpInst.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg()));
2825 TmpInst.addOperand(Op: MCOperand::createImm(Val: MI->getOperand(i: 1).getImm()));
2826 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
2827 return;
2828 }
2829 break;
2830
2831 case AArch64::DBG_VALUE:
2832 case AArch64::DBG_VALUE_LIST:
2833 if (isVerbose() && OutStreamer->hasRawTextSupport()) {
2834 SmallString<128> TmpStr;
2835 raw_svector_ostream OS(TmpStr);
2836 PrintDebugValueComment(MI, OS);
2837 OutStreamer->emitRawText(String: StringRef(OS.str()));
2838 }
2839 return;
2840
2841 case AArch64::EMITBKEY: {
2842 ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
2843 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
2844 ExceptionHandlingType != ExceptionHandling::ARM)
2845 return;
2846
2847 if (getFunctionCFISectionType(MF: *MF) == CFISection::None)
2848 return;
2849
2850 OutStreamer->emitCFIBKeyFrame();
2851 return;
2852 }
2853
2854 case AArch64::EMITMTETAGGED: {
2855 ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
2856 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
2857 ExceptionHandlingType != ExceptionHandling::ARM)
2858 return;
2859
2860 if (getFunctionCFISectionType(MF: *MF) != CFISection::None)
2861 OutStreamer->emitCFIMTETaggedFrame();
2862 return;
2863 }
2864
2865 case AArch64::AUT:
2866 case AArch64::AUTPAC:
2867 emitPtrauthAuthResign(MI);
2868 return;
2869
2870 case AArch64::LOADauthptrstatic:
2871 LowerLOADauthptrstatic(MI: *MI);
2872 return;
2873
2874 case AArch64::LOADgotPAC:
2875 case AArch64::MOVaddrPAC:
2876 LowerMOVaddrPAC(MI: *MI);
2877 return;
2878
2879 case AArch64::LOADgotAUTH:
2880 LowerLOADgotAUTH(MI: *MI);
2881 return;
2882
2883 case AArch64::BRA:
2884 case AArch64::BLRA:
2885 emitPtrauthBranch(MI);
2886 return;
2887
2888 // Tail calls use pseudo instructions so they have the proper code-gen
2889 // attributes (isCall, isReturn, etc.). We lower them to the real
2890 // instruction here.
2891 case AArch64::AUTH_TCRETURN:
2892 case AArch64::AUTH_TCRETURN_BTI: {
2893 Register Callee = MI->getOperand(i: 0).getReg();
2894 const uint64_t Key = MI->getOperand(i: 2).getImm();
2895 assert((Key == AArch64PACKey::IA || Key == AArch64PACKey::IB) &&
2896 "Invalid auth key for tail-call return");
2897
2898 const uint64_t Disc = MI->getOperand(i: 3).getImm();
2899 assert(isUInt<16>(Disc) && "Integer discriminator is too wide");
2900
2901 Register AddrDisc = MI->getOperand(i: 4).getReg();
2902
2903 Register ScratchReg = Callee == AArch64::X16 ? AArch64::X17 : AArch64::X16;
2904
2905 emitPtrauthTailCallHardening(TC: MI);
2906
2907 // See the comments in emitPtrauthBranch.
2908 if (Callee == AddrDisc)
2909 report_fatal_error(reason: "Call target is signed with its own value");
2910 Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, ScratchReg,
2911 /*MayUseAddrAsScratch=*/true);
2912
2913 const bool IsZero = DiscReg == AArch64::XZR;
2914 const unsigned Opcodes[2][2] = {{AArch64::BRAA, AArch64::BRAAZ},
2915 {AArch64::BRAB, AArch64::BRABZ}};
2916
2917 MCInst TmpInst;
2918 TmpInst.setOpcode(Opcodes[Key][IsZero]);
2919 TmpInst.addOperand(Op: MCOperand::createReg(Reg: Callee));
2920 if (!IsZero)
2921 TmpInst.addOperand(Op: MCOperand::createReg(Reg: DiscReg));
2922 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
2923 return;
2924 }
2925
2926 case AArch64::TCRETURNri:
2927 case AArch64::TCRETURNrix16x17:
2928 case AArch64::TCRETURNrix17:
2929 case AArch64::TCRETURNrinotx16:
2930 case AArch64::TCRETURNriALL: {
2931 emitPtrauthTailCallHardening(TC: MI);
2932
2933 recordIfImportCall(BranchInst: MI);
2934 MCInst TmpInst;
2935 TmpInst.setOpcode(AArch64::BR);
2936 TmpInst.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg()));
2937 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
2938 return;
2939 }
2940 case AArch64::TCRETURNdi: {
2941 emitPtrauthTailCallHardening(TC: MI);
2942
2943 MCOperand Dest;
2944 MCInstLowering.lowerOperand(MO: MI->getOperand(i: 0), MCOp&: Dest);
2945 recordIfImportCall(BranchInst: MI);
2946 MCInst TmpInst;
2947 TmpInst.setOpcode(AArch64::B);
2948 TmpInst.addOperand(Op: Dest);
2949 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
2950 return;
2951 }
2952 case AArch64::SpeculationBarrierISBDSBEndBB: {
2953 // Print DSB SYS + ISB
2954 MCInst TmpInstDSB;
2955 TmpInstDSB.setOpcode(AArch64::DSB);
2956 TmpInstDSB.addOperand(Op: MCOperand::createImm(Val: 0xf));
2957 EmitToStreamer(S&: *OutStreamer, Inst: TmpInstDSB);
2958 MCInst TmpInstISB;
2959 TmpInstISB.setOpcode(AArch64::ISB);
2960 TmpInstISB.addOperand(Op: MCOperand::createImm(Val: 0xf));
2961 EmitToStreamer(S&: *OutStreamer, Inst: TmpInstISB);
2962 return;
2963 }
2964 case AArch64::SpeculationBarrierSBEndBB: {
2965 // Print SB
2966 MCInst TmpInstSB;
2967 TmpInstSB.setOpcode(AArch64::SB);
2968 EmitToStreamer(S&: *OutStreamer, Inst: TmpInstSB);
2969 return;
2970 }
2971 case AArch64::TLSDESC_AUTH_CALLSEQ: {
2972 /// lower this to:
2973 /// adrp x0, :tlsdesc_auth:var
2974 /// ldr x16, [x0, #:tlsdesc_auth_lo12:var]
2975 /// add x0, x0, #:tlsdesc_auth_lo12:var
2976 /// blraa x16, x0
2977 /// (TPIDR_EL0 offset now in x0)
2978 const MachineOperand &MO_Sym = MI->getOperand(i: 0);
2979 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
2980 MCOperand SymTLSDescLo12, SymTLSDesc;
2981 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
2982 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
2983 MCInstLowering.lowerOperand(MO: MO_TLSDESC_LO12, MCOp&: SymTLSDescLo12);
2984 MCInstLowering.lowerOperand(MO: MO_TLSDESC, MCOp&: SymTLSDesc);
2985
2986 MCInst Adrp;
2987 Adrp.setOpcode(AArch64::ADRP);
2988 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
2989 Adrp.addOperand(Op: SymTLSDesc);
2990 EmitToStreamer(S&: *OutStreamer, Inst: Adrp);
2991
2992 MCInst Ldr;
2993 Ldr.setOpcode(AArch64::LDRXui);
2994 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
2995 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
2996 Ldr.addOperand(Op: SymTLSDescLo12);
2997 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
2998 EmitToStreamer(S&: *OutStreamer, Inst: Ldr);
2999
3000 MCInst Add;
3001 Add.setOpcode(AArch64::ADDXri);
3002 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3003 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3004 Add.addOperand(Op: SymTLSDescLo12);
3005 Add.addOperand(Op: MCOperand::createImm(Val: AArch64_AM::getShiftValue(Imm: 0)));
3006 EmitToStreamer(S&: *OutStreamer, Inst: Add);
3007
3008 // Authenticated TLSDESC accesses are not relaxed.
3009 // Thus, do not emit .tlsdesccall for AUTH TLSDESC.
3010
3011 MCInst Blraa;
3012 Blraa.setOpcode(AArch64::BLRAA);
3013 Blraa.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
3014 Blraa.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3015 EmitToStreamer(S&: *OutStreamer, Inst: Blraa);
3016
3017 return;
3018 }
3019 case AArch64::TLSDESC_CALLSEQ: {
3020 /// lower this to:
3021 /// adrp x0, :tlsdesc:var
3022 /// ldr x1, [x0, #:tlsdesc_lo12:var]
3023 /// add x0, x0, #:tlsdesc_lo12:var
3024 /// .tlsdesccall var
3025 /// blr x1
3026 /// (TPIDR_EL0 offset now in x0)
3027 const MachineOperand &MO_Sym = MI->getOperand(i: 0);
3028 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
3029 MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
3030 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
3031 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
3032 MCInstLowering.lowerOperand(MO: MO_Sym, MCOp&: Sym);
3033 MCInstLowering.lowerOperand(MO: MO_TLSDESC_LO12, MCOp&: SymTLSDescLo12);
3034 MCInstLowering.lowerOperand(MO: MO_TLSDESC, MCOp&: SymTLSDesc);
3035
3036 MCInst Adrp;
3037 Adrp.setOpcode(AArch64::ADRP);
3038 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3039 Adrp.addOperand(Op: SymTLSDesc);
3040 EmitToStreamer(S&: *OutStreamer, Inst: Adrp);
3041
3042 MCInst Ldr;
3043 if (STI->isTargetILP32()) {
3044 Ldr.setOpcode(AArch64::LDRWui);
3045 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::W1));
3046 } else {
3047 Ldr.setOpcode(AArch64::LDRXui);
3048 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X1));
3049 }
3050 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3051 Ldr.addOperand(Op: SymTLSDescLo12);
3052 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
3053 EmitToStreamer(S&: *OutStreamer, Inst: Ldr);
3054
3055 MCInst Add;
3056 if (STI->isTargetILP32()) {
3057 Add.setOpcode(AArch64::ADDWri);
3058 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::W0));
3059 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::W0));
3060 } else {
3061 Add.setOpcode(AArch64::ADDXri);
3062 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3063 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3064 }
3065 Add.addOperand(Op: SymTLSDescLo12);
3066 Add.addOperand(Op: MCOperand::createImm(Val: AArch64_AM::getShiftValue(Imm: 0)));
3067 EmitToStreamer(S&: *OutStreamer, Inst: Add);
3068
3069 // Emit a relocation-annotation. This expands to no code, but requests
3070 // the following instruction gets an R_AARCH64_TLSDESC_CALL.
3071 MCInst TLSDescCall;
3072 TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
3073 TLSDescCall.addOperand(Op: Sym);
3074 EmitToStreamer(S&: *OutStreamer, Inst: TLSDescCall);
3075#ifndef NDEBUG
3076 --InstsEmitted; // no code emitted
3077#endif
3078
3079 MCInst Blr;
3080 Blr.setOpcode(AArch64::BLR);
3081 Blr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X1));
3082 EmitToStreamer(S&: *OutStreamer, Inst: Blr);
3083
3084 return;
3085 }
3086
3087 case AArch64::JumpTableDest32:
3088 case AArch64::JumpTableDest16:
3089 case AArch64::JumpTableDest8:
3090 LowerJumpTableDest(OutStreamer&: *OutStreamer, MI: *MI);
3091 return;
3092
3093 case AArch64::BR_JumpTable:
3094 LowerHardenedBRJumpTable(MI: *MI);
3095 return;
3096
3097 case AArch64::FMOVH0:
3098 case AArch64::FMOVS0:
3099 case AArch64::FMOVD0:
3100 emitFMov0(MI: *MI);
3101 return;
3102
3103 case AArch64::MOPSMemoryCopyPseudo:
3104 case AArch64::MOPSMemoryMovePseudo:
3105 case AArch64::MOPSMemorySetPseudo:
3106 case AArch64::MOPSMemorySetTaggingPseudo:
3107 LowerMOPS(OutStreamer&: *OutStreamer, MI: *MI);
3108 return;
3109
3110 case TargetOpcode::STACKMAP:
3111 return LowerSTACKMAP(OutStreamer&: *OutStreamer, SM, MI: *MI);
3112
3113 case TargetOpcode::PATCHPOINT:
3114 return LowerPATCHPOINT(OutStreamer&: *OutStreamer, SM, MI: *MI);
3115
3116 case TargetOpcode::STATEPOINT:
3117 return LowerSTATEPOINT(OutStreamer&: *OutStreamer, SM, MI: *MI);
3118
3119 case TargetOpcode::FAULTING_OP:
3120 return LowerFAULTING_OP(FaultingMI: *MI);
3121
3122 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
3123 LowerPATCHABLE_FUNCTION_ENTER(MI: *MI);
3124 return;
3125
3126 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
3127 LowerPATCHABLE_FUNCTION_EXIT(MI: *MI);
3128 return;
3129
3130 case TargetOpcode::PATCHABLE_TAIL_CALL:
3131 LowerPATCHABLE_TAIL_CALL(MI: *MI);
3132 return;
3133 case TargetOpcode::PATCHABLE_EVENT_CALL:
3134 return LowerPATCHABLE_EVENT_CALL(MI: *MI, Typed: false);
3135 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
3136 return LowerPATCHABLE_EVENT_CALL(MI: *MI, Typed: true);
3137
3138 case AArch64::KCFI_CHECK:
3139 LowerKCFI_CHECK(MI: *MI);
3140 return;
3141
3142 case AArch64::HWASAN_CHECK_MEMACCESS:
3143 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
3144 case AArch64::HWASAN_CHECK_MEMACCESS_FIXEDSHADOW:
3145 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW:
3146 LowerHWASAN_CHECK_MEMACCESS(MI: *MI);
3147 return;
3148
3149 case AArch64::SEH_StackAlloc:
3150 TS->emitARM64WinCFIAllocStack(Size: MI->getOperand(i: 0).getImm());
3151 return;
3152
3153 case AArch64::SEH_SaveFPLR:
3154 TS->emitARM64WinCFISaveFPLR(Offset: MI->getOperand(i: 0).getImm());
3155 return;
3156
3157 case AArch64::SEH_SaveFPLR_X:
3158 assert(MI->getOperand(0).getImm() < 0 &&
3159 "Pre increment SEH opcode must have a negative offset");
3160 TS->emitARM64WinCFISaveFPLRX(Offset: -MI->getOperand(i: 0).getImm());
3161 return;
3162
3163 case AArch64::SEH_SaveReg:
3164 TS->emitARM64WinCFISaveReg(Reg: MI->getOperand(i: 0).getImm(),
3165 Offset: MI->getOperand(i: 1).getImm());
3166 return;
3167
3168 case AArch64::SEH_SaveReg_X:
3169 assert(MI->getOperand(1).getImm() < 0 &&
3170 "Pre increment SEH opcode must have a negative offset");
3171 TS->emitARM64WinCFISaveRegX(Reg: MI->getOperand(i: 0).getImm(),
3172 Offset: -MI->getOperand(i: 1).getImm());
3173 return;
3174
3175 case AArch64::SEH_SaveRegP:
3176 if (MI->getOperand(i: 1).getImm() == 30 && MI->getOperand(i: 0).getImm() >= 19 &&
3177 MI->getOperand(i: 0).getImm() <= 28) {
3178 assert((MI->getOperand(0).getImm() - 19) % 2 == 0 &&
3179 "Register paired with LR must be odd");
3180 TS->emitARM64WinCFISaveLRPair(Reg: MI->getOperand(i: 0).getImm(),
3181 Offset: MI->getOperand(i: 2).getImm());
3182 return;
3183 }
3184 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
3185 "Non-consecutive registers not allowed for save_regp");
3186 TS->emitARM64WinCFISaveRegP(Reg: MI->getOperand(i: 0).getImm(),
3187 Offset: MI->getOperand(i: 2).getImm());
3188 return;
3189
3190 case AArch64::SEH_SaveRegP_X:
3191 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
3192 "Non-consecutive registers not allowed for save_regp_x");
3193 assert(MI->getOperand(2).getImm() < 0 &&
3194 "Pre increment SEH opcode must have a negative offset");
3195 TS->emitARM64WinCFISaveRegPX(Reg: MI->getOperand(i: 0).getImm(),
3196 Offset: -MI->getOperand(i: 2).getImm());
3197 return;
3198
3199 case AArch64::SEH_SaveFReg:
3200 TS->emitARM64WinCFISaveFReg(Reg: MI->getOperand(i: 0).getImm(),
3201 Offset: MI->getOperand(i: 1).getImm());
3202 return;
3203
3204 case AArch64::SEH_SaveFReg_X:
3205 assert(MI->getOperand(1).getImm() < 0 &&
3206 "Pre increment SEH opcode must have a negative offset");
3207 TS->emitARM64WinCFISaveFRegX(Reg: MI->getOperand(i: 0).getImm(),
3208 Offset: -MI->getOperand(i: 1).getImm());
3209 return;
3210
3211 case AArch64::SEH_SaveFRegP:
3212 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
3213 "Non-consecutive registers not allowed for save_regp");
3214 TS->emitARM64WinCFISaveFRegP(Reg: MI->getOperand(i: 0).getImm(),
3215 Offset: MI->getOperand(i: 2).getImm());
3216 return;
3217
3218 case AArch64::SEH_SaveFRegP_X:
3219 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
3220 "Non-consecutive registers not allowed for save_regp_x");
3221 assert(MI->getOperand(2).getImm() < 0 &&
3222 "Pre increment SEH opcode must have a negative offset");
3223 TS->emitARM64WinCFISaveFRegPX(Reg: MI->getOperand(i: 0).getImm(),
3224 Offset: -MI->getOperand(i: 2).getImm());
3225 return;
3226
3227 case AArch64::SEH_SetFP:
3228 TS->emitARM64WinCFISetFP();
3229 return;
3230
3231 case AArch64::SEH_AddFP:
3232 TS->emitARM64WinCFIAddFP(Size: MI->getOperand(i: 0).getImm());
3233 return;
3234
3235 case AArch64::SEH_Nop:
3236 TS->emitARM64WinCFINop();
3237 return;
3238
3239 case AArch64::SEH_PrologEnd:
3240 TS->emitARM64WinCFIPrologEnd();
3241 return;
3242
3243 case AArch64::SEH_EpilogStart:
3244 TS->emitARM64WinCFIEpilogStart();
3245 return;
3246
3247 case AArch64::SEH_EpilogEnd:
3248 TS->emitARM64WinCFIEpilogEnd();
3249 return;
3250
3251 case AArch64::SEH_PACSignLR:
3252 TS->emitARM64WinCFIPACSignLR();
3253 return;
3254
3255 case AArch64::SEH_SaveAnyRegQP:
3256 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
3257 "Non-consecutive registers not allowed for save_any_reg");
3258 assert(MI->getOperand(2).getImm() >= 0 &&
3259 "SaveAnyRegQP SEH opcode offset must be non-negative");
3260 assert(MI->getOperand(2).getImm() <= 1008 &&
3261 "SaveAnyRegQP SEH opcode offset must fit into 6 bits");
3262 TS->emitARM64WinCFISaveAnyRegQP(Reg: MI->getOperand(i: 0).getImm(),
3263 Offset: MI->getOperand(i: 2).getImm());
3264 return;
3265
3266 case AArch64::SEH_SaveAnyRegQPX:
3267 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
3268 "Non-consecutive registers not allowed for save_any_reg");
3269 assert(MI->getOperand(2).getImm() < 0 &&
3270 "SaveAnyRegQPX SEH opcode offset must be negative");
3271 assert(MI->getOperand(2).getImm() >= -1008 &&
3272 "SaveAnyRegQPX SEH opcode offset must fit into 6 bits");
3273 TS->emitARM64WinCFISaveAnyRegQPX(Reg: MI->getOperand(i: 0).getImm(),
3274 Offset: -MI->getOperand(i: 2).getImm());
3275 return;
3276
3277 case AArch64::SEH_AllocZ:
3278 assert(MI->getOperand(0).getImm() >= 0 &&
3279 "AllocZ SEH opcode offset must be non-negative");
3280 assert(MI->getOperand(0).getImm() <= 255 &&
3281 "AllocZ SEH opcode offset must fit into 8 bits");
3282 TS->emitARM64WinCFIAllocZ(Offset: MI->getOperand(i: 0).getImm());
3283 return;
3284
3285 case AArch64::SEH_SaveZReg:
3286 assert(MI->getOperand(1).getImm() >= 0 &&
3287 "SaveZReg SEH opcode offset must be non-negative");
3288 assert(MI->getOperand(1).getImm() <= 255 &&
3289 "SaveZReg SEH opcode offset must fit into 8 bits");
3290 TS->emitARM64WinCFISaveZReg(Reg: MI->getOperand(i: 0).getImm(),
3291 Offset: MI->getOperand(i: 1).getImm());
3292 return;
3293
3294 case AArch64::SEH_SavePReg:
3295 assert(MI->getOperand(1).getImm() >= 0 &&
3296 "SavePReg SEH opcode offset must be non-negative");
3297 assert(MI->getOperand(1).getImm() <= 255 &&
3298 "SavePReg SEH opcode offset must fit into 8 bits");
3299 TS->emitARM64WinCFISavePReg(Reg: MI->getOperand(i: 0).getImm(),
3300 Offset: MI->getOperand(i: 1).getImm());
3301 return;
3302
3303 case AArch64::BLR:
3304 case AArch64::BR: {
3305 recordIfImportCall(BranchInst: MI);
3306 MCInst TmpInst;
3307 MCInstLowering.Lower(MI, OutMI&: TmpInst);
3308 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
3309 return;
3310 }
3311 case AArch64::CBWPri:
3312 case AArch64::CBXPri:
3313 case AArch64::CBWPrr:
3314 case AArch64::CBXPrr:
3315 emitCBPseudoExpansion(MI);
3316 return;
3317 }
3318
3319 // Finally, do the automated lowerings for everything else.
3320 MCInst TmpInst;
3321 MCInstLowering.Lower(MI, OutMI&: TmpInst);
3322 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
3323}
3324
3325void AArch64AsmPrinter::recordIfImportCall(
3326 const llvm::MachineInstr *BranchInst) {
3327 if (!EnableImportCallOptimization)
3328 return;
3329
3330 auto [GV, OpFlags] = BranchInst->getMF()->tryGetCalledGlobal(MI: BranchInst);
3331 if (GV && GV->hasDLLImportStorageClass()) {
3332 auto *CallSiteSymbol = MMI->getContext().createNamedTempSymbol(Name: "impcall");
3333 OutStreamer->emitLabel(Symbol: CallSiteSymbol);
3334
3335 auto *CalledSymbol = MCInstLowering.GetGlobalValueSymbol(GV, TargetFlags: OpFlags);
3336 SectionToImportedFunctionCalls[OutStreamer->getCurrentSectionOnly()]
3337 .push_back(x: {CallSiteSymbol, CalledSymbol});
3338 }
3339}
3340
3341void AArch64AsmPrinter::emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI,
3342 MCSymbol *LazyPointer) {
3343 // _ifunc:
3344 // adrp x16, lazy_pointer@GOTPAGE
3345 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF]
3346 // ldr x16, [x16]
3347 // br x16
3348
3349 {
3350 MCInst Adrp;
3351 Adrp.setOpcode(AArch64::ADRP);
3352 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
3353 MCOperand SymPage;
3354 MCInstLowering.lowerOperand(
3355 MO: MachineOperand::CreateMCSymbol(Sym: LazyPointer,
3356 TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGE),
3357 MCOp&: SymPage);
3358 Adrp.addOperand(Op: SymPage);
3359 EmitToStreamer(Inst: Adrp);
3360 }
3361
3362 {
3363 MCInst Ldr;
3364 Ldr.setOpcode(AArch64::LDRXui);
3365 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
3366 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
3367 MCOperand SymPageOff;
3368 MCInstLowering.lowerOperand(
3369 MO: MachineOperand::CreateMCSymbol(Sym: LazyPointer, TargetFlags: AArch64II::MO_GOT |
3370 AArch64II::MO_PAGEOFF),
3371 MCOp&: SymPageOff);
3372 Ldr.addOperand(Op: SymPageOff);
3373 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
3374 EmitToStreamer(Inst: Ldr);
3375 }
3376
3377 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
3378 .addReg(Reg: AArch64::X16)
3379 .addReg(Reg: AArch64::X16)
3380 .addImm(Val: 0));
3381
3382 EmitToStreamer(Inst: MCInstBuilder(TM.getTargetTriple().isArm64e() ? AArch64::BRAAZ
3383 : AArch64::BR)
3384 .addReg(Reg: AArch64::X16));
3385}
3386
3387void AArch64AsmPrinter::emitMachOIFuncStubHelperBody(Module &M,
3388 const GlobalIFunc &GI,
3389 MCSymbol *LazyPointer) {
3390 // These stub helpers are only ever called once, so here we're optimizing for
3391 // minimum size by using the pre-indexed store variants, which saves a few
3392 // bytes of instructions to bump & restore sp.
3393
3394 // _ifunc.stub_helper:
3395 // stp fp, lr, [sp, #-16]!
3396 // mov fp, sp
3397 // stp x1, x0, [sp, #-16]!
3398 // stp x3, x2, [sp, #-16]!
3399 // stp x5, x4, [sp, #-16]!
3400 // stp x7, x6, [sp, #-16]!
3401 // stp d1, d0, [sp, #-16]!
3402 // stp d3, d2, [sp, #-16]!
3403 // stp d5, d4, [sp, #-16]!
3404 // stp d7, d6, [sp, #-16]!
3405 // bl _resolver
3406 // adrp x16, lazy_pointer@GOTPAGE
3407 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF]
3408 // str x0, [x16]
3409 // mov x16, x0
3410 // ldp d7, d6, [sp], #16
3411 // ldp d5, d4, [sp], #16
3412 // ldp d3, d2, [sp], #16
3413 // ldp d1, d0, [sp], #16
3414 // ldp x7, x6, [sp], #16
3415 // ldp x5, x4, [sp], #16
3416 // ldp x3, x2, [sp], #16
3417 // ldp x1, x0, [sp], #16
3418 // ldp fp, lr, [sp], #16
3419 // br x16
3420
3421 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPXpre)
3422 .addReg(Reg: AArch64::SP)
3423 .addReg(Reg: AArch64::FP)
3424 .addReg(Reg: AArch64::LR)
3425 .addReg(Reg: AArch64::SP)
3426 .addImm(Val: -2));
3427
3428 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
3429 .addReg(Reg: AArch64::FP)
3430 .addReg(Reg: AArch64::SP)
3431 .addImm(Val: 0)
3432 .addImm(Val: 0));
3433
3434 for (int I = 0; I != 4; ++I)
3435 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPXpre)
3436 .addReg(Reg: AArch64::SP)
3437 .addReg(Reg: AArch64::X1 + 2 * I)
3438 .addReg(Reg: AArch64::X0 + 2 * I)
3439 .addReg(Reg: AArch64::SP)
3440 .addImm(Val: -2));
3441
3442 for (int I = 0; I != 4; ++I)
3443 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPDpre)
3444 .addReg(Reg: AArch64::SP)
3445 .addReg(Reg: AArch64::D1 + 2 * I)
3446 .addReg(Reg: AArch64::D0 + 2 * I)
3447 .addReg(Reg: AArch64::SP)
3448 .addImm(Val: -2));
3449
3450 EmitToStreamer(
3451 Inst: MCInstBuilder(AArch64::BL)
3452 .addOperand(Op: MCOperand::createExpr(Val: lowerConstant(CV: GI.getResolver()))));
3453
3454 {
3455 MCInst Adrp;
3456 Adrp.setOpcode(AArch64::ADRP);
3457 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
3458 MCOperand SymPage;
3459 MCInstLowering.lowerOperand(
3460 MO: MachineOperand::CreateES(SymName: LazyPointer->getName().data() + 1,
3461 TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGE),
3462 MCOp&: SymPage);
3463 Adrp.addOperand(Op: SymPage);
3464 EmitToStreamer(Inst: Adrp);
3465 }
3466
3467 {
3468 MCInst Ldr;
3469 Ldr.setOpcode(AArch64::LDRXui);
3470 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
3471 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
3472 MCOperand SymPageOff;
3473 MCInstLowering.lowerOperand(
3474 MO: MachineOperand::CreateES(SymName: LazyPointer->getName().data() + 1,
3475 TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGEOFF),
3476 MCOp&: SymPageOff);
3477 Ldr.addOperand(Op: SymPageOff);
3478 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
3479 EmitToStreamer(Inst: Ldr);
3480 }
3481
3482 EmitToStreamer(Inst: MCInstBuilder(AArch64::STRXui)
3483 .addReg(Reg: AArch64::X0)
3484 .addReg(Reg: AArch64::X16)
3485 .addImm(Val: 0));
3486
3487 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
3488 .addReg(Reg: AArch64::X16)
3489 .addReg(Reg: AArch64::X0)
3490 .addImm(Val: 0)
3491 .addImm(Val: 0));
3492
3493 for (int I = 3; I != -1; --I)
3494 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDPDpost)
3495 .addReg(Reg: AArch64::SP)
3496 .addReg(Reg: AArch64::D1 + 2 * I)
3497 .addReg(Reg: AArch64::D0 + 2 * I)
3498 .addReg(Reg: AArch64::SP)
3499 .addImm(Val: 2));
3500
3501 for (int I = 3; I != -1; --I)
3502 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDPXpost)
3503 .addReg(Reg: AArch64::SP)
3504 .addReg(Reg: AArch64::X1 + 2 * I)
3505 .addReg(Reg: AArch64::X0 + 2 * I)
3506 .addReg(Reg: AArch64::SP)
3507 .addImm(Val: 2));
3508
3509 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDPXpost)
3510 .addReg(Reg: AArch64::SP)
3511 .addReg(Reg: AArch64::FP)
3512 .addReg(Reg: AArch64::LR)
3513 .addReg(Reg: AArch64::SP)
3514 .addImm(Val: 2));
3515
3516 EmitToStreamer(Inst: MCInstBuilder(TM.getTargetTriple().isArm64e() ? AArch64::BRAAZ
3517 : AArch64::BR)
3518 .addReg(Reg: AArch64::X16));
3519}
3520
3521const MCExpr *AArch64AsmPrinter::lowerConstant(const Constant *CV,
3522 const Constant *BaseCV,
3523 uint64_t Offset) {
3524 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: CV)) {
3525 return MCSymbolRefExpr::create(Symbol: MCInstLowering.GetGlobalValueSymbol(GV, TargetFlags: 0),
3526 Ctx&: OutContext);
3527 }
3528
3529 return AsmPrinter::lowerConstant(CV, BaseCV, Offset);
3530}
3531
3532char AArch64AsmPrinter::ID = 0;
3533
3534INITIALIZE_PASS(AArch64AsmPrinter, "aarch64-asm-printer",
3535 "AArch64 Assembly Printer", false, false)
3536
3537// Force static initialization.
3538extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
3539LLVMInitializeAArch64AsmPrinter() {
3540 RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
3541 RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
3542 RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
3543 RegisterAsmPrinter<AArch64AsmPrinter> W(getTheARM64_32Target());
3544 RegisterAsmPrinter<AArch64AsmPrinter> V(getTheAArch64_32Target());
3545}
3546