1//===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a printer that converts from our internal representation
10// of machine-dependent LLVM code to the AArch64 assembly language.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AArch64AsmPrinter.h"
15#include "AArch64.h"
16#include "AArch64MCInstLower.h"
17#include "AArch64MachineFunctionInfo.h"
18#include "AArch64RegisterInfo.h"
19#include "AArch64Subtarget.h"
20#include "AArch64TargetObjectFile.h"
21#include "MCTargetDesc/AArch64AddressingModes.h"
22#include "MCTargetDesc/AArch64InstPrinter.h"
23#include "MCTargetDesc/AArch64MCAsmInfo.h"
24#include "MCTargetDesc/AArch64MCTargetDesc.h"
25#include "MCTargetDesc/AArch64TargetStreamer.h"
26#include "TargetInfo/AArch64TargetInfo.h"
27#include "Utils/AArch64BaseInfo.h"
28#include "llvm/ADT/ScopeExit.h"
29#include "llvm/ADT/SmallString.h"
30#include "llvm/ADT/SmallVector.h"
31#include "llvm/ADT/Statistic.h"
32#include "llvm/ADT/StringRef.h"
33#include "llvm/ADT/Twine.h"
34#include "llvm/BinaryFormat/COFF.h"
35#include "llvm/BinaryFormat/ELF.h"
36#include "llvm/BinaryFormat/MachO.h"
37#include "llvm/CodeGen/AsmPrinter.h"
38#include "llvm/CodeGen/AsmPrinterAnalysis.h"
39#include "llvm/CodeGen/FaultMaps.h"
40#include "llvm/CodeGen/MachineBasicBlock.h"
41#include "llvm/CodeGen/MachineFunction.h"
42#include "llvm/CodeGen/MachineInstr.h"
43#include "llvm/CodeGen/MachineJumpTableInfo.h"
44#include "llvm/CodeGen/MachineModuleInfoImpls.h"
45#include "llvm/CodeGen/MachineOperand.h"
46#include "llvm/CodeGen/StackMaps.h"
47#include "llvm/CodeGen/TargetRegisterInfo.h"
48#include "llvm/IR/Analysis.h"
49#include "llvm/IR/DataLayout.h"
50#include "llvm/IR/DebugInfoMetadata.h"
51#include "llvm/IR/Mangler.h"
52#include "llvm/IR/Module.h"
53#include "llvm/IR/PassManager.h"
54#include "llvm/MC/MCAsmInfo.h"
55#include "llvm/MC/MCContext.h"
56#include "llvm/MC/MCExpr.h"
57#include "llvm/MC/MCInst.h"
58#include "llvm/MC/MCInstBuilder.h"
59#include "llvm/MC/MCSectionELF.h"
60#include "llvm/MC/MCSectionMachO.h"
61#include "llvm/MC/MCStreamer.h"
62#include "llvm/MC/MCSymbol.h"
63#include "llvm/MC/MCValue.h"
64#include "llvm/MC/TargetRegistry.h"
65#include "llvm/Support/Casting.h"
66#include "llvm/Support/CommandLine.h"
67#include "llvm/Support/Compiler.h"
68#include "llvm/Support/ErrorHandling.h"
69#include "llvm/Support/raw_ostream.h"
70#include "llvm/Target/TargetMachine.h"
71#include "llvm/TargetParser/Triple.h"
72#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
73#include <cassert>
74#include <cstdint>
75#include <map>
76#include <memory>
77
78using namespace llvm;
79
80#define DEBUG_TYPE "AArch64AsmPrinter"
81
82// Doesn't count FPR128 ZCZ instructions which are handled
83// by TableGen pattern matching
84STATISTIC(NumZCZeroingInstrsFPR,
85 "Number of zero-cycle FPR zeroing instructions expanded from "
86 "canonical pseudo instructions");
87
88enum PtrauthCheckMode { Unchecked, Poison, Trap };
89static cl::opt<PtrauthCheckMode> PtrauthAuthChecks(
90 "aarch64-ptrauth-auth-checks", cl::Hidden,
91 cl::values(clEnumValN(Unchecked, "none", "don't test for failure"),
92 clEnumValN(Poison, "poison", "poison on failure"),
93 clEnumValN(Trap, "trap", "trap on failure")),
94 cl::desc("Check pointer authentication auth/resign failures"));
95
96namespace {
97
98class AArch64AsmPrinter : public AsmPrinter {
99 AArch64MCInstLower MCInstLowering;
100 FaultMaps FM;
101 const AArch64Subtarget *STI;
102 bool ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = false;
103 bool PtrauthInitFini = false;
104 bool PtrauthInitFiniAddressDisc = false;
105#ifndef NDEBUG
106 unsigned InstsEmitted;
107#endif
108 bool EnableImportCallOptimization = false;
109 MapVector<MCSection *, std::vector<std::pair<MCSymbol *, MCSymbol *>>>
110 SectionToImportedFunctionCalls;
111 unsigned PAuthIFuncNextUniqueID = 1;
112
113public:
114 static char ID;
115
116 AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
117 : AsmPrinter(TM, std::move(Streamer), ID),
118 MCInstLowering(OutContext, *this), FM(*this) {}
119
120 StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
121
122 /// Wrapper for MCInstLowering.lowerOperand() for the
123 /// tblgen'erated pseudo lowering.
124 bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
125 return MCInstLowering.lowerOperand(MO, MCOp);
126 }
127
128 const MCExpr *lowerConstantPtrAuth(const ConstantPtrAuth &CPA) override;
129
130 const MCExpr *lowerBlockAddressConstant(const BlockAddress &BA) override;
131
132 void emitStartOfAsmFile(Module &M) override;
133 void emitJumpTableImpl(const MachineJumpTableInfo &MJTI,
134 ArrayRef<unsigned> JumpTableIndices) override;
135 std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
136 codeview::JumpTableEntrySize>
137 getCodeViewJumpTableInfo(int JTI, const MachineInstr *BranchInstr,
138 const MCSymbol *BranchLabel) const override;
139
140 void emitFunctionEntryLabel() override;
141
142 void emitXXStructor(const DataLayout &DL, const Constant *CV) override;
143
144 void LowerJumpTableDest(MCStreamer &OutStreamer, const MachineInstr &MI);
145
146 void LowerHardenedBRJumpTable(const MachineInstr &MI);
147
148 void LowerMOPS(MCStreamer &OutStreamer, const MachineInstr &MI);
149
150 void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
151 const MachineInstr &MI);
152 void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
153 const MachineInstr &MI);
154 void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
155 const MachineInstr &MI);
156 void LowerFAULTING_OP(const MachineInstr &MI);
157
158 void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
159 void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
160 void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
161 void LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI, bool Typed);
162
163 typedef std::tuple<unsigned, bool, uint32_t, bool, uint64_t>
164 HwasanMemaccessTuple;
165 std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
166 void LowerKCFI_CHECK(const MachineInstr &MI);
167 void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
168 void emitHwasanMemaccessSymbols(Module &M);
169
170 void emitSled(const MachineInstr &MI, SledKind Kind);
171
172 // Returns whether Reg may be used to store sensitive temporary values when
173 // expanding PtrAuth pseudos. Some OSes may take extra care to protect a
174 // small subset of GPRs on context switches - use these registers then.
175 //
176 // If there are no preferred registers, returns true for any Reg.
177 bool isPtrauthRegSafe(Register Reg) const {
178 if (STI->isX16X17Safer())
179 return Reg == AArch64::X16 || Reg == AArch64::X17;
180
181 return true;
182 }
183
184 // Emit the sequence for BRA/BLRA (authenticate + branch/call).
185 void emitPtrauthBranch(const MachineInstr *MI);
186
187 void emitPtrauthCheckAuthenticatedValue(Register TestedReg,
188 Register ScratchReg,
189 AArch64PACKey::ID Key,
190 AArch64PAuth::AuthCheckMethod Method,
191 const MCSymbol *OnFailure = nullptr);
192
193 // Check authenticated LR before tail calling.
194 void emitPtrauthTailCallHardening(const MachineInstr *TC);
195
196 struct PtrAuthSchema {
197 static PtrAuthSchema CreateImmReg(AArch64PACKey::ID Key, uint64_t IntDisc,
198 const MachineOperand &AddrDiscOp);
199 static PtrAuthSchema CreateRegReg(AArch64PACKey::ID Key, Register AddrDisc,
200 Register PCDisc);
201
202 AArch64PACKey::ID Key;
203 uint64_t IntDisc;
204 Register AddrDisc;
205 bool AddrDiscIsKilled;
206 Register PCDisc;
207
208 bool addrDiscIsKilledAndNoneOf(std::initializer_list<Register> Regs) {
209 return AddrDiscIsKilled && !llvm::is_contained(Set: Regs, Element: AddrDisc);
210 }
211 };
212
213 // Helper for emitting AUTRELLOADPAC: increment Pointer by Addend and then by
214 // a 32-bit signed value loaded from memory. The instructions emitted are
215 //
216 // ldrsw Scratch, [Pointer, #Addend]!
217 // add Pointer, Pointer, Scratch
218 //
219 // for small Addend value, with longer sequences required for wider Addend.
220 void emitPtrauthApplyIndirectAddend(Register Pointer, Register Scratch,
221 int64_t Addend);
222
223 // Emit the sequence for AUT or AUTPAC (or their PC-blending variants).
224 // Addend is only used for AUTRELLOADPAC.
225 void emitPtrauthAuthResign(Register Pointer, Register Scratch,
226 PtrAuthSchema AuthSchema,
227 std::optional<PtrAuthSchema> SignSchema,
228 std::optional<int64_t> Addend, Value *DS);
229
230 // Emit R_AARCH64_PATCHINST, the deactivation symbol relocation. Returns true
231 // if no instruction should be emitted because the deactivation symbol is
232 // defined in the current module so this function emitted a NOP instead.
233 bool emitDeactivationSymbolRelocation(Value *DS);
234
235 // Emit the sequence for PAC.
236 void emitPtrauthSign(const MachineInstr *MI);
237
238 // Emit the sequence to compute the discriminator.
239 //
240 // The Scratch register passed to this function must be safe, as returned by
241 // isPtrauthRegSafe(ScratchReg).
242 //
243 // The returned register is either ScratchReg, AddrDisc, or XZR. Furthermore,
244 // it is guaranteed to be safe (or XZR), with the only exception of
245 // passing-through an *unmodified* unsafe AddrDisc register.
246 //
247 // If the expanded pseudo is allowed to clobber AddrDisc register, setting
248 // MayClobberAddrDisc may save one MOV instruction, provided
249 // isPtrauthRegSafe(AddrDisc) is true:
250 //
251 // mov x17, x16
252 // movk x17, #1234, lsl #48
253 // ; x16 is not used anymore
254 //
255 // can be replaced by
256 //
257 // movk x16, #1234, lsl #48
258 Register emitPtrauthDiscriminator(uint64_t Disc, Register AddrDisc,
259 Register ScratchReg,
260 bool MayClobberAddrDisc = false);
261
262 // Emit the sequence for LOADauthptrstatic
263 void LowerLOADauthptrstatic(const MachineInstr &MI);
264
265 // Emit the sequence for LOADgotPAC/MOVaddrPAC (either GOT adrp-ldr or
266 // adrp-add followed by PAC sign)
267 void LowerMOVaddrPAC(const MachineInstr &MI);
268
269 // Emit the sequence for LOADgotAUTH (load signed pointer from signed ELF GOT
270 // and authenticate it with, if FPAC bit is not set, check+trap sequence after
271 // authenticating)
272 void LowerLOADgotAUTH(const MachineInstr &MI);
273
274 void emitAddImm(MCRegister Val, int64_t Addend, MCRegister Tmp);
275 void emitAddress(MCRegister Reg, const MCExpr *Expr, MCRegister Tmp,
276 bool DSOLocal, const MCSubtargetInfo &STI);
277
278 const MCExpr *emitPAuthRelocationAsIRelative(
279 const MCExpr *Target, uint64_t Disc, AArch64PACKey::ID KeyID,
280 bool HasAddressDiversity, bool IsDSOLocal, const MCExpr *DSExpr);
281
282 /// tblgen'erated driver function for lowering simple MI->MC
283 /// pseudo instructions.
284 bool lowerPseudoInstExpansion(const MachineInstr *MI, MCInst &Inst);
285
286 // Emit Build Attributes
287 void emitAttributes(unsigned Flags, uint64_t PAuthABIPlatform,
288 uint64_t PAuthABIVersion, AArch64TargetStreamer *TS);
289
290 // Emit expansion of Compare-and-branch pseudo instructions
291 void emitCBPseudoExpansion(const MachineInstr *MI);
292
293 void EmitToStreamer(MCStreamer &S, const MCInst &Inst);
294 void EmitToStreamer(const MCInst &Inst) {
295 EmitToStreamer(S&: *OutStreamer, Inst);
296 }
297
298 void emitInstruction(const MachineInstr *MI) override;
299
300 void emitFunctionHeaderComment() override;
301
302 void getAnalysisUsage(AnalysisUsage &AU) const override {
303 AsmPrinter::getAnalysisUsage(AU);
304 AU.setPreservesAll();
305 }
306
307 bool runOnMachineFunction(MachineFunction &MF) override {
308 if (auto *PSIW = getAnalysisIfAvailable<ProfileSummaryInfoWrapperPass>())
309 PSI = &PSIW->getPSI();
310 if (auto *SDPIW =
311 getAnalysisIfAvailable<StaticDataProfileInfoWrapperPass>())
312 SDPI = &SDPIW->getStaticDataProfileInfo();
313
314 AArch64FI = MF.getInfo<AArch64FunctionInfo>();
315 STI = &MF.getSubtarget<AArch64Subtarget>();
316
317 SetupMachineFunction(MF);
318
319 if (STI->isTargetCOFF()) {
320 bool Local = MF.getFunction().hasLocalLinkage();
321 COFF::SymbolStorageClass Scl =
322 Local ? COFF::IMAGE_SYM_CLASS_STATIC : COFF::IMAGE_SYM_CLASS_EXTERNAL;
323 int Type =
324 COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
325
326 OutStreamer->beginCOFFSymbolDef(Symbol: CurrentFnSym);
327 OutStreamer->emitCOFFSymbolStorageClass(StorageClass: Scl);
328 OutStreamer->emitCOFFSymbolType(Type);
329 OutStreamer->endCOFFSymbolDef();
330 }
331
332 // Emit the rest of the function body.
333 emitFunctionBody();
334
335 // Emit the XRay table for this function.
336 emitXRayTable();
337
338 // We didn't modify anything.
339 return false;
340 }
341
342 const MCExpr *lowerConstant(const Constant *CV,
343 const Constant *BaseCV = nullptr,
344 uint64_t Offset = 0) override;
345
346private:
347 void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
348 bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
349 bool printAsmRegInClass(const MachineOperand &MO,
350 const TargetRegisterClass *RC, unsigned AltName,
351 raw_ostream &O);
352
353 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
354 const char *ExtraCode, raw_ostream &O) override;
355 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
356 const char *ExtraCode, raw_ostream &O) override;
357
358 void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
359
360 void emitFunctionBodyEnd() override;
361 void emitGlobalAlias(const Module &M, const GlobalAlias &GA) override;
362
363 MCSymbol *GetCPISymbol(unsigned CPID) const override;
364 void emitEndOfAsmFile(Module &M) override;
365
366 AArch64FunctionInfo *AArch64FI = nullptr;
367
368 /// Emit the LOHs contained in AArch64FI.
369 void emitLOHs();
370
371 void emitMovXReg(Register Dest, Register Src);
372 void emitMOVZ(Register Dest, uint64_t Imm, unsigned Shift);
373 void emitMOVK(Register Dest, uint64_t Imm, unsigned Shift);
374
375 void emitAUT(AArch64PACKey::ID Key, Register Pointer, Register Disc);
376 void emitPAC(AArch64PACKey::ID Key, Register Pointer, Register Disc);
377 void emitBLRA(bool IsCall, AArch64PACKey::ID Key, Register Target,
378 Register Disc);
379
380 /// Emit instruction to set float register to zero.
381 void emitFMov0(const MachineInstr &MI);
382 void emitFMov0AsFMov(const MachineInstr &MI, Register DestReg);
383
384 using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
385
386 MInstToMCSymbol LOHInstToLabel;
387
388 bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const override {
389 return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags;
390 }
391
392 const MCSubtargetInfo *getIFuncMCSubtargetInfo() const override {
393 assert(STI);
394 return STI;
395 }
396 void emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI,
397 MCSymbol *LazyPointer) override;
398 void emitMachOIFuncStubHelperBody(Module &M, const GlobalIFunc &GI,
399 MCSymbol *LazyPointer) override;
400
401 /// Checks if this instruction is part of a sequence that is eligle for import
402 /// call optimization and, if so, records it to be emitted in the import call
403 /// section.
404 void recordIfImportCall(const MachineInstr *BranchInst);
405};
406
407} // end anonymous namespace
408
409// Get boolean module flag (0 or 1), treating absent flag as having value 0.
410static bool getOptionalBooleanModuleFlag(Module &M, StringRef Name) {
411 Metadata *Flag = M.getModuleFlag(Key: Name);
412 if (!Flag)
413 return false;
414
415 uint64_t Value = mdconst::extract<ConstantInt>(MD&: Flag)->getZExtValue();
416 assert((Value == 0 || Value == 1) && "Boolean flag is expected, if present");
417 return Value;
418}
419
420void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
421 const Triple &TT = TM.getTargetTriple();
422
423 if (TT.isOSBinFormatCOFF()) {
424 emitCOFFFeatureSymbol(M);
425 emitCOFFReplaceableFunctionData(M);
426
427 if (M.getModuleFlag(Key: "import-call-optimization"))
428 EnableImportCallOptimization = true;
429 }
430
431 PtrauthInitFini = getOptionalBooleanModuleFlag(M, Name: "ptrauth-init-fini");
432 PtrauthInitFiniAddressDisc = getOptionalBooleanModuleFlag(
433 M, Name: "ptrauth-init-fini-address-discrimination");
434
435 if (!TT.isOSBinFormatELF())
436 return;
437
438 // For emitting build attributes and .note.gnu.property section
439 auto *TS =
440 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
441 // Assemble feature flags that may require creation of build attributes and a
442 // note section.
443 unsigned BAFlags = 0;
444 unsigned GNUFlags = 0;
445 if (const auto *BTE = mdconst::extract_or_null<ConstantInt>(
446 MD: M.getModuleFlag(Key: "branch-target-enforcement"))) {
447 if (!BTE->isZero()) {
448 BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_BTI_Flag;
449 GNUFlags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
450 }
451 }
452
453 if (const auto *GCS = mdconst::extract_or_null<ConstantInt>(
454 MD: M.getModuleFlag(Key: "guarded-control-stack"))) {
455 if (!GCS->isZero()) {
456 BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_GCS_Flag;
457 GNUFlags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_GCS;
458 }
459 }
460
461 if (const auto *Sign = mdconst::extract_or_null<ConstantInt>(
462 MD: M.getModuleFlag(Key: "sign-return-address"))) {
463 if (!Sign->isZero()) {
464 BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_PAC_Flag;
465 GNUFlags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
466 }
467 }
468
469 uint64_t PAuthABIPlatform = -1;
470 if (const auto *PAP = mdconst::extract_or_null<ConstantInt>(
471 MD: M.getModuleFlag(Key: "aarch64-elf-pauthabi-platform"))) {
472 PAuthABIPlatform = PAP->getZExtValue();
473 }
474
475 uint64_t PAuthABIVersion = -1;
476 if (const auto *PAV = mdconst::extract_or_null<ConstantInt>(
477 MD: M.getModuleFlag(Key: "aarch64-elf-pauthabi-version"))) {
478 PAuthABIVersion = PAV->getZExtValue();
479 }
480
481 // For LLVM_LINUX experimental platform, version value of 0 means no PAuth
482 // support. Do not emit corresponding PAuthABI GNU property note and AArch64
483 // build attributes for this case to keep Linux binaries not using PAuth
484 // unaffected.
485 if (PAuthABIPlatform == ELF::AARCH64_PAUTH_PLATFORM_LLVM_LINUX &&
486 PAuthABIVersion == 0) {
487 PAuthABIPlatform = uint64_t(-1);
488 PAuthABIVersion = uint64_t(-1);
489 }
490
491 // Emit AArch64 Build Attributes
492 emitAttributes(Flags: BAFlags, PAuthABIPlatform, PAuthABIVersion, TS);
493 // Emit a .note.gnu.property section with the flags.
494 TS->emitNoteSection(Flags: GNUFlags, PAuthABIPlatform, PAuthABIVersion);
495}
496
497void AArch64AsmPrinter::emitFunctionHeaderComment() {
498 const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>();
499 std::optional<std::string> OutlinerString = FI->getOutliningStyle();
500 if (OutlinerString != std::nullopt)
501 OutStreamer->getCommentOS() << ' ' << OutlinerString;
502}
503
504void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
505{
506 const Function &F = MF->getFunction();
507 if (F.hasFnAttribute(Kind: "patchable-function-entry")) {
508 unsigned Num;
509 if (F.getFnAttribute(Kind: "patchable-function-entry")
510 .getValueAsString()
511 .getAsInteger(Radix: 10, Result&: Num))
512 return;
513 emitNops(N: Num);
514 return;
515 }
516
517 emitSled(MI, Kind: SledKind::FUNCTION_ENTER);
518}
519
520void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI) {
521 emitSled(MI, Kind: SledKind::FUNCTION_EXIT);
522}
523
524void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) {
525 emitSled(MI, Kind: SledKind::TAIL_CALL);
526}
527
528void AArch64AsmPrinter::emitSled(const MachineInstr &MI, SledKind Kind) {
529 static const int8_t NoopsInSledCount = 7;
530 // We want to emit the following pattern:
531 //
532 // .Lxray_sled_N:
533 // ALIGN
534 // B #32
535 // ; 7 NOP instructions (28 bytes)
536 // .tmpN
537 //
538 // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
539 // over the full 32 bytes (8 instructions) with the following pattern:
540 //
541 // STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
542 // LDR W17, #12 ; W17 := function ID
543 // LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
544 // BLR X16 ; call the tracing trampoline
545 // ;DATA: 32 bits of function ID
546 // ;DATA: lower 32 bits of the address of the trampoline
547 // ;DATA: higher 32 bits of the address of the trampoline
548 // LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
549 //
550 OutStreamer->emitCodeAlignment(Alignment: Align(4), STI: getSubtargetInfo());
551 auto CurSled = OutContext.createTempSymbol(Name: "xray_sled_", AlwaysAddSuffix: true);
552 OutStreamer->emitLabel(Symbol: CurSled);
553 auto Target = OutContext.createTempSymbol();
554
555 // Emit "B #32" instruction, which jumps over the next 28 bytes.
556 // The operand has to be the number of 4-byte instructions to jump over,
557 // including the current instruction.
558 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::B).addImm(Val: 8));
559
560 for (int8_t I = 0; I < NoopsInSledCount; I++)
561 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::NOP));
562
563 OutStreamer->emitLabel(Symbol: Target);
564 recordSled(Sled: CurSled, MI, Kind, Version: 2);
565}
566
567void AArch64AsmPrinter::emitAttributes(unsigned Flags,
568 uint64_t PAuthABIPlatform,
569 uint64_t PAuthABIVersion,
570 AArch64TargetStreamer *TS) {
571
572 PAuthABIPlatform = (uint64_t(-1) == PAuthABIPlatform) ? 0 : PAuthABIPlatform;
573 PAuthABIVersion = (uint64_t(-1) == PAuthABIVersion) ? 0 : PAuthABIVersion;
574
575 if (PAuthABIPlatform || PAuthABIVersion) {
576 TS->emitAttributesSubsection(
577 VendorName: AArch64BuildAttributes::getVendorName(
578 Vendor: AArch64BuildAttributes::AEABI_PAUTHABI),
579 IsOptional: AArch64BuildAttributes::SubsectionOptional::REQUIRED,
580 ParameterType: AArch64BuildAttributes::SubsectionType::ULEB128);
581 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
582 Vendor: AArch64BuildAttributes::AEABI_PAUTHABI),
583 Tag: AArch64BuildAttributes::TAG_PAUTH_PLATFORM,
584 Value: PAuthABIPlatform, String: "");
585 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
586 Vendor: AArch64BuildAttributes::AEABI_PAUTHABI),
587 Tag: AArch64BuildAttributes::TAG_PAUTH_SCHEMA, Value: PAuthABIVersion,
588 String: "");
589 }
590
591 unsigned BTIValue =
592 (Flags & AArch64BuildAttributes::Feature_BTI_Flag) ? 1 : 0;
593 unsigned PACValue =
594 (Flags & AArch64BuildAttributes::Feature_PAC_Flag) ? 1 : 0;
595 unsigned GCSValue =
596 (Flags & AArch64BuildAttributes::Feature_GCS_Flag) ? 1 : 0;
597
598 if (BTIValue || PACValue || GCSValue) {
599 TS->emitAttributesSubsection(
600 VendorName: AArch64BuildAttributes::getVendorName(
601 Vendor: AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
602 IsOptional: AArch64BuildAttributes::SubsectionOptional::OPTIONAL,
603 ParameterType: AArch64BuildAttributes::SubsectionType::ULEB128);
604 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
605 Vendor: AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
606 Tag: AArch64BuildAttributes::TAG_FEATURE_BTI, Value: BTIValue, String: "");
607 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
608 Vendor: AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
609 Tag: AArch64BuildAttributes::TAG_FEATURE_PAC, Value: PACValue, String: "");
610 TS->emitAttribute(VendorName: AArch64BuildAttributes::getVendorName(
611 Vendor: AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
612 Tag: AArch64BuildAttributes::TAG_FEATURE_GCS, Value: GCSValue, String: "");
613 }
614}
615
616// Emit the following code for Intrinsic::{xray_customevent,xray_typedevent}
617// (built-in functions __xray_customevent/__xray_typedevent).
618//
619// .Lxray_event_sled_N:
620// b 1f
621// save x0 and x1 (and also x2 for TYPED_EVENT_CALL)
622// set up x0 and x1 (and also x2 for TYPED_EVENT_CALL)
623// bl __xray_CustomEvent or __xray_TypedEvent
624// restore x0 and x1 (and also x2 for TYPED_EVENT_CALL)
625// 1:
626//
627// There are 6 instructions for EVENT_CALL and 9 for TYPED_EVENT_CALL.
628//
629// Then record a sled of kind CUSTOM_EVENT or TYPED_EVENT.
630// After patching, b .+N will become a nop.
631void AArch64AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI,
632 bool Typed) {
633 auto &O = *OutStreamer;
634 MCSymbol *CurSled = OutContext.createTempSymbol(Name: "xray_sled_", AlwaysAddSuffix: true);
635 O.emitLabel(Symbol: CurSled);
636 bool MachO = TM.getTargetTriple().isOSBinFormatMachO();
637 auto *Sym = MCSymbolRefExpr::create(
638 Symbol: OutContext.getOrCreateSymbol(
639 Name: Twine(MachO ? "_" : "") +
640 (Typed ? "__xray_TypedEvent" : "__xray_CustomEvent")),
641 Ctx&: OutContext);
642 if (Typed) {
643 O.AddComment(T: "Begin XRay typed event");
644 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::B).addImm(Val: 9));
645 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::STPXpre)
646 .addReg(Reg: AArch64::SP)
647 .addReg(Reg: AArch64::X0)
648 .addReg(Reg: AArch64::X1)
649 .addReg(Reg: AArch64::SP)
650 .addImm(Val: -4));
651 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::STRXui)
652 .addReg(Reg: AArch64::X2)
653 .addReg(Reg: AArch64::SP)
654 .addImm(Val: 2));
655 emitMovXReg(Dest: AArch64::X0, Src: MI.getOperand(i: 0).getReg());
656 emitMovXReg(Dest: AArch64::X1, Src: MI.getOperand(i: 1).getReg());
657 emitMovXReg(Dest: AArch64::X2, Src: MI.getOperand(i: 2).getReg());
658 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::BL).addExpr(Val: Sym));
659 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::LDRXui)
660 .addReg(Reg: AArch64::X2)
661 .addReg(Reg: AArch64::SP)
662 .addImm(Val: 2));
663 O.AddComment(T: "End XRay typed event");
664 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::LDPXpost)
665 .addReg(Reg: AArch64::SP)
666 .addReg(Reg: AArch64::X0)
667 .addReg(Reg: AArch64::X1)
668 .addReg(Reg: AArch64::SP)
669 .addImm(Val: 4));
670
671 recordSled(Sled: CurSled, MI, Kind: SledKind::TYPED_EVENT, Version: 2);
672 } else {
673 O.AddComment(T: "Begin XRay custom event");
674 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::B).addImm(Val: 6));
675 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::STPXpre)
676 .addReg(Reg: AArch64::SP)
677 .addReg(Reg: AArch64::X0)
678 .addReg(Reg: AArch64::X1)
679 .addReg(Reg: AArch64::SP)
680 .addImm(Val: -2));
681 emitMovXReg(Dest: AArch64::X0, Src: MI.getOperand(i: 0).getReg());
682 emitMovXReg(Dest: AArch64::X1, Src: MI.getOperand(i: 1).getReg());
683 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::BL).addExpr(Val: Sym));
684 O.AddComment(T: "End XRay custom event");
685 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::LDPXpost)
686 .addReg(Reg: AArch64::SP)
687 .addReg(Reg: AArch64::X0)
688 .addReg(Reg: AArch64::X1)
689 .addReg(Reg: AArch64::SP)
690 .addImm(Val: 2));
691
692 recordSled(Sled: CurSled, MI, Kind: SledKind::CUSTOM_EVENT, Version: 2);
693 }
694}
695
696void AArch64AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
697 Register AddrReg = MI.getOperand(i: 0).getReg();
698 assert(std::next(MI.getIterator())->isCall() &&
699 "KCFI_CHECK not followed by a call instruction");
700 assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg &&
701 "KCFI_CHECK call target doesn't match call operand");
702
703 // Default to using the intra-procedure-call temporary registers for
704 // comparing the hashes.
705 unsigned ScratchRegs[] = {AArch64::W16, AArch64::W17};
706 if (AddrReg == AArch64::XZR) {
707 // Checking XZR makes no sense. Instead of emitting a load, zero
708 // ScratchRegs[0] and use it for the ESR AddrIndex below.
709 AddrReg = getXRegFromWReg(Reg: ScratchRegs[0]);
710 emitMovXReg(Dest: AddrReg, Src: AArch64::XZR);
711 } else {
712 // If one of the scratch registers is used for the call target (e.g.
713 // with AArch64::TCRETURNriBTI), we can clobber another caller-saved
714 // temporary register instead (in this case, AArch64::W9) as the check
715 // is immediately followed by the call instruction.
716 for (auto &Reg : ScratchRegs) {
717 if (Reg == getWRegFromXReg(Reg: AddrReg)) {
718 Reg = AArch64::W9;
719 break;
720 }
721 }
722 assert(ScratchRegs[0] != AddrReg && ScratchRegs[1] != AddrReg &&
723 "Invalid scratch registers for KCFI_CHECK");
724
725 // Adjust the offset for patchable-function-prefix. This assumes that
726 // patchable-function-prefix is the same for all functions.
727 int64_t PrefixNops =
728 MI.getMF()->getFunction().getFnAttributeAsParsedInteger(
729 Kind: "patchable-function-prefix");
730
731 // Load the target function type hash.
732 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::LDURWi)
733 .addReg(Reg: ScratchRegs[0])
734 .addReg(Reg: AddrReg)
735 .addImm(Val: -(PrefixNops * 4 + 4)));
736 }
737
738 // Load the expected type hash.
739 const int64_t Type = MI.getOperand(i: 1).getImm();
740 emitMOVK(Dest: ScratchRegs[1], Imm: Type & 0xFFFF, Shift: 0);
741 emitMOVK(Dest: ScratchRegs[1], Imm: (Type >> 16) & 0xFFFF, Shift: 16);
742
743 // Compare the hashes and trap if there's a mismatch.
744 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::SUBSWrs)
745 .addReg(Reg: AArch64::WZR)
746 .addReg(Reg: ScratchRegs[0])
747 .addReg(Reg: ScratchRegs[1])
748 .addImm(Val: 0));
749
750 MCSymbol *Pass = OutContext.createTempSymbol();
751 EmitToStreamer(S&: *OutStreamer,
752 Inst: MCInstBuilder(AArch64::Bcc)
753 .addImm(Val: AArch64CC::EQ)
754 .addExpr(Val: MCSymbolRefExpr::create(Symbol: Pass, Ctx&: OutContext)));
755
756 // The base ESR is 0x8000 and the register information is encoded in bits
757 // 0-9 as follows:
758 // - 0-4: n, where the register Xn contains the target address
759 // - 5-9: m, where the register Wm contains the expected type hash
760 // Where n, m are in [0, 30].
761 unsigned TypeIndex = ScratchRegs[1] - AArch64::W0;
762 unsigned AddrIndex;
763 switch (AddrReg) {
764 default:
765 AddrIndex = AddrReg - AArch64::X0;
766 break;
767 case AArch64::FP:
768 AddrIndex = 29;
769 break;
770 case AArch64::LR:
771 AddrIndex = 30;
772 break;
773 }
774
775 assert(AddrIndex < 31 && TypeIndex < 31);
776
777 unsigned ESR = 0x8000 | ((TypeIndex & 31) << 5) | (AddrIndex & 31);
778 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::BRK).addImm(Val: ESR));
779 OutStreamer->emitLabel(Symbol: Pass);
780}
781
782void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
783 Register Reg = MI.getOperand(i: 0).getReg();
784
785 // The HWASan pass won't emit a CHECK_MEMACCESS intrinsic with a pointer
786 // statically known to be zero. However, conceivably, the HWASan pass may
787 // encounter a "cannot currently statically prove to be null" pointer (and is
788 // therefore unable to omit the intrinsic) that later optimization passes
789 // convert into a statically known-null pointer.
790 if (Reg == AArch64::XZR)
791 return;
792
793 bool IsShort =
794 ((MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES) ||
795 (MI.getOpcode() ==
796 AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW));
797 uint32_t AccessInfo = MI.getOperand(i: 1).getImm();
798 bool IsFixedShadow =
799 ((MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_FIXEDSHADOW) ||
800 (MI.getOpcode() ==
801 AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW));
802 uint64_t FixedShadowOffset = IsFixedShadow ? MI.getOperand(i: 2).getImm() : 0;
803
804 MCSymbol *&Sym = HwasanMemaccessSymbols[HwasanMemaccessTuple(
805 Reg, IsShort, AccessInfo, IsFixedShadow, FixedShadowOffset)];
806 if (!Sym) {
807 // FIXME: Make this work on non-ELF.
808 if (!TM.getTargetTriple().isOSBinFormatELF())
809 report_fatal_error(reason: "llvm.hwasan.check.memaccess only supported on ELF");
810
811 std::string SymName = "__hwasan_check_x" + utostr(X: Reg - AArch64::X0) + "_" +
812 utostr(X: AccessInfo);
813 if (IsFixedShadow)
814 SymName += "_fixed_" + utostr(X: FixedShadowOffset);
815 if (IsShort)
816 SymName += "_short_v2";
817 Sym = OutContext.getOrCreateSymbol(Name: SymName);
818 }
819
820 EmitToStreamer(S&: *OutStreamer,
821 Inst: MCInstBuilder(AArch64::BL)
822 .addExpr(Val: MCSymbolRefExpr::create(Symbol: Sym, Ctx&: OutContext)));
823}
824
825void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) {
826 if (HwasanMemaccessSymbols.empty())
827 return;
828
829 const Triple &TT = TM.getTargetTriple();
830 assert(TT.isOSBinFormatELF());
831 // AArch64Subtarget is huge, so heap allocate it so we don't run out of stack
832 // space.
833 auto STI = std::make_unique<AArch64Subtarget>(
834 args: TT, args: TM.getTargetCPU(), args: TM.getTargetCPU(), args: TM.getTargetFeatureString(), args&: TM,
835 args: true);
836 this->STI = STI.get();
837
838 MCSymbol *HwasanTagMismatchV1Sym =
839 OutContext.getOrCreateSymbol(Name: "__hwasan_tag_mismatch");
840 MCSymbol *HwasanTagMismatchV2Sym =
841 OutContext.getOrCreateSymbol(Name: "__hwasan_tag_mismatch_v2");
842
843 const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
844 MCSymbolRefExpr::create(Symbol: HwasanTagMismatchV1Sym, Ctx&: OutContext);
845 const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
846 MCSymbolRefExpr::create(Symbol: HwasanTagMismatchV2Sym, Ctx&: OutContext);
847
848 for (auto &P : HwasanMemaccessSymbols) {
849 unsigned Reg = std::get<0>(t: P.first);
850 bool IsShort = std::get<1>(t: P.first);
851 uint32_t AccessInfo = std::get<2>(t: P.first);
852 bool IsFixedShadow = std::get<3>(t: P.first);
853 uint64_t FixedShadowOffset = std::get<4>(t: P.first);
854 const MCSymbolRefExpr *HwasanTagMismatchRef =
855 IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
856 MCSymbol *Sym = P.second;
857
858 bool HasMatchAllTag =
859 (AccessInfo >> HWASanAccessInfo::HasMatchAllShift) & 1;
860 uint8_t MatchAllTag =
861 (AccessInfo >> HWASanAccessInfo::MatchAllShift) & 0xff;
862 unsigned Size =
863 1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf);
864 bool CompileKernel =
865 (AccessInfo >> HWASanAccessInfo::CompileKernelShift) & 1;
866
867 OutStreamer->switchSection(Section: OutContext.getELFSection(
868 Section: ".text.hot", Type: ELF::SHT_PROGBITS,
869 Flags: ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, EntrySize: 0, Group: Sym->getName(),
870 /*IsComdat=*/true));
871
872 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_ELF_TypeFunction);
873 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Weak);
874 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Hidden);
875 OutStreamer->emitLabel(Symbol: Sym);
876
877 EmitToStreamer(Inst: MCInstBuilder(AArch64::SBFMXri)
878 .addReg(Reg: AArch64::X16)
879 .addReg(Reg)
880 .addImm(Val: 4)
881 .addImm(Val: 55));
882
883 if (IsFixedShadow) {
884 // Aarch64 makes it difficult to embed large constants in the code.
885 // Fortuitously, kShadowBaseAlignment == 32, so we use the 32-bit
886 // left-shift option in the MOV instruction. Combined with the 16-bit
887 // immediate, this is enough to represent any offset up to 2**48.
888 emitMOVZ(Dest: AArch64::X17, Imm: FixedShadowOffset >> 32, Shift: 32);
889 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRBBroX)
890 .addReg(Reg: AArch64::W16)
891 .addReg(Reg: AArch64::X17)
892 .addReg(Reg: AArch64::X16)
893 .addImm(Val: 0)
894 .addImm(Val: 0));
895 } else {
896 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRBBroX)
897 .addReg(Reg: AArch64::W16)
898 .addReg(Reg: IsShort ? AArch64::X20 : AArch64::X9)
899 .addReg(Reg: AArch64::X16)
900 .addImm(Val: 0)
901 .addImm(Val: 0));
902 }
903
904 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSXrs)
905 .addReg(Reg: AArch64::XZR)
906 .addReg(Reg: AArch64::X16)
907 .addReg(Reg)
908 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSR, Imm: 56)));
909 MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
910 EmitToStreamer(Inst: MCInstBuilder(AArch64::Bcc)
911 .addImm(Val: AArch64CC::NE)
912 .addExpr(Val: MCSymbolRefExpr::create(
913 Symbol: HandleMismatchOrPartialSym, Ctx&: OutContext)));
914 MCSymbol *ReturnSym = OutContext.createTempSymbol();
915 OutStreamer->emitLabel(Symbol: ReturnSym);
916 EmitToStreamer(Inst: MCInstBuilder(AArch64::RET).addReg(Reg: AArch64::LR));
917 OutStreamer->emitLabel(Symbol: HandleMismatchOrPartialSym);
918
919 if (HasMatchAllTag) {
920 EmitToStreamer(Inst: MCInstBuilder(AArch64::UBFMXri)
921 .addReg(Reg: AArch64::X17)
922 .addReg(Reg)
923 .addImm(Val: 56)
924 .addImm(Val: 63));
925 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSXri)
926 .addReg(Reg: AArch64::XZR)
927 .addReg(Reg: AArch64::X17)
928 .addImm(Val: MatchAllTag)
929 .addImm(Val: 0));
930 EmitToStreamer(
931 Inst: MCInstBuilder(AArch64::Bcc)
932 .addImm(Val: AArch64CC::EQ)
933 .addExpr(Val: MCSymbolRefExpr::create(Symbol: ReturnSym, Ctx&: OutContext)));
934 }
935
936 if (IsShort) {
937 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSWri)
938 .addReg(Reg: AArch64::WZR)
939 .addReg(Reg: AArch64::W16)
940 .addImm(Val: 15)
941 .addImm(Val: 0));
942 MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
943 EmitToStreamer(
944 Inst: MCInstBuilder(AArch64::Bcc)
945 .addImm(Val: AArch64CC::HI)
946 .addExpr(Val: MCSymbolRefExpr::create(Symbol: HandleMismatchSym, Ctx&: OutContext)));
947
948 EmitToStreamer(Inst: MCInstBuilder(AArch64::ANDXri)
949 .addReg(Reg: AArch64::X17)
950 .addReg(Reg)
951 .addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: 0xf, regSize: 64)));
952 if (Size != 1)
953 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
954 .addReg(Reg: AArch64::X17)
955 .addReg(Reg: AArch64::X17)
956 .addImm(Val: Size - 1)
957 .addImm(Val: 0));
958 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSWrs)
959 .addReg(Reg: AArch64::WZR)
960 .addReg(Reg: AArch64::W16)
961 .addReg(Reg: AArch64::W17)
962 .addImm(Val: 0));
963 EmitToStreamer(
964 Inst: MCInstBuilder(AArch64::Bcc)
965 .addImm(Val: AArch64CC::LS)
966 .addExpr(Val: MCSymbolRefExpr::create(Symbol: HandleMismatchSym, Ctx&: OutContext)));
967
968 EmitToStreamer(Inst: MCInstBuilder(AArch64::ORRXri)
969 .addReg(Reg: AArch64::X16)
970 .addReg(Reg)
971 .addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: 0xf, regSize: 64)));
972 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRBBui)
973 .addReg(Reg: AArch64::W16)
974 .addReg(Reg: AArch64::X16)
975 .addImm(Val: 0));
976 EmitToStreamer(
977 Inst: MCInstBuilder(AArch64::SUBSXrs)
978 .addReg(Reg: AArch64::XZR)
979 .addReg(Reg: AArch64::X16)
980 .addReg(Reg)
981 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSR, Imm: 56)));
982 EmitToStreamer(
983 Inst: MCInstBuilder(AArch64::Bcc)
984 .addImm(Val: AArch64CC::EQ)
985 .addExpr(Val: MCSymbolRefExpr::create(Symbol: ReturnSym, Ctx&: OutContext)));
986
987 OutStreamer->emitLabel(Symbol: HandleMismatchSym);
988 }
989
990 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPXpre)
991 .addReg(Reg: AArch64::SP)
992 .addReg(Reg: AArch64::X0)
993 .addReg(Reg: AArch64::X1)
994 .addReg(Reg: AArch64::SP)
995 .addImm(Val: -32));
996 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPXi)
997 .addReg(Reg: AArch64::FP)
998 .addReg(Reg: AArch64::LR)
999 .addReg(Reg: AArch64::SP)
1000 .addImm(Val: 29));
1001
1002 if (Reg != AArch64::X0)
1003 emitMovXReg(Dest: AArch64::X0, Src: Reg);
1004 emitMOVZ(Dest: AArch64::X1, Imm: AccessInfo & HWASanAccessInfo::RuntimeMask, Shift: 0);
1005
1006 if (CompileKernel) {
1007 // The Linux kernel's dynamic loader doesn't support GOT relative
1008 // relocations, but it doesn't support late binding either, so just call
1009 // the function directly.
1010 EmitToStreamer(Inst: MCInstBuilder(AArch64::B).addExpr(Val: HwasanTagMismatchRef));
1011 } else {
1012 // Intentionally load the GOT entry and branch to it, rather than possibly
1013 // late binding the function, which may clobber the registers before we
1014 // have a chance to save them.
1015 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADRP)
1016 .addReg(Reg: AArch64::X16)
1017 .addExpr(Val: MCSpecifierExpr::create(Expr: HwasanTagMismatchRef,
1018 S: AArch64::S_GOT_PAGE,
1019 Ctx&: OutContext)));
1020 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
1021 .addReg(Reg: AArch64::X16)
1022 .addReg(Reg: AArch64::X16)
1023 .addExpr(Val: MCSpecifierExpr::create(Expr: HwasanTagMismatchRef,
1024 S: AArch64::S_GOT_LO12,
1025 Ctx&: OutContext)));
1026 EmitToStreamer(Inst: MCInstBuilder(AArch64::BR).addReg(Reg: AArch64::X16));
1027 }
1028 }
1029 this->STI = nullptr;
1030}
1031
1032static void emitAuthenticatedPointer(MCStreamer &OutStreamer,
1033 MCSymbol *StubLabel,
1034 const MCExpr *StubAuthPtrRef) {
1035 // sym$auth_ptr$key$disc:
1036 OutStreamer.emitLabel(Symbol: StubLabel);
1037 OutStreamer.emitValue(Value: StubAuthPtrRef, /*size=*/Size: 8);
1038}
1039
1040void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
1041 emitHwasanMemaccessSymbols(M);
1042
1043 const Triple &TT = TM.getTargetTriple();
1044 if (TT.isOSBinFormatMachO()) {
1045 // Output authenticated pointers as indirect symbols, if we have any.
1046 MachineModuleInfoMachO &MMIMacho =
1047 MMI->getObjFileInfo<MachineModuleInfoMachO>();
1048
1049 auto Stubs = MMIMacho.getAuthGVStubList();
1050
1051 if (!Stubs.empty()) {
1052 // Switch to the "__auth_ptr" section.
1053 OutStreamer->switchSection(
1054 Section: OutContext.getMachOSection(Segment: "__DATA", Section: "__auth_ptr", TypeAndAttributes: MachO::S_REGULAR,
1055 K: SectionKind::getMetadata()));
1056 emitAlignment(Alignment: Align(8));
1057
1058 for (const auto &Stub : Stubs)
1059 emitAuthenticatedPointer(OutStreamer&: *OutStreamer, StubLabel: Stub.first, StubAuthPtrRef: Stub.second);
1060
1061 OutStreamer->addBlankLine();
1062 }
1063
1064 // Funny Darwin hack: This flag tells the linker that no global symbols
1065 // contain code that falls through to other global symbols (e.g. the obvious
1066 // implementation of multiple entry points). If this doesn't occur, the
1067 // linker can safely perform dead code stripping. Since LLVM never
1068 // generates code that does this, it is always safe to set.
1069 OutStreamer->emitSubsectionsViaSymbols();
1070 }
1071
1072 if (TT.isOSBinFormatELF()) {
1073 // Output authenticated pointers as indirect symbols, if we have any.
1074 MachineModuleInfoELF &MMIELF = MMI->getObjFileInfo<MachineModuleInfoELF>();
1075
1076 auto Stubs = MMIELF.getAuthGVStubList();
1077
1078 if (!Stubs.empty()) {
1079 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
1080 OutStreamer->switchSection(Section: TLOF.getDataSection());
1081 emitAlignment(Alignment: Align(8));
1082
1083 for (const auto &Stub : Stubs)
1084 emitAuthenticatedPointer(OutStreamer&: *OutStreamer, StubLabel: Stub.first, StubAuthPtrRef: Stub.second);
1085
1086 OutStreamer->addBlankLine();
1087 }
1088
1089 // With signed ELF GOT enabled, the linker looks at the symbol type to
1090 // choose between keys IA (for STT_FUNC) and DA (for other types). Symbols
1091 // for functions not defined in the module have STT_NOTYPE type by default.
1092 // This makes linker to emit signing schema with DA key (instead of IA) for
1093 // corresponding R_AARCH64_AUTH_GLOB_DAT dynamic reloc. To avoid that, force
1094 // all function symbols used in the module to have STT_FUNC type. See
1095 // https://github.com/ARM-software/abi-aa/blob/main/pauthabielf64/pauthabielf64.rst#default-signing-schema
1096 const auto *PtrAuthELFGOTFlag = mdconst::extract_or_null<ConstantInt>(
1097 MD: M.getModuleFlag(Key: "ptrauth-elf-got"));
1098 if (PtrAuthELFGOTFlag && PtrAuthELFGOTFlag->getZExtValue() == 1)
1099 for (const GlobalValue &GV : M.global_values())
1100 if (!GV.use_empty() && isa<Function>(Val: GV) &&
1101 !GV.getName().starts_with(Prefix: "llvm."))
1102 OutStreamer->emitSymbolAttribute(Symbol: getSymbol(GV: &GV),
1103 Attribute: MCSA_ELF_TypeFunction);
1104 }
1105
1106 // Emit stack and fault map information.
1107 FM.serializeToFaultMapSection();
1108
1109 // If import call optimization is enabled, emit the appropriate section.
1110 // We do this whether or not we recorded any import calls.
1111 if (EnableImportCallOptimization && TT.isOSBinFormatCOFF()) {
1112 OutStreamer->switchSection(Section: getObjFileLowering().getImportCallSection());
1113
1114 // Section always starts with some magic.
1115 constexpr char ImpCallMagic[12] = "Imp_Call_V1";
1116 OutStreamer->emitBytes(Data: StringRef{ImpCallMagic, sizeof(ImpCallMagic)});
1117
1118 // Layout of this section is:
1119 // Per section that contains calls to imported functions:
1120 // uint32_t SectionSize: Size in bytes for information in this section.
1121 // uint32_t Section Number
1122 // Per call to imported function in section:
1123 // uint32_t Kind: the kind of imported function.
1124 // uint32_t BranchOffset: the offset of the branch instruction in its
1125 // parent section.
1126 // uint32_t TargetSymbolId: the symbol id of the called function.
1127 for (auto &[Section, CallsToImportedFuncs] :
1128 SectionToImportedFunctionCalls) {
1129 unsigned SectionSize =
1130 sizeof(uint32_t) * (2 + 3 * CallsToImportedFuncs.size());
1131 OutStreamer->emitInt32(Value: SectionSize);
1132 OutStreamer->emitCOFFSecNumber(Symbol: Section->getBeginSymbol());
1133 for (auto &[CallsiteSymbol, CalledSymbol] : CallsToImportedFuncs) {
1134 // Kind is always IMAGE_REL_ARM64_DYNAMIC_IMPORT_CALL (0x13).
1135 OutStreamer->emitInt32(Value: 0x13);
1136 OutStreamer->emitCOFFSecOffset(Symbol: CallsiteSymbol);
1137 OutStreamer->emitCOFFSymbolIndex(Symbol: CalledSymbol);
1138 }
1139 }
1140 }
1141}
1142
1143void AArch64AsmPrinter::emitLOHs() {
1144 SmallVector<MCSymbol *, 3> MCArgs;
1145
1146 for (const auto &D : AArch64FI->getLOHContainer()) {
1147 for (const MachineInstr *MI : D.getArgs()) {
1148 MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(x: MI);
1149 assert(LabelIt != LOHInstToLabel.end() &&
1150 "Label hasn't been inserted for LOH related instruction");
1151 MCArgs.push_back(Elt: LabelIt->second);
1152 }
1153 OutStreamer->emitLOHDirective(Kind: D.getKind(), Args: MCArgs);
1154 MCArgs.clear();
1155 }
1156}
1157
1158void AArch64AsmPrinter::emitFunctionBodyEnd() {
1159 if (!AArch64FI->getLOHRelated().empty())
1160 emitLOHs();
1161}
1162
1163/// GetCPISymbol - Return the symbol for the specified constant pool entry.
1164MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
1165 // Darwin uses a linker-private symbol name for constant-pools (to
1166 // avoid addends on the relocation?), ELF has no such concept and
1167 // uses a normal private symbol.
1168 if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
1169 return OutContext.getOrCreateSymbol(
1170 Name: Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
1171 Twine(getFunctionNumber()) + "_" + Twine(CPID));
1172
1173 return AsmPrinter::GetCPISymbol(CPID);
1174}
1175
1176void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
1177 raw_ostream &O) {
1178 const MachineOperand &MO = MI->getOperand(i: OpNum);
1179 switch (MO.getType()) {
1180 default:
1181 llvm_unreachable("<unknown operand type>");
1182 case MachineOperand::MO_Register: {
1183 Register Reg = MO.getReg();
1184 assert(Reg.isPhysical());
1185 assert(!MO.getSubReg() && "Subregs should be eliminated!");
1186 O << AArch64InstPrinter::getRegisterName(Reg);
1187 break;
1188 }
1189 case MachineOperand::MO_Immediate: {
1190 O << MO.getImm();
1191 break;
1192 }
1193 case MachineOperand::MO_GlobalAddress: {
1194 PrintSymbolOperand(MO, OS&: O);
1195 break;
1196 }
1197 case MachineOperand::MO_BlockAddress: {
1198 MCSymbol *Sym = GetBlockAddressSymbol(BA: MO.getBlockAddress());
1199 Sym->print(OS&: O, MAI);
1200 break;
1201 }
1202 }
1203}
1204
1205bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
1206 raw_ostream &O) {
1207 Register Reg = MO.getReg();
1208 switch (Mode) {
1209 default:
1210 return true; // Unknown mode.
1211 case 'w':
1212 Reg = getWRegFromXReg(Reg);
1213 break;
1214 case 'x':
1215 Reg = getXRegFromWReg(Reg);
1216 break;
1217 case 't':
1218 Reg = getXRegFromXRegTuple(RegTuple: Reg);
1219 break;
1220 }
1221
1222 O << AArch64InstPrinter::getRegisterName(Reg);
1223 return false;
1224}
1225
1226// Prints the register in MO using class RC using the offset in the
1227// new register class. This should not be used for cross class
1228// printing.
1229bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
1230 const TargetRegisterClass *RC,
1231 unsigned AltName, raw_ostream &O) {
1232 assert(MO.isReg() && "Should only get here with a register!");
1233 const TargetRegisterInfo *RI = STI->getRegisterInfo();
1234 Register Reg = MO.getReg();
1235 MCRegister RegToPrint = RC->getRegister(i: RI->getEncodingValue(Reg));
1236 if (!RI->regsOverlap(RegA: RegToPrint, RegB: Reg))
1237 return true;
1238 O << AArch64InstPrinter::getRegisterName(Reg: RegToPrint, AltIdx: AltName);
1239 return false;
1240}
1241
1242bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
1243 const char *ExtraCode, raw_ostream &O) {
1244 const MachineOperand &MO = MI->getOperand(i: OpNum);
1245
1246 // First try the generic code, which knows about modifiers like 'c' and 'n'.
1247 if (!AsmPrinter::PrintAsmOperand(MI, OpNo: OpNum, ExtraCode, OS&: O))
1248 return false;
1249
1250 // Does this asm operand have a single letter operand modifier?
1251 if (ExtraCode && ExtraCode[0]) {
1252 if (ExtraCode[1] != 0)
1253 return true; // Unknown modifier.
1254
1255 switch (ExtraCode[0]) {
1256 default:
1257 return true; // Unknown modifier.
1258 case 'w': // Print W register
1259 case 'x': // Print X register
1260 if (MO.isReg())
1261 return printAsmMRegister(MO, Mode: ExtraCode[0], O);
1262 if (MO.isImm() && MO.getImm() == 0) {
1263 unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
1264 O << AArch64InstPrinter::getRegisterName(Reg);
1265 return false;
1266 }
1267 printOperand(MI, OpNum, O);
1268 return false;
1269 case 'b': // Print B register.
1270 case 'h': // Print H register.
1271 case 's': // Print S register.
1272 case 'd': // Print D register.
1273 case 'q': // Print Q register.
1274 case 'z': // Print Z register.
1275 if (MO.isReg()) {
1276 const TargetRegisterClass *RC;
1277 switch (ExtraCode[0]) {
1278 case 'b':
1279 RC = &AArch64::FPR8RegClass;
1280 break;
1281 case 'h':
1282 RC = &AArch64::FPR16RegClass;
1283 break;
1284 case 's':
1285 RC = &AArch64::FPR32RegClass;
1286 break;
1287 case 'd':
1288 RC = &AArch64::FPR64RegClass;
1289 break;
1290 case 'q':
1291 RC = &AArch64::FPR128RegClass;
1292 break;
1293 case 'z':
1294 RC = &AArch64::ZPRRegClass;
1295 break;
1296 default:
1297 return true;
1298 }
1299 return printAsmRegInClass(MO, RC, AltName: AArch64::NoRegAltName, O);
1300 }
1301 printOperand(MI, OpNum, O);
1302 return false;
1303 }
1304 }
1305
1306 // According to ARM, we should emit x and v registers unless we have a
1307 // modifier.
1308 if (MO.isReg()) {
1309 Register Reg = MO.getReg();
1310
1311 // If this is a w or x register, print an x register.
1312 if (AArch64::GPR32allRegClass.contains(Reg) ||
1313 AArch64::GPR64allRegClass.contains(Reg))
1314 return printAsmMRegister(MO, Mode: 'x', O);
1315
1316 // If this is an x register tuple, print an x register.
1317 if (AArch64::GPR64x8ClassRegClass.contains(Reg))
1318 return printAsmMRegister(MO, Mode: 't', O);
1319
1320 unsigned AltName = AArch64::NoRegAltName;
1321 const TargetRegisterClass *RegClass;
1322 if (AArch64::ZPRRegClass.contains(Reg)) {
1323 RegClass = &AArch64::ZPRRegClass;
1324 } else if (AArch64::PPRRegClass.contains(Reg)) {
1325 RegClass = &AArch64::PPRRegClass;
1326 } else if (AArch64::PNRRegClass.contains(Reg)) {
1327 RegClass = &AArch64::PNRRegClass;
1328 } else {
1329 RegClass = &AArch64::FPR128RegClass;
1330 AltName = AArch64::vreg;
1331 }
1332
1333 // If this is a b, h, s, d, or q register, print it as a v register.
1334 return printAsmRegInClass(MO, RC: RegClass, AltName, O);
1335 }
1336
1337 printOperand(MI, OpNum, O);
1338 return false;
1339}
1340
1341bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
1342 unsigned OpNum,
1343 const char *ExtraCode,
1344 raw_ostream &O) {
1345 if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
1346 return true; // Unknown modifier.
1347
1348 const MachineOperand &MO = MI->getOperand(i: OpNum);
1349 assert(MO.isReg() && "unexpected inline asm memory operand");
1350 O << "[" << AArch64InstPrinter::getRegisterName(Reg: MO.getReg()) << "]";
1351 return false;
1352}
1353
1354void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
1355 raw_ostream &OS) {
1356 unsigned NOps = MI->getNumOperands();
1357 assert(NOps == 4);
1358 OS << '\t' << MAI.getCommentString() << "DEBUG_VALUE: ";
1359 // cast away const; DIetc do not take const operands for some reason.
1360 OS << MI->getDebugVariable()->getName();
1361 OS << " <- ";
1362 // Frame address. Currently handles register +- offset only.
1363 assert(MI->isIndirectDebugValue());
1364 OS << '[';
1365 for (unsigned I = 0, E = llvm::size(Range: MI->debug_operands()); I < E; ++I) {
1366 if (I != 0)
1367 OS << ", ";
1368 printOperand(MI, OpNum: I, O&: OS);
1369 }
1370 OS << ']';
1371 OS << "+";
1372 printOperand(MI, OpNum: NOps - 2, O&: OS);
1373}
1374
1375void AArch64AsmPrinter::emitJumpTableImpl(const MachineJumpTableInfo &MJTI,
1376 ArrayRef<unsigned> JumpTableIndices) {
1377 // Fast return if there is nothing to emit to avoid creating empty sections.
1378 if (JumpTableIndices.empty())
1379 return;
1380 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
1381 const auto &F = MF->getFunction();
1382 ArrayRef<MachineJumpTableEntry> JT = MJTI.getJumpTables();
1383
1384 MCSection *ReadOnlySec = nullptr;
1385 if (TM.Options.EnableStaticDataPartitioning) {
1386 ReadOnlySec =
1387 TLOF.getSectionForJumpTable(F, TM, JTE: &JT[JumpTableIndices.front()]);
1388 } else {
1389 ReadOnlySec = TLOF.getSectionForJumpTable(F, TM);
1390 }
1391 OutStreamer->switchSection(Section: ReadOnlySec);
1392
1393 auto AFI = MF->getInfo<AArch64FunctionInfo>();
1394 for (unsigned JTI : JumpTableIndices) {
1395 const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
1396
1397 // If this jump table was deleted, ignore it.
1398 if (JTBBs.empty()) continue;
1399
1400 unsigned Size = AFI->getJumpTableEntrySize(Idx: JTI);
1401 emitAlignment(Alignment: Align(Size));
1402 OutStreamer->emitLabel(Symbol: GetJTISymbol(JTID: JTI));
1403
1404 const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(Idx: JTI);
1405 const MCExpr *Base = MCSymbolRefExpr::create(Symbol: BaseSym, Ctx&: OutContext);
1406
1407 for (auto *JTBB : JTBBs) {
1408 const MCExpr *Value =
1409 MCSymbolRefExpr::create(Symbol: JTBB->getSymbol(), Ctx&: OutContext);
1410
1411 // Each entry is:
1412 // .byte/.hword (LBB - Lbase)>>2
1413 // or plain:
1414 // .word LBB - Lbase
1415 Value = MCBinaryExpr::createSub(LHS: Value, RHS: Base, Ctx&: OutContext);
1416 if (Size != 4)
1417 Value = MCBinaryExpr::createLShr(
1418 LHS: Value, RHS: MCConstantExpr::create(Value: 2, Ctx&: OutContext), Ctx&: OutContext);
1419
1420 OutStreamer->emitValue(Value, Size);
1421 }
1422 }
1423}
1424
1425std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
1426 codeview::JumpTableEntrySize>
1427AArch64AsmPrinter::getCodeViewJumpTableInfo(int JTI,
1428 const MachineInstr *BranchInstr,
1429 const MCSymbol *BranchLabel) const {
1430 const auto AFI = MF->getInfo<AArch64FunctionInfo>();
1431 const auto Base = AArch64FI->getJumpTableEntryPCRelSymbol(Idx: JTI);
1432 codeview::JumpTableEntrySize EntrySize;
1433 switch (AFI->getJumpTableEntrySize(Idx: JTI)) {
1434 case 1:
1435 EntrySize = codeview::JumpTableEntrySize::UInt8ShiftLeft;
1436 break;
1437 case 2:
1438 EntrySize = codeview::JumpTableEntrySize::UInt16ShiftLeft;
1439 break;
1440 case 4:
1441 EntrySize = codeview::JumpTableEntrySize::Int32;
1442 break;
1443 default:
1444 llvm_unreachable("Unexpected jump table entry size");
1445 }
1446 return std::make_tuple(args: Base, args: 0, args&: BranchLabel, args&: EntrySize);
1447}
1448
1449void AArch64AsmPrinter::emitFunctionEntryLabel() {
1450 const Triple &TT = TM.getTargetTriple();
1451 if (TT.isOSBinFormatELF() &&
1452 (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall ||
1453 MF->getFunction().getCallingConv() ==
1454 CallingConv::AArch64_SVE_VectorCall ||
1455 MF->getInfo<AArch64FunctionInfo>()->isSVECC())) {
1456 auto *TS =
1457 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
1458 TS->emitDirectiveVariantPCS(Symbol: CurrentFnSym);
1459 }
1460
1461 AsmPrinter::emitFunctionEntryLabel();
1462
1463 if (TT.isWindowsArm64EC() && !MF->getFunction().hasLocalLinkage()) {
1464 // For ARM64EC targets, a function definition's name is mangled differently
1465 // from the normal symbol, emit required aliases here.
1466 auto emitFunctionAlias = [&](MCSymbol *Src, MCSymbol *Dst) {
1467 OutStreamer->emitSymbolAttribute(Symbol: Src, Attribute: MCSA_WeakAntiDep);
1468 OutStreamer->emitAssignment(
1469 Symbol: Src, Value: MCSymbolRefExpr::create(Symbol: Dst, Ctx&: MMI->getContext()));
1470 };
1471
1472 auto getSymbolFromMetadata = [&](StringRef Name) {
1473 MCSymbol *Sym = nullptr;
1474 if (MDNode *Node = MF->getFunction().getMetadata(Kind: Name)) {
1475 StringRef NameStr = cast<MDString>(Val: Node->getOperand(I: 0))->getString();
1476 Sym = MMI->getContext().getOrCreateSymbol(Name: NameStr);
1477 }
1478 return Sym;
1479 };
1480
1481 SmallVector<MDNode *> UnmangledNames;
1482 MF->getFunction().getMetadata(Kind: "arm64ec_unmangled_name", MDs&: UnmangledNames);
1483 for (MDNode *Node : UnmangledNames) {
1484 StringRef NameStr = cast<MDString>(Val: Node->getOperand(I: 0))->getString();
1485 MCSymbol *UnmangledSym = MMI->getContext().getOrCreateSymbol(Name: NameStr);
1486 if (std::optional<std::string> MangledName =
1487 getArm64ECMangledFunctionName(Name: UnmangledSym->getName())) {
1488 MCSymbol *ECMangledSym =
1489 MMI->getContext().getOrCreateSymbol(Name: *MangledName);
1490 emitFunctionAlias(UnmangledSym, ECMangledSym);
1491 }
1492 }
1493 if (MCSymbol *ECMangledSym =
1494 getSymbolFromMetadata("arm64ec_ecmangled_name"))
1495 emitFunctionAlias(ECMangledSym, CurrentFnSym);
1496 }
1497}
1498
1499void AArch64AsmPrinter::emitXXStructor(const DataLayout &DL,
1500 const Constant *CV) {
1501 LLVMContext &C = CV->getContext();
1502 assert(!isa<ConstantPtrAuth>(CV) &&
1503 "ctors/dtors are to be signed by asm printer");
1504
1505 if (PtrauthInitFini) {
1506 IntegerType *Int32Ty = IntegerType::get(C, NumBits: 32);
1507 IntegerType *Int64Ty = IntegerType::get(C, NumBits: 64);
1508 PointerType *PtrTy = PointerType::get(C, AddressSpace: 0);
1509
1510 ConstantInt *Key = ConstantInt::get(Ty: Int32Ty, V: AArch64PAuth::InitFiniKey);
1511 ConstantInt *IntDisc = ConstantInt::get(
1512 Ty: Int64Ty, V: AArch64PAuth::InitFiniPointerConstantDiscriminator);
1513 Constant *Null = ConstantPointerNull::get(T: PtrTy);
1514 Constant *AddressDisc = Null;
1515 if (PtrauthInitFiniAddressDisc) {
1516 uint64_t Marker = ConstantPtrAuth::AddrDiscriminator_CtorsDtors;
1517 AddressDisc =
1518 ConstantExpr::getIntToPtr(C: ConstantInt::get(Ty: Int64Ty, V: Marker), Ty: PtrTy);
1519 }
1520
1521 CV = ConstantPtrAuth::get(Ptr: const_cast<Constant *>(CV), Key, Disc: IntDisc,
1522 AddrDisc: AddressDisc, /*DeactivationSymbol=*/Null);
1523 }
1524
1525 // Signed pointers will be lowered by AArch64AsmPrinter::lowerConstantPtrAuth.
1526 AsmPrinter::emitXXStructor(DL, CV);
1527}
1528
1529void AArch64AsmPrinter::emitGlobalAlias(const Module &M,
1530 const GlobalAlias &GA) {
1531 if (auto F = dyn_cast_or_null<Function>(Val: GA.getAliasee())) {
1532 // Global aliases must point to a definition, but unmangled patchable
1533 // symbols are special and need to point to an undefined symbol with "EXP+"
1534 // prefix. Such undefined symbol is resolved by the linker by creating
1535 // x86 thunk that jumps back to the actual EC target.
1536 if (MDNode *Node = F->getMetadata(Kind: "arm64ec_exp_name")) {
1537 StringRef ExpStr = cast<MDString>(Val: Node->getOperand(I: 0))->getString();
1538 MCSymbol *ExpSym = MMI->getContext().getOrCreateSymbol(Name: ExpStr);
1539 MCSymbol *Sym = MMI->getContext().getOrCreateSymbol(Name: GA.getName());
1540
1541 OutStreamer->beginCOFFSymbolDef(Symbol: ExpSym);
1542 OutStreamer->emitCOFFSymbolStorageClass(StorageClass: COFF::IMAGE_SYM_CLASS_EXTERNAL);
1543 OutStreamer->emitCOFFSymbolType(Type: COFF::IMAGE_SYM_DTYPE_FUNCTION
1544 << COFF::SCT_COMPLEX_TYPE_SHIFT);
1545 OutStreamer->endCOFFSymbolDef();
1546
1547 OutStreamer->beginCOFFSymbolDef(Symbol: Sym);
1548 OutStreamer->emitCOFFSymbolStorageClass(StorageClass: COFF::IMAGE_SYM_CLASS_EXTERNAL);
1549 OutStreamer->emitCOFFSymbolType(Type: COFF::IMAGE_SYM_DTYPE_FUNCTION
1550 << COFF::SCT_COMPLEX_TYPE_SHIFT);
1551 OutStreamer->endCOFFSymbolDef();
1552 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Weak);
1553 OutStreamer->emitAssignment(
1554 Symbol: Sym, Value: MCSymbolRefExpr::create(Symbol: ExpSym, Ctx&: MMI->getContext()));
1555 return;
1556 }
1557 }
1558 AsmPrinter::emitGlobalAlias(M, GA);
1559}
1560
1561/// Small jump tables contain an unsigned byte or half, representing the offset
1562/// from the lowest-addressed possible destination to the desired basic
1563/// block. Since all instructions are 4-byte aligned, this is further compressed
1564/// by counting in instructions rather than bytes (i.e. divided by 4). So, to
1565/// materialize the correct destination we need:
1566///
1567/// adr xDest, .LBB0_0
1568/// ldrb wScratch, [xTable, xEntry] (with "lsl #1" for ldrh).
1569/// add xDest, xDest, xScratch (with "lsl #2" for smaller entries)
1570void AArch64AsmPrinter::LowerJumpTableDest(llvm::MCStreamer &OutStreamer,
1571 const llvm::MachineInstr &MI) {
1572 Register DestReg = MI.getOperand(i: 0).getReg();
1573 Register ScratchReg = MI.getOperand(i: 1).getReg();
1574 Register ScratchRegW =
1575 STI->getRegisterInfo()->getSubReg(Reg: ScratchReg, Idx: AArch64::sub_32);
1576 Register TableReg = MI.getOperand(i: 2).getReg();
1577 Register EntryReg = MI.getOperand(i: 3).getReg();
1578 int JTIdx = MI.getOperand(i: 4).getIndex();
1579 int Size = AArch64FI->getJumpTableEntrySize(Idx: JTIdx);
1580
1581 // This has to be first because the compression pass based its reachability
1582 // calculations on the start of the JumpTableDest instruction.
1583 auto Label =
1584 MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(Idx: JTIdx);
1585
1586 // If we don't already have a symbol to use as the base, use the ADR
1587 // instruction itself.
1588 if (!Label) {
1589 Label = MF->getContext().createTempSymbol();
1590 AArch64FI->setJumpTableEntryInfo(Idx: JTIdx, Size, PCRelSym: Label);
1591 OutStreamer.emitLabel(Symbol: Label);
1592 }
1593
1594 auto LabelExpr = MCSymbolRefExpr::create(Symbol: Label, Ctx&: MF->getContext());
1595 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::ADR)
1596 .addReg(Reg: DestReg)
1597 .addExpr(Val: LabelExpr));
1598
1599 // Load the number of instruction-steps to offset from the label.
1600 unsigned LdrOpcode;
1601 switch (Size) {
1602 case 1: LdrOpcode = AArch64::LDRBBroX; break;
1603 case 2: LdrOpcode = AArch64::LDRHHroX; break;
1604 case 4: LdrOpcode = AArch64::LDRSWroX; break;
1605 default:
1606 llvm_unreachable("Unknown jump table size");
1607 }
1608
1609 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(LdrOpcode)
1610 .addReg(Reg: Size == 4 ? ScratchReg : ScratchRegW)
1611 .addReg(Reg: TableReg)
1612 .addReg(Reg: EntryReg)
1613 .addImm(Val: 0)
1614 .addImm(Val: Size == 1 ? 0 : 1));
1615
1616 // Add to the already materialized base label address, multiplying by 4 if
1617 // compressed.
1618 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::ADDXrs)
1619 .addReg(Reg: DestReg)
1620 .addReg(Reg: DestReg)
1621 .addReg(Reg: ScratchReg)
1622 .addImm(Val: Size == 4 ? 0 : 2));
1623}
1624
1625void AArch64AsmPrinter::LowerHardenedBRJumpTable(const MachineInstr &MI) {
1626 const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
1627 assert(MJTI && "Can't lower jump-table dispatch without JTI");
1628
1629 const std::vector<MachineJumpTableEntry> &JTs = MJTI->getJumpTables();
1630 assert(!JTs.empty() && "Invalid JT index for jump-table dispatch");
1631
1632 // Emit:
1633 // mov x17, #<size of table> ; depending on table size, with MOVKs
1634 // cmp x16, x17 ; or #imm if table size fits in 12-bit
1635 // csel x16, x16, xzr, ls ; check for index overflow
1636 //
1637 // adrp x17, Ltable@PAGE ; materialize table address
1638 // add x17, Ltable@PAGEOFF
1639 // ldrsw x16, [x17, x16, lsl #2] ; load table entry
1640 //
1641 // Lanchor:
1642 // adr x17, Lanchor ; compute target address
1643 // add x16, x17, x16
1644 // br x16 ; branch to target
1645
1646 MachineOperand JTOp = MI.getOperand(i: 0);
1647
1648 unsigned JTI = JTOp.getIndex();
1649 assert(!AArch64FI->getJumpTableEntryPCRelSymbol(JTI) &&
1650 "unsupported compressed jump table");
1651
1652 const uint64_t NumTableEntries = JTs[JTI].MBBs.size();
1653
1654 // cmp only supports a 12-bit immediate. If we need more, materialize the
1655 // immediate, using x17 as a scratch register.
1656 uint64_t MaxTableEntry = NumTableEntries - 1;
1657 if (isUInt<12>(x: MaxTableEntry)) {
1658 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::SUBSXri)
1659 .addReg(Reg: AArch64::XZR)
1660 .addReg(Reg: AArch64::X16)
1661 .addImm(Val: MaxTableEntry)
1662 .addImm(Val: 0));
1663 } else {
1664 emitMOVZ(Dest: AArch64::X17, Imm: static_cast<uint16_t>(MaxTableEntry), Shift: 0);
1665 // It's sad that we have to manually materialize instructions, but we can't
1666 // trivially reuse the main pseudo expansion logic.
1667 // A MOVK sequence is easy enough to generate and handles the general case.
1668 for (int Offset = 16; Offset < 64; Offset += 16) {
1669 if ((MaxTableEntry >> Offset) == 0)
1670 break;
1671 emitMOVK(Dest: AArch64::X17, Imm: static_cast<uint16_t>(MaxTableEntry >> Offset),
1672 Shift: Offset);
1673 }
1674 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::SUBSXrs)
1675 .addReg(Reg: AArch64::XZR)
1676 .addReg(Reg: AArch64::X16)
1677 .addReg(Reg: AArch64::X17)
1678 .addImm(Val: 0));
1679 }
1680
1681 // This picks entry #0 on failure.
1682 // We might want to trap instead.
1683 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::CSELXr)
1684 .addReg(Reg: AArch64::X16)
1685 .addReg(Reg: AArch64::X16)
1686 .addReg(Reg: AArch64::XZR)
1687 .addImm(Val: AArch64CC::LS));
1688
1689 // Prepare the @PAGE/@PAGEOFF low/high operands.
1690 MachineOperand JTMOHi(JTOp), JTMOLo(JTOp);
1691 MCOperand JTMCHi, JTMCLo;
1692
1693 JTMOHi.setTargetFlags(AArch64II::MO_PAGE);
1694 JTMOLo.setTargetFlags(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
1695
1696 MCInstLowering.lowerOperand(MO: JTMOHi, MCOp&: JTMCHi);
1697 MCInstLowering.lowerOperand(MO: JTMOLo, MCOp&: JTMCLo);
1698
1699 EmitToStreamer(
1700 S&: *OutStreamer,
1701 Inst: MCInstBuilder(AArch64::ADRP).addReg(Reg: AArch64::X17).addOperand(Op: JTMCHi));
1702
1703 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ADDXri)
1704 .addReg(Reg: AArch64::X17)
1705 .addReg(Reg: AArch64::X17)
1706 .addOperand(Op: JTMCLo)
1707 .addImm(Val: 0));
1708
1709 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::LDRSWroX)
1710 .addReg(Reg: AArch64::X16)
1711 .addReg(Reg: AArch64::X17)
1712 .addReg(Reg: AArch64::X16)
1713 .addImm(Val: 0)
1714 .addImm(Val: 1));
1715
1716 MCSymbol *AdrLabel = MF->getContext().createTempSymbol();
1717 const auto *AdrLabelE = MCSymbolRefExpr::create(Symbol: AdrLabel, Ctx&: MF->getContext());
1718 AArch64FI->setJumpTableEntryInfo(Idx: JTI, Size: 4, PCRelSym: AdrLabel);
1719
1720 OutStreamer->emitLabel(Symbol: AdrLabel);
1721 EmitToStreamer(
1722 S&: *OutStreamer,
1723 Inst: MCInstBuilder(AArch64::ADR).addReg(Reg: AArch64::X17).addExpr(Val: AdrLabelE));
1724
1725 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ADDXrs)
1726 .addReg(Reg: AArch64::X16)
1727 .addReg(Reg: AArch64::X17)
1728 .addReg(Reg: AArch64::X16)
1729 .addImm(Val: 0));
1730
1731 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::BR).addReg(Reg: AArch64::X16));
1732}
1733
1734void AArch64AsmPrinter::LowerMOPS(llvm::MCStreamer &OutStreamer,
1735 const llvm::MachineInstr &MI) {
1736 unsigned Opcode = MI.getOpcode();
1737 assert(STI->hasMOPS());
1738 assert(STI->hasMTE() || Opcode != AArch64::MOPSMemorySetTaggingPseudo);
1739
1740 const auto Ops = [Opcode]() -> std::array<unsigned, 3> {
1741 if (Opcode == AArch64::MOPSMemoryCopyPseudo)
1742 return {AArch64::CPYFP, AArch64::CPYFM, AArch64::CPYFE};
1743 if (Opcode == AArch64::MOPSMemoryMovePseudo)
1744 return {AArch64::CPYP, AArch64::CPYM, AArch64::CPYE};
1745 if (Opcode == AArch64::MOPSMemorySetPseudo)
1746 return {AArch64::SETP, AArch64::SETM, AArch64::SETE};
1747 if (Opcode == AArch64::MOPSMemorySetTaggingPseudo)
1748 return {AArch64::SETGP, AArch64::SETGM, AArch64::MOPSSETGE};
1749 llvm_unreachable("Unhandled memory operation pseudo");
1750 }();
1751 const bool IsSet = Opcode == AArch64::MOPSMemorySetPseudo ||
1752 Opcode == AArch64::MOPSMemorySetTaggingPseudo;
1753
1754 for (auto Op : Ops) {
1755 int i = 0;
1756 auto MCIB = MCInstBuilder(Op);
1757 // Destination registers
1758 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1759 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1760 if (!IsSet)
1761 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1762 // Input registers
1763 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1764 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1765 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1766
1767 EmitToStreamer(S&: OutStreamer, Inst: MCIB);
1768 }
1769}
1770
1771void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
1772 const MachineInstr &MI) {
1773 unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
1774
1775 auto &Ctx = OutStreamer.getContext();
1776 MCSymbol *MILabel = Ctx.createTempSymbol();
1777 OutStreamer.emitLabel(Symbol: MILabel);
1778
1779 SM.recordStackMap(L: *MILabel, MI);
1780 assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1781
1782 // Scan ahead to trim the shadow.
1783 const MachineBasicBlock &MBB = *MI.getParent();
1784 MachineBasicBlock::const_iterator MII(MI);
1785 ++MII;
1786 while (NumNOPBytes > 0) {
1787 if (MII == MBB.end() || MII->isCall() ||
1788 MII->getOpcode() == AArch64::DBG_VALUE ||
1789 MII->getOpcode() == TargetOpcode::PATCHPOINT ||
1790 MII->getOpcode() == TargetOpcode::STACKMAP)
1791 break;
1792 ++MII;
1793 NumNOPBytes -= 4;
1794 }
1795
1796 // Emit nops.
1797 for (unsigned i = 0; i < NumNOPBytes; i += 4)
1798 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::NOP));
1799}
1800
1801// Lower a patchpoint of the form:
1802// [<def>], <id>, <numBytes>, <target>, <numArgs>
1803void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1804 const MachineInstr &MI) {
1805 auto &Ctx = OutStreamer.getContext();
1806 MCSymbol *MILabel = Ctx.createTempSymbol();
1807 OutStreamer.emitLabel(Symbol: MILabel);
1808 SM.recordPatchPoint(L: *MILabel, MI);
1809
1810 PatchPointOpers Opers(&MI);
1811
1812 int64_t CallTarget = Opers.getCallTarget().getImm();
1813 unsigned EncodedBytes = 0;
1814 if (CallTarget) {
1815 assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
1816 "High 16 bits of call target should be zero.");
1817 Register ScratchReg = MI.getOperand(i: Opers.getNextScratchIdx()).getReg();
1818 EncodedBytes = 16;
1819 // Materialize the jump address:
1820 emitMOVZ(Dest: ScratchReg, Imm: (CallTarget >> 32) & 0xFFFF, Shift: 32);
1821 emitMOVK(Dest: ScratchReg, Imm: (CallTarget >> 16) & 0xFFFF, Shift: 16);
1822 emitMOVK(Dest: ScratchReg, Imm: CallTarget & 0xFFFF, Shift: 0);
1823 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::BLR).addReg(Reg: ScratchReg));
1824 }
1825 // Emit padding.
1826 unsigned NumBytes = Opers.getNumPatchBytes();
1827 assert(NumBytes >= EncodedBytes &&
1828 "Patchpoint can't request size less than the length of a call.");
1829 assert((NumBytes - EncodedBytes) % 4 == 0 &&
1830 "Invalid number of NOP bytes requested!");
1831 for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
1832 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::NOP));
1833}
1834
1835void AArch64AsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1836 const MachineInstr &MI) {
1837 StatepointOpers SOpers(&MI);
1838 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
1839 assert(PatchBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1840 for (unsigned i = 0; i < PatchBytes; i += 4)
1841 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::NOP));
1842 } else {
1843 // Lower call target and choose correct opcode
1844 const MachineOperand &CallTarget = SOpers.getCallTarget();
1845 MCOperand CallTargetMCOp;
1846 unsigned CallOpcode;
1847 switch (CallTarget.getType()) {
1848 case MachineOperand::MO_GlobalAddress:
1849 case MachineOperand::MO_ExternalSymbol:
1850 MCInstLowering.lowerOperand(MO: CallTarget, MCOp&: CallTargetMCOp);
1851 CallOpcode = AArch64::BL;
1852 break;
1853 case MachineOperand::MO_Immediate:
1854 CallTargetMCOp = MCOperand::createImm(Val: CallTarget.getImm());
1855 CallOpcode = AArch64::BL;
1856 break;
1857 case MachineOperand::MO_Register:
1858 CallTargetMCOp = MCOperand::createReg(Reg: CallTarget.getReg());
1859 CallOpcode = AArch64::BLR;
1860 break;
1861 default:
1862 llvm_unreachable("Unsupported operand type in statepoint call target");
1863 break;
1864 }
1865
1866 EmitToStreamer(S&: OutStreamer,
1867 Inst: MCInstBuilder(CallOpcode).addOperand(Op: CallTargetMCOp));
1868 }
1869
1870 auto &Ctx = OutStreamer.getContext();
1871 MCSymbol *MILabel = Ctx.createTempSymbol();
1872 OutStreamer.emitLabel(Symbol: MILabel);
1873 SM.recordStatepoint(L: *MILabel, MI);
1874}
1875
1876void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) {
1877 // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
1878 // <opcode>, <operands>
1879
1880 Register DefRegister = FaultingMI.getOperand(i: 0).getReg();
1881 FaultMaps::FaultKind FK =
1882 static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(i: 1).getImm());
1883 MCSymbol *HandlerLabel = FaultingMI.getOperand(i: 2).getMBB()->getSymbol();
1884 unsigned Opcode = FaultingMI.getOperand(i: 3).getImm();
1885 unsigned OperandsBeginIdx = 4;
1886
1887 auto &Ctx = OutStreamer->getContext();
1888 MCSymbol *FaultingLabel = Ctx.createTempSymbol();
1889 OutStreamer->emitLabel(Symbol: FaultingLabel);
1890
1891 assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
1892 FM.recordFaultingOp(FaultTy: FK, FaultingLabel, HandlerLabel);
1893
1894 MCInst MI;
1895 MI.setOpcode(Opcode);
1896
1897 if (DefRegister != (Register)0)
1898 MI.addOperand(Op: MCOperand::createReg(Reg: DefRegister));
1899
1900 for (const MachineOperand &MO :
1901 llvm::drop_begin(RangeOrContainer: FaultingMI.operands(), N: OperandsBeginIdx)) {
1902 MCOperand Dest;
1903 lowerOperand(MO, MCOp&: Dest);
1904 MI.addOperand(Op: Dest);
1905 }
1906
1907 OutStreamer->AddComment(T: "on-fault: " + HandlerLabel->getName());
1908 EmitToStreamer(Inst: MI);
1909}
1910
1911void AArch64AsmPrinter::emitMovXReg(Register Dest, Register Src) {
1912 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ORRXrs)
1913 .addReg(Reg: Dest)
1914 .addReg(Reg: AArch64::XZR)
1915 .addReg(Reg: Src)
1916 .addImm(Val: 0));
1917}
1918
1919void AArch64AsmPrinter::emitMOVZ(Register Dest, uint64_t Imm, unsigned Shift) {
1920 bool Is64Bit = AArch64::GPR64RegClass.contains(Reg: Dest);
1921 EmitToStreamer(S&: *OutStreamer,
1922 Inst: MCInstBuilder(Is64Bit ? AArch64::MOVZXi : AArch64::MOVZWi)
1923 .addReg(Reg: Dest)
1924 .addImm(Val: Imm)
1925 .addImm(Val: Shift));
1926}
1927
1928void AArch64AsmPrinter::emitMOVK(Register Dest, uint64_t Imm, unsigned Shift) {
1929 bool Is64Bit = AArch64::GPR64RegClass.contains(Reg: Dest);
1930 EmitToStreamer(S&: *OutStreamer,
1931 Inst: MCInstBuilder(Is64Bit ? AArch64::MOVKXi : AArch64::MOVKWi)
1932 .addReg(Reg: Dest)
1933 .addReg(Reg: Dest)
1934 .addImm(Val: Imm)
1935 .addImm(Val: Shift));
1936}
1937
1938void AArch64AsmPrinter::emitAUT(AArch64PACKey::ID Key, Register Pointer,
1939 Register Disc) {
1940 bool IsZeroDisc = Disc == AArch64::XZR;
1941 unsigned Opcode = getAUTOpcodeForKey(K: Key, Zero: IsZeroDisc);
1942
1943 // autiza x16 ; if IsZeroDisc
1944 // autia x16, x17 ; if !IsZeroDisc
1945 MCInst AUTInst;
1946 AUTInst.setOpcode(Opcode);
1947 AUTInst.addOperand(Op: MCOperand::createReg(Reg: Pointer));
1948 AUTInst.addOperand(Op: MCOperand::createReg(Reg: Pointer));
1949 if (!IsZeroDisc)
1950 AUTInst.addOperand(Op: MCOperand::createReg(Reg: Disc));
1951
1952 EmitToStreamer(Inst: AUTInst);
1953}
1954
1955void AArch64AsmPrinter::emitPAC(AArch64PACKey::ID Key, Register Pointer,
1956 Register Disc) {
1957 bool IsZeroDisc = Disc == AArch64::XZR;
1958 unsigned Opcode = getPACOpcodeForKey(K: Key, Zero: IsZeroDisc);
1959
1960 // paciza x16 ; if IsZeroDisc
1961 // pacia x16, x17 ; if !IsZeroDisc
1962 MCInst PACInst;
1963 PACInst.setOpcode(Opcode);
1964 PACInst.addOperand(Op: MCOperand::createReg(Reg: Pointer));
1965 PACInst.addOperand(Op: MCOperand::createReg(Reg: Pointer));
1966 if (!IsZeroDisc)
1967 PACInst.addOperand(Op: MCOperand::createReg(Reg: Disc));
1968
1969 EmitToStreamer(Inst: PACInst);
1970}
1971
1972void AArch64AsmPrinter::emitBLRA(bool IsCall, AArch64PACKey::ID Key,
1973 Register Target, Register Disc) {
1974 bool IsZeroDisc = Disc == AArch64::XZR;
1975 unsigned Opcode = getBranchOpcodeForKey(IsCall, K: Key, Zero: IsZeroDisc);
1976
1977 // blraaz x16 ; if IsZeroDisc
1978 // blraa x16, x17 ; if !IsZeroDisc
1979 MCInst Inst;
1980 Inst.setOpcode(Opcode);
1981 Inst.addOperand(Op: MCOperand::createReg(Reg: Target));
1982 if (!IsZeroDisc)
1983 Inst.addOperand(Op: MCOperand::createReg(Reg: Disc));
1984 EmitToStreamer(Inst);
1985}
1986
1987void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) {
1988 Register DestReg = MI.getOperand(i: 0).getReg();
1989 if (!STI->hasZeroCycleZeroingFPWorkaround() && STI->isNeonAvailable()) {
1990 if (STI->hasZeroCycleZeroingFPR64()) {
1991 // Convert H/S register to corresponding D register
1992 const AArch64RegisterInfo *TRI = STI->getRegisterInfo();
1993 if (AArch64::FPR16RegClass.contains(Reg: DestReg))
1994 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
1995 RC: &AArch64::FPR64RegClass);
1996 else if (AArch64::FPR32RegClass.contains(Reg: DestReg))
1997 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::ssub,
1998 RC: &AArch64::FPR64RegClass);
1999 else
2000 assert(AArch64::FPR64RegClass.contains(DestReg));
2001
2002 MCInst MOVI;
2003 MOVI.setOpcode(AArch64::MOVID);
2004 MOVI.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2005 MOVI.addOperand(Op: MCOperand::createImm(Val: 0));
2006 EmitToStreamer(S&: *OutStreamer, Inst: MOVI);
2007 ++NumZCZeroingInstrsFPR;
2008 } else if (STI->hasZeroCycleZeroingFPR128()) {
2009 // Convert H/S/D register to corresponding Q register
2010 const AArch64RegisterInfo *TRI = STI->getRegisterInfo();
2011 if (AArch64::FPR16RegClass.contains(Reg: DestReg)) {
2012 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
2013 RC: &AArch64::FPR128RegClass);
2014 } else if (AArch64::FPR32RegClass.contains(Reg: DestReg)) {
2015 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::ssub,
2016 RC: &AArch64::FPR128RegClass);
2017 } else {
2018 assert(AArch64::FPR64RegClass.contains(DestReg));
2019 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::dsub,
2020 RC: &AArch64::FPR128RegClass);
2021 }
2022
2023 MCInst MOVI;
2024 MOVI.setOpcode(AArch64::MOVIv2d_ns);
2025 MOVI.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2026 MOVI.addOperand(Op: MCOperand::createImm(Val: 0));
2027 EmitToStreamer(S&: *OutStreamer, Inst: MOVI);
2028 ++NumZCZeroingInstrsFPR;
2029 } else {
2030 emitFMov0AsFMov(MI, DestReg);
2031 }
2032 } else {
2033 emitFMov0AsFMov(MI, DestReg);
2034 }
2035}
2036
2037void AArch64AsmPrinter::emitFMov0AsFMov(const MachineInstr &MI,
2038 Register DestReg) {
2039 MCInst FMov;
2040 switch (MI.getOpcode()) {
2041 default:
2042 llvm_unreachable("Unexpected opcode");
2043 case AArch64::FMOVH0:
2044 FMov.setOpcode(STI->hasFullFP16() ? AArch64::FMOVWHr : AArch64::FMOVWSr);
2045 if (!STI->hasFullFP16())
2046 DestReg = (AArch64::S0 + (DestReg - AArch64::H0));
2047 FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2048 FMov.addOperand(Op: MCOperand::createReg(Reg: AArch64::WZR));
2049 break;
2050 case AArch64::FMOVS0:
2051 FMov.setOpcode(AArch64::FMOVWSr);
2052 FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2053 FMov.addOperand(Op: MCOperand::createReg(Reg: AArch64::WZR));
2054 break;
2055 case AArch64::FMOVD0:
2056 FMov.setOpcode(AArch64::FMOVXDr);
2057 FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2058 FMov.addOperand(Op: MCOperand::createReg(Reg: AArch64::XZR));
2059 break;
2060 }
2061 EmitToStreamer(S&: *OutStreamer, Inst: FMov);
2062}
2063
2064Register AArch64AsmPrinter::emitPtrauthDiscriminator(uint64_t Disc,
2065 Register AddrDisc,
2066 Register ScratchReg,
2067 bool MayClobberAddrDisc) {
2068 assert(isPtrauthRegSafe(ScratchReg) &&
2069 "Safe scratch register must be provided by the caller");
2070 assert(isUInt<16>(Disc) && "Constant discriminator is too wide");
2071
2072 // So far we've used NoRegister in pseudos. Now we need real encodings.
2073 if (!AddrDisc.isValid())
2074 AddrDisc = AArch64::XZR;
2075
2076 // If there is no constant discriminator, there's no blend involved:
2077 // just use the address discriminator register as-is (XZR or not).
2078 if (!Disc)
2079 return AddrDisc;
2080
2081 // If there's only a constant discriminator, MOV it into the scratch register.
2082 if (AddrDisc == AArch64::XZR) {
2083 emitMOVZ(Dest: ScratchReg, Imm: Disc, Shift: 0);
2084 return ScratchReg;
2085 }
2086
2087 // If there are both, emit a blend into the scratch register.
2088
2089 // Check if we can save one MOV instruction.
2090 if (MayClobberAddrDisc && isPtrauthRegSafe(Reg: AddrDisc)) {
2091 ScratchReg = AddrDisc;
2092 } else {
2093 emitMovXReg(Dest: ScratchReg, Src: AddrDisc);
2094 assert(ScratchReg != AddrDisc &&
2095 "Forbidden to clobber AddrDisc, but have to");
2096 }
2097
2098 emitMOVK(Dest: ScratchReg, Imm: Disc, Shift: 48);
2099 return ScratchReg;
2100}
2101
2102/// Emit a code sequence to check an authenticated pointer value.
2103///
2104/// This function emits a sequence of instructions that checks if TestedReg was
2105/// authenticated successfully. On success, execution continues at the next
2106/// instruction after the sequence.
2107///
2108/// The action performed on failure depends on the OnFailure argument:
2109/// * if OnFailure is not nullptr, control is transferred to that label after
2110/// clearing the PAC field
2111/// * otherwise, BRK instruction is emitted to generate an error
2112void AArch64AsmPrinter::emitPtrauthCheckAuthenticatedValue(
2113 Register TestedReg, Register ScratchReg, AArch64PACKey::ID Key,
2114 AArch64PAuth::AuthCheckMethod Method, const MCSymbol *OnFailure) {
2115 // Insert a sequence to check if authentication of TestedReg succeeded,
2116 // such as:
2117 //
2118 // - checked and clearing:
2119 // ; x16 is TestedReg, x17 is ScratchReg
2120 // mov x17, x16
2121 // xpaci x17
2122 // cmp x16, x17
2123 // b.eq Lsuccess
2124 // mov x16, x17
2125 // b Lend
2126 // Lsuccess:
2127 // ; skipped if authentication failed
2128 // Lend:
2129 // ...
2130 //
2131 // - checked and trapping:
2132 // mov x17, x16
2133 // xpaci x17
2134 // cmp x16, x17
2135 // b.eq Lsuccess
2136 // brk #<0xc470 + aut key>
2137 // Lsuccess:
2138 // ...
2139 //
2140 // See the documentation on AuthCheckMethod enumeration constants for
2141 // the specific code sequences that can be used to perform the check.
2142 using AArch64PAuth::AuthCheckMethod;
2143
2144 if (Method == AuthCheckMethod::None)
2145 return;
2146 if (Method == AuthCheckMethod::DummyLoad) {
2147 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRWui)
2148 .addReg(Reg: getWRegFromXReg(Reg: ScratchReg))
2149 .addReg(Reg: TestedReg)
2150 .addImm(Val: 0));
2151 assert(!OnFailure && "DummyLoad always traps on error");
2152 return;
2153 }
2154
2155 MCSymbol *SuccessSym = createTempSymbol(Name: "auth_success_");
2156 if (Method == AuthCheckMethod::XPAC || Method == AuthCheckMethod::XPACHint) {
2157 // mov Xscratch, Xtested
2158 emitMovXReg(Dest: ScratchReg, Src: TestedReg);
2159
2160 if (Method == AuthCheckMethod::XPAC) {
2161 // xpac(i|d) Xscratch
2162 unsigned XPACOpc = getXPACOpcodeForKey(K: Key);
2163 EmitToStreamer(
2164 Inst: MCInstBuilder(XPACOpc).addReg(Reg: ScratchReg).addReg(Reg: ScratchReg));
2165 } else {
2166 // xpaclri
2167
2168 // Note that this method applies XPAC to TestedReg instead of ScratchReg.
2169 assert(TestedReg == AArch64::LR &&
2170 "XPACHint mode is only compatible with checking the LR register");
2171 assert((Key == AArch64PACKey::IA || Key == AArch64PACKey::IB) &&
2172 "XPACHint mode is only compatible with I-keys");
2173 EmitToStreamer(Inst: MCInstBuilder(AArch64::XPACLRI));
2174 }
2175
2176 // cmp Xtested, Xscratch
2177 EmitToStreamer(Inst: MCInstBuilder(AArch64::SUBSXrs)
2178 .addReg(Reg: AArch64::XZR)
2179 .addReg(Reg: TestedReg)
2180 .addReg(Reg: ScratchReg)
2181 .addImm(Val: 0));
2182
2183 // b.eq Lsuccess
2184 EmitToStreamer(
2185 Inst: MCInstBuilder(AArch64::Bcc)
2186 .addImm(Val: AArch64CC::EQ)
2187 .addExpr(Val: MCSymbolRefExpr::create(Symbol: SuccessSym, Ctx&: OutContext)));
2188 } else if (Method == AuthCheckMethod::HighBitsNoTBI) {
2189 // eor Xscratch, Xtested, Xtested, lsl #1
2190 EmitToStreamer(Inst: MCInstBuilder(AArch64::EORXrs)
2191 .addReg(Reg: ScratchReg)
2192 .addReg(Reg: TestedReg)
2193 .addReg(Reg: TestedReg)
2194 .addImm(Val: 1));
2195 // tbz Xscratch, #62, Lsuccess
2196 EmitToStreamer(
2197 Inst: MCInstBuilder(AArch64::TBZX)
2198 .addReg(Reg: ScratchReg)
2199 .addImm(Val: 62)
2200 .addExpr(Val: MCSymbolRefExpr::create(Symbol: SuccessSym, Ctx&: OutContext)));
2201 } else {
2202 llvm_unreachable("Unsupported check method");
2203 }
2204
2205 if (!OnFailure) {
2206 // Trapping sequences do a 'brk'.
2207 // brk #<0xc470 + aut key>
2208 EmitToStreamer(Inst: MCInstBuilder(AArch64::BRK).addImm(Val: 0xc470 | Key));
2209 } else {
2210 // Non-trapping checked sequences return the stripped result in TestedReg,
2211 // skipping over success-only code (such as re-signing the pointer) by
2212 // jumping to OnFailure label.
2213 // Note that this can introduce an authentication oracle (such as based on
2214 // the high bits of the re-signed value).
2215
2216 // FIXME: The XPAC method can be optimized by applying XPAC to TestedReg
2217 // instead of ScratchReg, thus eliminating one `mov` instruction.
2218 // Both XPAC and XPACHint can be further optimized by not using a
2219 // conditional branch jumping over an unconditional one.
2220
2221 switch (Method) {
2222 case AuthCheckMethod::XPACHint:
2223 // LR is already XPAC-ed at this point.
2224 break;
2225 case AuthCheckMethod::XPAC:
2226 // mov Xtested, Xscratch
2227 emitMovXReg(Dest: TestedReg, Src: ScratchReg);
2228 break;
2229 default:
2230 // If Xtested was not XPAC-ed so far, emit XPAC here.
2231 // xpac(i|d) Xtested
2232 unsigned XPACOpc = getXPACOpcodeForKey(K: Key);
2233 EmitToStreamer(
2234 Inst: MCInstBuilder(XPACOpc).addReg(Reg: TestedReg).addReg(Reg: TestedReg));
2235 }
2236
2237 // b Lend
2238 const auto *OnFailureExpr = MCSymbolRefExpr::create(Symbol: OnFailure, Ctx&: OutContext);
2239 EmitToStreamer(Inst: MCInstBuilder(AArch64::B).addExpr(Val: OnFailureExpr));
2240 }
2241
2242 // If the auth check succeeds, we can continue.
2243 // Lsuccess:
2244 OutStreamer->emitLabel(Symbol: SuccessSym);
2245}
2246
2247// With Pointer Authentication, it may be needed to explicitly check the
2248// authenticated value in LR before performing a tail call.
2249// Otherwise, the callee may re-sign the invalid return address,
2250// introducing a signing oracle.
2251void AArch64AsmPrinter::emitPtrauthTailCallHardening(const MachineInstr *TC) {
2252 if (!AArch64FI->shouldSignReturnAddress(MF: *MF))
2253 return;
2254
2255 auto LRCheckMethod = STI->getAuthenticatedLRCheckMethod(MF: *MF);
2256 if (LRCheckMethod == AArch64PAuth::AuthCheckMethod::None)
2257 return;
2258
2259 const AArch64RegisterInfo *TRI = STI->getRegisterInfo();
2260 Register ScratchReg =
2261 TC->readsRegister(Reg: AArch64::X16, TRI) ? AArch64::X17 : AArch64::X16;
2262 assert(!TC->readsRegister(ScratchReg, TRI) &&
2263 "Neither x16 nor x17 is available as a scratch register");
2264 AArch64PACKey::ID Key =
2265 AArch64FI->shouldSignWithBKey() ? AArch64PACKey::IB : AArch64PACKey::IA;
2266 emitPtrauthCheckAuthenticatedValue(TestedReg: AArch64::LR, ScratchReg, Key,
2267 Method: LRCheckMethod);
2268}
2269
2270bool AArch64AsmPrinter::emitDeactivationSymbolRelocation(Value *DS) {
2271 if (!DS)
2272 return false;
2273
2274 if (isa<GlobalAlias>(Val: DS)) {
2275 // Just emit the nop directly.
2276 EmitToStreamer(Inst: MCInstBuilder(AArch64::NOP));
2277 return true;
2278 }
2279 MCSymbol *Dot = OutContext.createTempSymbol();
2280 OutStreamer->emitLabel(Symbol: Dot);
2281 const MCExpr *DeactDotExpr = MCSymbolRefExpr::create(Symbol: Dot, Ctx&: OutContext);
2282
2283 const MCExpr *DSExpr = MCSymbolRefExpr::create(
2284 Symbol: OutContext.getOrCreateSymbol(Name: DS->getName()), Ctx&: OutContext);
2285 OutStreamer->emitRelocDirective(Offset: *DeactDotExpr, Name: "R_AARCH64_PATCHINST", Expr: DSExpr,
2286 Loc: SMLoc());
2287 return false;
2288}
2289
2290AArch64AsmPrinter::PtrAuthSchema AArch64AsmPrinter::PtrAuthSchema::CreateImmReg(
2291 AArch64PACKey::ID Key, uint64_t IntDisc, const MachineOperand &AddrDiscOp) {
2292 PtrAuthSchema Schema;
2293 Schema.Key = Key;
2294 Schema.IntDisc = IntDisc;
2295 Schema.AddrDisc = AddrDiscOp.getReg();
2296 Schema.AddrDiscIsKilled = AddrDiscOp.isKill();
2297 Schema.PCDisc = Register();
2298 return Schema;
2299}
2300
2301AArch64AsmPrinter::PtrAuthSchema AArch64AsmPrinter::PtrAuthSchema::CreateRegReg(
2302 AArch64PACKey::ID Key, Register AddrDisc, Register PCDisc) {
2303 assert(PCDisc.isValid() && "Use CreateImmReg for non-PC schemas");
2304 PtrAuthSchema Schema;
2305 Schema.Key = Key;
2306 Schema.IntDisc = 0;
2307 Schema.AddrDisc = AddrDisc;
2308 Schema.AddrDiscIsKilled = false;
2309 Schema.PCDisc = PCDisc;
2310 return Schema;
2311}
2312
2313void AArch64AsmPrinter::emitPtrauthApplyIndirectAddend(Register Pointer,
2314 Register Scratch,
2315 int64_t Addend) {
2316 if (isInt<9>(x: Addend)) {
2317 // ldrsw Scratch, [Pointer, #Addend]! ; note: Pointer+Addend is used later.
2318 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRSWpre)
2319 .addReg(Reg: Pointer)
2320 .addReg(Reg: Scratch)
2321 .addReg(Reg: Pointer)
2322 .addImm(/*simm9:*/ Val: Addend));
2323 } else {
2324 // Pointer += Addend computation has 2 variants
2325 if (isUInt<24>(x: Addend)) {
2326 // Variant 1: add Pointer, Pointer, (Addend >> shift12) lsl shift12
2327 // This can take up to 2 instructions.
2328 for (int BitPos = 0; BitPos != 24 && (Addend >> BitPos); BitPos += 12) {
2329 EmitToStreamer(
2330 Inst: MCInstBuilder(AArch64::ADDXri)
2331 .addReg(Reg: Pointer)
2332 .addReg(Reg: Pointer)
2333 .addImm(Val: (Addend >> BitPos) & 0xfff)
2334 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: BitPos)));
2335 }
2336 } else {
2337 // Variant 2: accumulate constant in Scratch 16 bits at a time,
2338 // and add it to Pointer. This can take 2-5 instructions.
2339 emitMOVZ(Dest: Scratch, Imm: Addend & 0xffff, Shift: 0);
2340 for (int Offset = 16; Offset < 64; Offset += 16) {
2341 if (unsigned Fragment = (Addend >> Offset) & 0xffff)
2342 emitMOVK(Dest: Scratch, Imm: Fragment, Shift: Offset);
2343 }
2344
2345 // add Pointer, Pointer, Scratch
2346 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXrs)
2347 .addReg(Reg: Pointer)
2348 .addReg(Reg: Pointer)
2349 .addReg(Reg: Scratch)
2350 .addImm(Val: 0));
2351 }
2352 // ldrsw Scratch, [Pointer]
2353 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRSWui)
2354 .addReg(Reg: Scratch)
2355 .addReg(Reg: Pointer)
2356 .addImm(Val: 0));
2357 }
2358 // add Pointer, Pointer, Scratch
2359 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXrs)
2360 .addReg(Reg: Pointer)
2361 .addReg(Reg: Pointer)
2362 .addReg(Reg: Scratch)
2363 .addImm(Val: 0));
2364}
2365
2366static PtrauthCheckMode getCheckMode(const MachineFunction *MF) {
2367 const AArch64Subtarget &STI = MF->getSubtarget<AArch64Subtarget>();
2368
2369 // If an override is passed via command line argument, just use that value.
2370 if (PtrauthAuthChecks.getNumOccurrences())
2371 return PtrauthAuthChecks;
2372
2373 // Otherwise, on an FPAC CPU, you get traps whether you want them or not:
2374 // there's no point in emitting checks or traps.
2375 if (STI.hasFPAC())
2376 return PtrauthCheckMode::Unchecked;
2377
2378 bool ShouldTrap = MF->getFunction().hasFnAttribute(Kind: "ptrauth-auth-traps");
2379 return ShouldTrap ? PtrauthCheckMode::Trap : PtrauthCheckMode::Poison;
2380}
2381
2382// We expand non-signing AUT* pseudo instructions into a sequence of the form
2383//
2384// ; 1. Authenticate Pointer
2385//
2386// or
2387//
2388// ; 1. Authenticate Pointer
2389// ; 2. Check that Pointer is valid, trap otherwise
2390//
2391// We expand AUT*PAC pseudo instructions into a sequence of the form
2392// (with addend only applied if Addend argument is given):
2393//
2394// ; 1. Authenticate Pointer
2395// ; 3. Apply addend and sign Pointer
2396//
2397// or
2398//
2399// ; 1. Authenticate Pointer
2400// ; 2. Check that Pointer is valid, trap otherwise
2401// ; 3. Apply addend and sign Pointer
2402//
2403// or
2404//
2405// ; 1. Authenticate Pointer
2406// ; 2. Check that Pointer is valid, jump to .Lon_failure otherwise
2407// ; 3. Apply addend and sign Pointer
2408// .Lon_failure:
2409//
2410void AArch64AsmPrinter::emitPtrauthAuthResign(
2411 Register Pointer, Register Scratch, PtrAuthSchema AuthSchema,
2412 std::optional<PtrAuthSchema> SignSchema, std::optional<int64_t> Addend,
2413 Value *DS) {
2414 const PtrauthCheckMode CheckMode = getCheckMode(MF);
2415 const bool IsAuthWithPC = AuthSchema.PCDisc.isValid();
2416 assert(!SignSchema || !SignSchema->PCDisc.isValid());
2417
2418 Register SignAddrDiscOrNone = SignSchema ? SignSchema->AddrDisc : Register();
2419
2420 // 1. Authenticate Pointer - this is the only common step.
2421 // It is more complex than signing because AUTI[AB]171615 may be used.
2422
2423 if (IsAuthWithPC) {
2424 assert(Pointer == AArch64::X17 && Scratch == AArch64::X16 &&
2425 "AUTPCPAC must use x17/x16 as Pointer/Scratch");
2426
2427 assert(AuthSchema.AddrDisc == AArch64::X16 &&
2428 "AUTPCPAC requires address discriminator in X16");
2429
2430 assert(AuthSchema.PCDisc == AArch64::X15 &&
2431 "AUTPCPAC requires PC discriminator in X15");
2432
2433 assert(AuthSchema.IntDisc == 0 && "AUTPCPAC does not support IntDisc");
2434
2435 assert((AuthSchema.Key == AArch64PACKey::IB ||
2436 AuthSchema.Key == AArch64PACKey::IA) &&
2437 "AUTPCPAC only supports AUT-ing with IA/IB");
2438
2439 if (!emitDeactivationSymbolRelocation(DS)) {
2440 unsigned AutOpc = (AuthSchema.Key == AArch64PACKey::IB)
2441 ? AArch64::AUTIB171615
2442 : AArch64::AUTIA171615;
2443 EmitToStreamer(Inst: MCInstBuilder(AutOpc));
2444 }
2445 } else {
2446 // emitPtrauthDiscriminator is allowed to clobber AuthSchema.AddrDisc as
2447 // long as it is not used past this point neither externally (the register
2448 // operand is "killed"), nor internally (it does not alias anything being
2449 // used later by this pseudo instruction).
2450 //
2451 // Note that, while rather unlikely, it is technically possible to use the
2452 // Pointer to compute its own discriminator.
2453 Register AUTDiscReg = emitPtrauthDiscriminator(
2454 Disc: AuthSchema.IntDisc, AddrDisc: AuthSchema.AddrDisc, ScratchReg: Scratch,
2455 MayClobberAddrDisc: AuthSchema.addrDiscIsKilledAndNoneOf(Regs: {Pointer, SignAddrDiscOrNone}));
2456 if (!emitDeactivationSymbolRelocation(DS))
2457 emitAUT(Key: AuthSchema.Key, Pointer, Disc: AUTDiscReg);
2458 }
2459
2460 // The other two steps are optional, define lambdas for them:
2461 // 2. Check that Pointer is valid, on failure jump to label or trap.
2462 auto EmitCheck = [&](MCSymbol *OnFailure = nullptr) {
2463 emitPtrauthCheckAuthenticatedValue(TestedReg: Pointer, ScratchReg: Scratch, Key: AuthSchema.Key,
2464 Method: AArch64PAuth::AuthCheckMethod::XPAC,
2465 OnFailure);
2466 };
2467 // 3. Apply addend and sign Pointer.
2468 auto EmitResignOnSuccess = [&]() {
2469 if (Addend.has_value())
2470 emitPtrauthApplyIndirectAddend(Pointer, Scratch, Addend: *Addend);
2471
2472 assert(Pointer != SignSchema->AddrDisc && "Pointer is early-clobbered");
2473 Register PACDiscReg =
2474 emitPtrauthDiscriminator(Disc: SignSchema->IntDisc, AddrDisc: SignSchema->AddrDisc,
2475 ScratchReg: Scratch, MayClobberAddrDisc: SignSchema->AddrDiscIsKilled);
2476 emitPAC(Key: SignSchema->Key, Pointer, Disc: PACDiscReg);
2477 };
2478
2479 // Emit checking and resigning as needed.
2480
2481 if (!SignSchema) {
2482 if (CheckMode == PtrauthCheckMode::Trap)
2483 EmitCheck();
2484 // For authentication-only pseudos, Poison is demoted to Unchecked.
2485 return;
2486 }
2487
2488 switch (CheckMode) {
2489 case Unchecked:
2490 EmitResignOnSuccess();
2491 break;
2492 case Trap:
2493 EmitCheck();
2494 EmitResignOnSuccess();
2495 break;
2496 case Poison:
2497 MCSymbol *OnFailure = createTempSymbol(Name: "resign_end_");
2498 EmitCheck(OnFailure);
2499 EmitResignOnSuccess();
2500 OutStreamer->emitLabel(Symbol: OnFailure);
2501 break;
2502 }
2503}
2504
2505void AArch64AsmPrinter::emitPtrauthSign(const MachineInstr *MI) {
2506 Register Val = MI->getOperand(i: 1).getReg();
2507 auto Key = (AArch64PACKey::ID)MI->getOperand(i: 2).getImm();
2508 uint64_t Disc = MI->getOperand(i: 3).getImm();
2509 Register AddrDisc = MI->getOperand(i: 4).getReg();
2510 bool AddrDiscKilled = MI->getOperand(i: 4).isKill();
2511
2512 // As long as at least one of Val and AddrDisc is in GPR64noip, a scratch
2513 // register is available.
2514 Register ScratchReg = Val == AArch64::X16 ? AArch64::X17 : AArch64::X16;
2515 assert(ScratchReg != AddrDisc &&
2516 "Neither X16 nor X17 is available as a scratch register");
2517
2518 // Compute pac discriminator
2519 Register DiscReg = emitPtrauthDiscriminator(
2520 Disc, AddrDisc, ScratchReg, /*MayClobberAddrDisc=*/AddrDiscKilled);
2521
2522 if (emitDeactivationSymbolRelocation(DS: MI->getDeactivationSymbol()))
2523 return;
2524
2525 emitPAC(Key, Pointer: Val, Disc: DiscReg);
2526}
2527
2528void AArch64AsmPrinter::emitPtrauthBranch(const MachineInstr *MI) {
2529 bool IsCall = MI->getOpcode() == AArch64::BLRA;
2530 unsigned BrTarget = MI->getOperand(i: 0).getReg();
2531
2532 auto Key = (AArch64PACKey::ID)MI->getOperand(i: 1).getImm();
2533 uint64_t Disc = MI->getOperand(i: 2).getImm();
2534
2535 unsigned AddrDisc = MI->getOperand(i: 3).getReg();
2536
2537 // Make sure AddrDisc is solely used to compute the discriminator.
2538 // While hardly meaningful, it is still possible to describe an authentication
2539 // of a pointer against its own value (instead of storage address) with
2540 // intrinsics, so use report_fatal_error instead of assert.
2541 if (BrTarget == AddrDisc)
2542 report_fatal_error(reason: "Branch target is signed with its own value");
2543
2544 // If we are printing BLRA pseudo, try to save one MOV by making use of the
2545 // fact that x16 and x17 are described as clobbered by the MI instruction and
2546 // AddrDisc is not used as any other input.
2547 //
2548 // Back in the day, emitPtrauthDiscriminator was restricted to only returning
2549 // either x16 or x17, meaning the returned register is always among the
2550 // implicit-def'ed registers of BLRA pseudo. Now this property can be violated
2551 // if isX16X17Safer predicate is false, thus manually check if AddrDisc is
2552 // among x16 and x17 to prevent clobbering unexpected registers.
2553 //
2554 // Unlike BLRA, BRA pseudo is used to perform computed goto, and thus not
2555 // declared as clobbering x16/x17.
2556 //
2557 // FIXME: Make use of `killed` flags and register masks instead.
2558 bool AddrDiscIsImplicitDef =
2559 IsCall && (AddrDisc == AArch64::X16 || AddrDisc == AArch64::X17);
2560 Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, ScratchReg: AArch64::X17,
2561 MayClobberAddrDisc: AddrDiscIsImplicitDef);
2562 emitBLRA(IsCall, Key, Target: BrTarget, Disc: DiscReg);
2563}
2564
2565void AArch64AsmPrinter::emitAddImm(MCRegister Reg, int64_t Addend,
2566 MCRegister Tmp) {
2567 if (Addend != 0) {
2568 const uint64_t AbsOffset = (Addend > 0 ? Addend : -((uint64_t)Addend));
2569 const bool IsNeg = Addend < 0;
2570 if (isUInt<24>(x: AbsOffset)) {
2571 for (int BitPos = 0; BitPos != 24 && (AbsOffset >> BitPos);
2572 BitPos += 12) {
2573 EmitToStreamer(
2574 Inst: MCInstBuilder(IsNeg ? AArch64::SUBXri : AArch64::ADDXri)
2575 .addReg(Reg)
2576 .addReg(Reg)
2577 .addImm(Val: (AbsOffset >> BitPos) & 0xfff)
2578 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: BitPos)));
2579 }
2580 } else {
2581 const uint64_t UAddend = Addend;
2582 EmitToStreamer(Inst: MCInstBuilder(IsNeg ? AArch64::MOVNXi : AArch64::MOVZXi)
2583 .addReg(Reg: Tmp)
2584 .addImm(Val: (IsNeg ? ~UAddend : UAddend) & 0xffff)
2585 .addImm(/*shift=*/Val: 0));
2586 auto NeedMovk = [IsNeg, UAddend](int BitPos) -> bool {
2587 assert(BitPos == 16 || BitPos == 32 || BitPos == 48);
2588 uint64_t Shifted = UAddend >> BitPos;
2589 if (!IsNeg)
2590 return Shifted != 0;
2591 for (int I = 0; I != 64 - BitPos; I += 16)
2592 if (((Shifted >> I) & 0xffff) != 0xffff)
2593 return true;
2594 return false;
2595 };
2596 for (int BitPos = 16; BitPos != 64 && NeedMovk(BitPos); BitPos += 16)
2597 emitMOVK(Dest: Tmp, Imm: (UAddend >> BitPos) & 0xffff, Shift: BitPos);
2598
2599 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXrs)
2600 .addReg(Reg)
2601 .addReg(Reg)
2602 .addReg(Reg: Tmp)
2603 .addImm(/*shift=*/Val: 0));
2604 }
2605 }
2606}
2607
2608void AArch64AsmPrinter::emitAddress(MCRegister Reg, const MCExpr *Expr,
2609 MCRegister Tmp, bool DSOLocal,
2610 const MCSubtargetInfo &STI) {
2611 MCValue Val;
2612 if (!Expr->evaluateAsRelocatable(Res&: Val, Asm: nullptr))
2613 report_fatal_error(reason: "emitAddress could not evaluate");
2614 if (DSOLocal) {
2615 EmitToStreamer(
2616 Inst: MCInstBuilder(AArch64::ADRP)
2617 .addReg(Reg)
2618 .addExpr(Val: MCSpecifierExpr::create(Expr, S: AArch64::S_ABS_PAGE,
2619 Ctx&: OutStreamer->getContext())));
2620 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
2621 .addReg(Reg)
2622 .addReg(Reg)
2623 .addExpr(Val: MCSpecifierExpr::create(
2624 Expr, S: AArch64::S_LO12, Ctx&: OutStreamer->getContext()))
2625 .addImm(Val: 0));
2626 } else {
2627 auto *SymRef =
2628 MCSymbolRefExpr::create(Symbol: Val.getAddSym(), Ctx&: OutStreamer->getContext());
2629 EmitToStreamer(
2630 Inst: MCInstBuilder(AArch64::ADRP)
2631 .addReg(Reg)
2632 .addExpr(Val: MCSpecifierExpr::create(Expr: SymRef, S: AArch64::S_GOT_PAGE,
2633 Ctx&: OutStreamer->getContext())));
2634 EmitToStreamer(
2635 Inst: MCInstBuilder(AArch64::LDRXui)
2636 .addReg(Reg)
2637 .addReg(Reg)
2638 .addExpr(Val: MCSpecifierExpr::create(Expr: SymRef, S: AArch64::S_GOT_LO12,
2639 Ctx&: OutStreamer->getContext())));
2640 emitAddImm(Reg, Addend: Val.getConstant(), Tmp);
2641 }
2642}
2643
2644static bool targetSupportsIRelativeRelocation(const Triple &TT) {
2645 // IFUNCs are ELF-only.
2646 if (!TT.isOSBinFormatELF())
2647 return false;
2648
2649 // IFUNCs are supported on glibc, bionic, and some but not all of the BSDs.
2650 return TT.isOSGlibc() || TT.isAndroid() || TT.isOSFreeBSD() ||
2651 TT.isOSDragonFly() || TT.isOSNetBSD();
2652}
2653
2654// Emit an ifunc resolver that returns a signed pointer to the specified target,
2655// and return a FUNCINIT reference to the resolver. In the linked binary, this
2656// function becomes the target of an IRELATIVE relocation. This resolver is used
2657// to relocate signed pointers in global variable initializers in special cases
2658// where the standard R_AARCH64_AUTH_ABS64 relocation would not work.
2659//
2660// Example (signed null pointer, not address discriminated):
2661//
2662// .8byte .Lpauth_ifunc0
2663// .pushsection .text.startup,"ax",@progbits
2664// .Lpauth_ifunc0:
2665// mov x0, #0
2666// mov x1, #12345
2667// b __emupac_pacda
2668//
2669// Example (signed null pointer, address discriminated):
2670//
2671// .Ltmp:
2672// .8byte .Lpauth_ifunc0
2673// .pushsection .text.startup,"ax",@progbits
2674// .Lpauth_ifunc0:
2675// mov x0, #0
2676// adrp x1, .Ltmp
2677// add x1, x1, :lo12:.Ltmp
2678// b __emupac_pacda
2679// .popsection
2680//
2681// Example (signed pointer to symbol, not address discriminated):
2682//
2683// .Ltmp:
2684// .8byte .Lpauth_ifunc0
2685// .pushsection .text.startup,"ax",@progbits
2686// .Lpauth_ifunc0:
2687// adrp x0, symbol
2688// add x0, x0, :lo12:symbol
2689// mov x1, #12345
2690// b __emupac_pacda
2691// .popsection
2692//
2693// Example (signed null pointer, not address discriminated, with deactivation
2694// symbol ds):
2695//
2696// .8byte .Lpauth_ifunc0
2697// .pushsection .text.startup,"ax",@progbits
2698// .Lpauth_ifunc0:
2699// mov x0, #0
2700// mov x1, #12345
2701// .reloc ., R_AARCH64_PATCHINST, ds
2702// b __emupac_pacda
2703// ret
2704// .popsection
2705const MCExpr *AArch64AsmPrinter::emitPAuthRelocationAsIRelative(
2706 const MCExpr *Target, uint64_t Disc, AArch64PACKey::ID KeyID,
2707 bool HasAddressDiversity, bool IsDSOLocal, const MCExpr *DSExpr) {
2708 const Triple &TT = TM.getTargetTriple();
2709
2710 // We only emit an IRELATIVE relocation if the target supports IRELATIVE.
2711 if (!targetSupportsIRelativeRelocation(TT))
2712 return nullptr;
2713
2714 // For now, only the DA key is supported.
2715 if (KeyID != AArch64PACKey::DA)
2716 return nullptr;
2717
2718 // AArch64Subtarget is huge, so heap allocate it so we don't run out of stack
2719 // space.
2720 auto STI = std::make_unique<AArch64Subtarget>(
2721 args: TT, args: TM.getTargetCPU(), args: TM.getTargetCPU(), args: TM.getTargetFeatureString(), args&: TM,
2722 args: true);
2723 this->STI = STI.get();
2724
2725 MCSymbol *Place = OutStreamer->getContext().createTempSymbol();
2726 OutStreamer->emitLabel(Symbol: Place);
2727 OutStreamer->pushSection();
2728
2729 const MCSymbolELF *Group =
2730 static_cast<MCSectionELF *>(OutStreamer->getCurrentSectionOnly())
2731 ->getGroup();
2732 auto Flags = ELF::SHF_ALLOC | ELF::SHF_EXECINSTR;
2733 if (Group)
2734 Flags |= ELF::SHF_GROUP;
2735 OutStreamer->switchSection(Section: OutStreamer->getContext().getELFSection(
2736 Section: ".text.startup", Type: ELF::SHT_PROGBITS, Flags, EntrySize: 0, Group, IsComdat: true,
2737 UniqueID: Group ? MCSection::NonUniqueID : PAuthIFuncNextUniqueID++, LinkedToSym: nullptr));
2738
2739 MCSymbol *IRelativeSym =
2740 OutStreamer->getContext().createLinkerPrivateSymbol(Name: "pauth_ifunc");
2741 OutStreamer->emitLabel(Symbol: IRelativeSym);
2742 if (isa<MCConstantExpr>(Val: Target)) {
2743 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::MOVZXi)
2744 .addReg(Reg: AArch64::X0)
2745 .addExpr(Val: Target)
2746 .addImm(Val: 0),
2747 STI: *STI);
2748 } else {
2749 emitAddress(Reg: AArch64::X0, Expr: Target, Tmp: AArch64::X16, DSOLocal: IsDSOLocal, STI: *STI);
2750 }
2751 if (HasAddressDiversity) {
2752 auto *PlacePlusDisc = MCBinaryExpr::createAdd(
2753 LHS: MCSymbolRefExpr::create(Symbol: Place, Ctx&: OutStreamer->getContext()),
2754 RHS: MCConstantExpr::create(Value: Disc, Ctx&: OutStreamer->getContext()),
2755 Ctx&: OutStreamer->getContext());
2756 emitAddress(Reg: AArch64::X1, Expr: PlacePlusDisc, Tmp: AArch64::X16, /*IsDSOLocal=*/DSOLocal: true,
2757 STI: *STI);
2758 } else {
2759 if (!isUInt<16>(x: Disc)) {
2760 OutContext.reportError(L: SMLoc(), Msg: "AArch64 PAC Discriminator '" +
2761 Twine(Disc) +
2762 "' out of range [0, 0xFFFF]");
2763 }
2764 emitMOVZ(Dest: AArch64::X1, Imm: Disc, Shift: 0);
2765 }
2766
2767 if (DSExpr) {
2768 MCSymbol *PrePACInst = OutStreamer->getContext().createTempSymbol();
2769 OutStreamer->emitLabel(Symbol: PrePACInst);
2770
2771 auto *PrePACInstExpr =
2772 MCSymbolRefExpr::create(Symbol: PrePACInst, Ctx&: OutStreamer->getContext());
2773 OutStreamer->emitRelocDirective(Offset: *PrePACInstExpr, Name: "R_AARCH64_PATCHINST",
2774 Expr: DSExpr, Loc: SMLoc());
2775 }
2776
2777 // We don't know the subtarget because this is being emitted for a global
2778 // initializer. Because the performance of IFUNC resolvers is unimportant, we
2779 // always call the EmuPAC runtime, which will end up using the PAC instruction
2780 // if the target supports PAC.
2781 MCSymbol *EmuPAC =
2782 OutStreamer->getContext().getOrCreateSymbol(Name: "__emupac_pacda");
2783 const MCSymbolRefExpr *EmuPACRef =
2784 MCSymbolRefExpr::create(Symbol: EmuPAC, Ctx&: OutStreamer->getContext());
2785 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::B).addExpr(Val: EmuPACRef),
2786 STI: *STI);
2787
2788 // We need a RET despite the above tail call because the deactivation symbol
2789 // may replace the tail call with a NOP.
2790 if (DSExpr)
2791 OutStreamer->emitInstruction(
2792 Inst: MCInstBuilder(AArch64::RET).addReg(Reg: AArch64::LR), STI: *STI);
2793 OutStreamer->popSection();
2794
2795 return MCSpecifierExpr::create(
2796 Expr: MCSymbolRefExpr::create(Symbol: IRelativeSym, Ctx&: OutStreamer->getContext()),
2797 S: AArch64::S_FUNCINIT, Ctx&: OutStreamer->getContext());
2798}
2799
2800const MCExpr *
2801AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) {
2802 MCContext &Ctx = OutContext;
2803
2804 // Figure out the base symbol and the addend, if any.
2805 APInt Offset(64, 0);
2806 const Value *BaseGV = CPA.getPointer()->stripAndAccumulateConstantOffsets(
2807 DL: getDataLayout(), Offset, /*AllowNonInbounds=*/true);
2808
2809 auto *BaseGVB = dyn_cast<GlobalValue>(Val: BaseGV);
2810
2811 const MCExpr *Sym;
2812 if (BaseGVB) {
2813 // If there is an addend, turn that into the appropriate MCExpr.
2814 Sym = MCSymbolRefExpr::create(Symbol: getSymbol(GV: BaseGVB), Ctx);
2815 if (Offset.sgt(RHS: 0))
2816 Sym = MCBinaryExpr::createAdd(
2817 LHS: Sym, RHS: MCConstantExpr::create(Value: Offset.getSExtValue(), Ctx), Ctx);
2818 else if (Offset.slt(RHS: 0))
2819 Sym = MCBinaryExpr::createSub(
2820 LHS: Sym, RHS: MCConstantExpr::create(Value: (-Offset).getSExtValue(), Ctx), Ctx);
2821 } else if (isa<ConstantPointerNull>(Val: BaseGV)) {
2822 Sym = MCConstantExpr::create(Value: Offset.getSExtValue(), Ctx);
2823 } else {
2824 reportFatalUsageError(reason: "unsupported constant expression in ptrauth pointer");
2825 }
2826
2827 const MCExpr *DSExpr = nullptr;
2828 if (auto *DS = dyn_cast<GlobalValue>(Val: CPA.getDeactivationSymbol())) {
2829 if (isa<GlobalAlias>(Val: DS))
2830 return Sym;
2831 DSExpr = MCSymbolRefExpr::create(Symbol: getSymbol(GV: DS), Ctx);
2832 }
2833
2834 uint64_t KeyID = CPA.getKey()->getZExtValue();
2835 // We later rely on valid KeyID value in AArch64PACKeyIDToString call from
2836 // AArch64AuthMCExpr::printImpl, so fail fast.
2837 if (KeyID > AArch64PACKey::LAST) {
2838 CPA.getContext().emitError(ErrorStr: "AArch64 PAC Key ID '" + Twine(KeyID) +
2839 "' out of range [0, " +
2840 Twine((unsigned)AArch64PACKey::LAST) + "]");
2841 KeyID = 0;
2842 }
2843
2844 uint64_t Disc = CPA.getDiscriminator()->getZExtValue();
2845
2846 // Check if we can represent this with an IRELATIVE and emit it if so.
2847 if (auto *IFuncSym = emitPAuthRelocationAsIRelative(
2848 Target: Sym, Disc, KeyID: AArch64PACKey::ID(KeyID), HasAddressDiversity: CPA.hasAddressDiscriminator(),
2849 IsDSOLocal: BaseGVB && BaseGVB->isDSOLocal(), DSExpr))
2850 return IFuncSym;
2851
2852 if (!isUInt<16>(x: Disc)) {
2853 CPA.getContext().emitError(ErrorStr: "AArch64 PAC Discriminator '" + Twine(Disc) +
2854 "' out of range [0, 0xFFFF]");
2855 Disc = 0;
2856 }
2857
2858 if (DSExpr)
2859 report_fatal_error(reason: "deactivation symbols unsupported in constant "
2860 "expressions on this target");
2861
2862 // Finally build the complete @AUTH expr.
2863 return AArch64AuthMCExpr::create(Expr: Sym, Discriminator: Disc, Key: AArch64PACKey::ID(KeyID),
2864 HasAddressDiversity: CPA.hasAddressDiscriminator(), Ctx);
2865}
2866
2867void AArch64AsmPrinter::LowerLOADauthptrstatic(const MachineInstr &MI) {
2868 unsigned DstReg = MI.getOperand(i: 0).getReg();
2869 const MachineOperand &GAOp = MI.getOperand(i: 1);
2870 const uint64_t KeyC = MI.getOperand(i: 2).getImm();
2871 assert(KeyC <= AArch64PACKey::LAST &&
2872 "key is out of range [0, AArch64PACKey::LAST]");
2873 const auto Key = (AArch64PACKey::ID)KeyC;
2874 const uint64_t Disc = MI.getOperand(i: 3).getImm();
2875 assert(isUInt<16>(Disc) &&
2876 "constant discriminator is out of range [0, 0xffff]");
2877
2878 // Emit instruction sequence like the following:
2879 // ADRP x16, symbol$auth_ptr$key$disc
2880 // LDR x16, [x16, :lo12:symbol$auth_ptr$key$disc]
2881 //
2882 // Where the $auth_ptr$ symbol is the stub slot containing the signed pointer
2883 // to symbol.
2884 MCSymbol *AuthPtrStubSym;
2885 if (TM.getTargetTriple().isOSBinFormatELF()) {
2886 const auto &TLOF =
2887 static_cast<const AArch64_ELFTargetObjectFile &>(getObjFileLowering());
2888
2889 assert(GAOp.getOffset() == 0 &&
2890 "non-zero offset for $auth_ptr$ stub slots is not supported");
2891 const MCSymbol *GASym = TM.getSymbol(GV: GAOp.getGlobal());
2892 AuthPtrStubSym = TLOF.getAuthPtrSlotSymbol(TM, MMI, RawSym: GASym, Key, Discriminator: Disc);
2893 } else {
2894 assert(TM.getTargetTriple().isOSBinFormatMachO() &&
2895 "LOADauthptrstatic is implemented only for MachO/ELF");
2896
2897 const auto &TLOF = static_cast<const AArch64_MachoTargetObjectFile &>(
2898 getObjFileLowering());
2899
2900 assert(GAOp.getOffset() == 0 &&
2901 "non-zero offset for $auth_ptr$ stub slots is not supported");
2902 const MCSymbol *GASym = TM.getSymbol(GV: GAOp.getGlobal());
2903 AuthPtrStubSym = TLOF.getAuthPtrSlotSymbol(TM, MMI, RawSym: GASym, Key, Discriminator: Disc);
2904 }
2905
2906 MachineOperand StubMOHi =
2907 MachineOperand::CreateMCSymbol(Sym: AuthPtrStubSym, TargetFlags: AArch64II::MO_PAGE);
2908 MachineOperand StubMOLo = MachineOperand::CreateMCSymbol(
2909 Sym: AuthPtrStubSym, TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2910 MCOperand StubMCHi, StubMCLo;
2911
2912 MCInstLowering.lowerOperand(MO: StubMOHi, MCOp&: StubMCHi);
2913 MCInstLowering.lowerOperand(MO: StubMOLo, MCOp&: StubMCLo);
2914
2915 EmitToStreamer(
2916 S&: *OutStreamer,
2917 Inst: MCInstBuilder(AArch64::ADRP).addReg(Reg: DstReg).addOperand(Op: StubMCHi));
2918
2919 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::LDRXui)
2920 .addReg(Reg: DstReg)
2921 .addReg(Reg: DstReg)
2922 .addOperand(Op: StubMCLo));
2923}
2924
2925void AArch64AsmPrinter::LowerMOVaddrPAC(const MachineInstr &MI) {
2926 const bool IsGOTLoad = MI.getOpcode() == AArch64::LOADgotPAC;
2927 const bool IsELFSignedGOT = MI.getParent()
2928 ->getParent()
2929 ->getInfo<AArch64FunctionInfo>()
2930 ->hasELFSignedGOT();
2931 MachineOperand GAOp = MI.getOperand(i: 0);
2932 const uint64_t KeyC = MI.getOperand(i: 1).getImm();
2933 assert(KeyC <= AArch64PACKey::LAST &&
2934 "key is out of range [0, AArch64PACKey::LAST]");
2935 const auto Key = (AArch64PACKey::ID)KeyC;
2936 const unsigned AddrDisc = MI.getOperand(i: 2).getReg();
2937 const uint64_t Disc = MI.getOperand(i: 3).getImm();
2938
2939 const int64_t Offset = GAOp.getOffset();
2940 GAOp.setOffset(0);
2941
2942 // Emit:
2943 // target materialization:
2944 // - via GOT:
2945 // - unsigned GOT:
2946 // adrp x16, :got:target
2947 // ldr x16, [x16, :got_lo12:target]
2948 // add offset to x16 if offset != 0
2949 // - ELF signed GOT:
2950 // adrp x17, :got:target
2951 // add x17, x17, :got_auth_lo12:target
2952 // ldr x16, [x17]
2953 // aut{i|d}a x16, x17
2954 // check+trap sequence (if no FPAC)
2955 // add offset to x16 if offset != 0
2956 //
2957 // - direct:
2958 // adrp x16, target
2959 // add x16, x16, :lo12:target
2960 // add offset to x16 if offset != 0
2961 //
2962 // add offset to x16:
2963 // - abs(offset) fits 24 bits:
2964 // add/sub x16, x16, #<offset>[, #lsl 12] (up to 2 instructions)
2965 // - abs(offset) does not fit 24 bits:
2966 // - offset < 0:
2967 // movn+movk sequence filling x17 register with the offset (up to 4
2968 // instructions)
2969 // add x16, x16, x17
2970 // - offset > 0:
2971 // movz+movk sequence filling x17 register with the offset (up to 4
2972 // instructions)
2973 // add x16, x16, x17
2974 //
2975 // signing:
2976 // - 0 discriminator:
2977 // paciza x16
2978 // - Non-0 discriminator, no address discriminator:
2979 // mov x17, #Disc
2980 // pacia x16, x17
2981 // - address discriminator (with potentially folded immediate discriminator):
2982 // pacia x16, xAddrDisc
2983
2984 MachineOperand GAMOHi(GAOp), GAMOLo(GAOp);
2985 MCOperand GAMCHi, GAMCLo;
2986
2987 GAMOHi.setTargetFlags(AArch64II::MO_PAGE);
2988 GAMOLo.setTargetFlags(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2989 if (IsGOTLoad) {
2990 GAMOHi.addTargetFlag(F: AArch64II::MO_GOT);
2991 GAMOLo.addTargetFlag(F: AArch64II::MO_GOT);
2992 }
2993
2994 MCInstLowering.lowerOperand(MO: GAMOHi, MCOp&: GAMCHi);
2995 MCInstLowering.lowerOperand(MO: GAMOLo, MCOp&: GAMCLo);
2996
2997 EmitToStreamer(
2998 Inst: MCInstBuilder(AArch64::ADRP)
2999 .addReg(Reg: IsGOTLoad && IsELFSignedGOT ? AArch64::X17 : AArch64::X16)
3000 .addOperand(Op: GAMCHi));
3001
3002 if (IsGOTLoad) {
3003 if (IsELFSignedGOT) {
3004 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
3005 .addReg(Reg: AArch64::X17)
3006 .addReg(Reg: AArch64::X17)
3007 .addOperand(Op: GAMCLo)
3008 .addImm(Val: 0));
3009
3010 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
3011 .addReg(Reg: AArch64::X16)
3012 .addReg(Reg: AArch64::X17)
3013 .addImm(Val: 0));
3014
3015 assert(GAOp.isGlobal());
3016 assert(GAOp.getGlobal()->getValueType() != nullptr);
3017
3018 bool IsFunctionTy = GAOp.getGlobal()->getValueType()->isFunctionTy();
3019 auto AuthKey = IsFunctionTy ? AArch64PACKey::IA : AArch64PACKey::DA;
3020 emitAUT(Key: AuthKey, Pointer: AArch64::X16, Disc: AArch64::X17);
3021
3022 if (!STI->hasFPAC())
3023 emitPtrauthCheckAuthenticatedValue(TestedReg: AArch64::X16, ScratchReg: AArch64::X17, Key: AuthKey,
3024 Method: AArch64PAuth::AuthCheckMethod::XPAC);
3025 } else {
3026 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
3027 .addReg(Reg: AArch64::X16)
3028 .addReg(Reg: AArch64::X16)
3029 .addOperand(Op: GAMCLo));
3030 }
3031 } else {
3032 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
3033 .addReg(Reg: AArch64::X16)
3034 .addReg(Reg: AArch64::X16)
3035 .addOperand(Op: GAMCLo)
3036 .addImm(Val: 0));
3037 }
3038
3039 emitAddImm(Reg: AArch64::X16, Addend: Offset, Tmp: AArch64::X17);
3040 Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, ScratchReg: AArch64::X17);
3041
3042 emitPAC(Key, Pointer: AArch64::X16, Disc: DiscReg);
3043}
3044
3045void AArch64AsmPrinter::LowerLOADgotAUTH(const MachineInstr &MI) {
3046 Register DstReg = MI.getOperand(i: 0).getReg();
3047 Register AuthResultReg = STI->hasFPAC() ? DstReg : AArch64::X16;
3048 const MachineOperand &GAMO = MI.getOperand(i: 1);
3049 assert(GAMO.getOffset() == 0);
3050
3051 if (MI.getMF()->getTarget().getCodeModel() == CodeModel::Tiny) {
3052 MCOperand GAMC;
3053 MCInstLowering.lowerOperand(MO: GAMO, MCOp&: GAMC);
3054 EmitToStreamer(
3055 Inst: MCInstBuilder(AArch64::ADR).addReg(Reg: AArch64::X17).addOperand(Op: GAMC));
3056 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
3057 .addReg(Reg: AuthResultReg)
3058 .addReg(Reg: AArch64::X17)
3059 .addImm(Val: 0));
3060 } else {
3061 MachineOperand GAHiOp(GAMO);
3062 MachineOperand GALoOp(GAMO);
3063 GAHiOp.addTargetFlag(F: AArch64II::MO_PAGE);
3064 GALoOp.addTargetFlag(F: AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
3065
3066 MCOperand GAMCHi, GAMCLo;
3067 MCInstLowering.lowerOperand(MO: GAHiOp, MCOp&: GAMCHi);
3068 MCInstLowering.lowerOperand(MO: GALoOp, MCOp&: GAMCLo);
3069
3070 EmitToStreamer(
3071 Inst: MCInstBuilder(AArch64::ADRP).addReg(Reg: AArch64::X17).addOperand(Op: GAMCHi));
3072
3073 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
3074 .addReg(Reg: AArch64::X17)
3075 .addReg(Reg: AArch64::X17)
3076 .addOperand(Op: GAMCLo)
3077 .addImm(Val: 0));
3078
3079 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
3080 .addReg(Reg: AuthResultReg)
3081 .addReg(Reg: AArch64::X17)
3082 .addImm(Val: 0));
3083 }
3084
3085 assert(GAMO.isGlobal());
3086 MCSymbol *UndefWeakSym;
3087 if (GAMO.getGlobal()->hasExternalWeakLinkage()) {
3088 UndefWeakSym = createTempSymbol(Name: "undef_weak");
3089 EmitToStreamer(
3090 Inst: MCInstBuilder(AArch64::CBZX)
3091 .addReg(Reg: AuthResultReg)
3092 .addExpr(Val: MCSymbolRefExpr::create(Symbol: UndefWeakSym, Ctx&: OutContext)));
3093 }
3094
3095 assert(GAMO.getGlobal()->getValueType() != nullptr);
3096
3097 bool IsFunctionTy = GAMO.getGlobal()->getValueType()->isFunctionTy();
3098 auto AuthKey = IsFunctionTy ? AArch64PACKey::IA : AArch64PACKey::DA;
3099 emitAUT(Key: AuthKey, Pointer: AuthResultReg, Disc: AArch64::X17);
3100
3101 if (GAMO.getGlobal()->hasExternalWeakLinkage())
3102 OutStreamer->emitLabel(Symbol: UndefWeakSym);
3103
3104 if (!STI->hasFPAC()) {
3105 emitPtrauthCheckAuthenticatedValue(TestedReg: AuthResultReg, ScratchReg: AArch64::X17, Key: AuthKey,
3106 Method: AArch64PAuth::AuthCheckMethod::XPAC);
3107
3108 emitMovXReg(Dest: DstReg, Src: AuthResultReg);
3109 }
3110}
3111
3112const MCExpr *
3113AArch64AsmPrinter::lowerBlockAddressConstant(const BlockAddress &BA) {
3114 const MCExpr *BAE = AsmPrinter::lowerBlockAddressConstant(BA);
3115 const Function &Fn = *BA.getFunction();
3116
3117 if (std::optional<uint16_t> BADisc =
3118 STI->getPtrAuthBlockAddressDiscriminatorIfEnabled(ParentFn: Fn))
3119 return AArch64AuthMCExpr::create(Expr: BAE, Discriminator: *BADisc, Key: AArch64PACKey::IA,
3120 /*HasAddressDiversity=*/false, Ctx&: OutContext);
3121
3122 return BAE;
3123}
3124
3125void AArch64AsmPrinter::emitCBPseudoExpansion(const MachineInstr *MI) {
3126 bool IsImm = false;
3127 unsigned Width = 0;
3128
3129 switch (MI->getOpcode()) {
3130 default:
3131 llvm_unreachable("This is not a CB pseudo instruction");
3132 case AArch64::CBBAssertExt:
3133 IsImm = false;
3134 Width = 8;
3135 break;
3136 case AArch64::CBHAssertExt:
3137 IsImm = false;
3138 Width = 16;
3139 break;
3140 case AArch64::CBWPrr:
3141 Width = 32;
3142 break;
3143 case AArch64::CBXPrr:
3144 Width = 64;
3145 break;
3146 case AArch64::CBWPri:
3147 IsImm = true;
3148 Width = 32;
3149 break;
3150 case AArch64::CBXPri:
3151 IsImm = true;
3152 Width = 64;
3153 break;
3154 }
3155
3156 AArch64CC::CondCode CC =
3157 static_cast<AArch64CC::CondCode>(MI->getOperand(i: 0).getImm());
3158 bool NeedsRegSwap = false;
3159 bool NeedsImmDec = false;
3160 bool NeedsImmInc = false;
3161
3162#define GET_CB_OPC(IsImm, Width, ImmCond, RegCond) \
3163 (IsImm \
3164 ? (Width == 32 ? AArch64::CB##ImmCond##Wri : AArch64::CB##ImmCond##Xri) \
3165 : (Width == 8 \
3166 ? AArch64::CBB##RegCond##Wrr \
3167 : (Width == 16 ? AArch64::CBH##RegCond##Wrr \
3168 : (Width == 32 ? AArch64::CB##RegCond##Wrr \
3169 : AArch64::CB##RegCond##Xrr))))
3170 unsigned MCOpC;
3171
3172 // Decide if we need to either swap register operands or increment/decrement
3173 // immediate operands
3174 switch (CC) {
3175 default:
3176 llvm_unreachable("Invalid CB condition code");
3177 case AArch64CC::EQ:
3178 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ EQ, /* Reg-Reg */ EQ);
3179 break;
3180 case AArch64CC::NE:
3181 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ NE, /* Reg-Reg */ NE);
3182 break;
3183 case AArch64CC::HS:
3184 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ HI, /* Reg-Reg */ HS);
3185 NeedsImmDec = IsImm;
3186 break;
3187 case AArch64CC::LO:
3188 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ LO, /* Reg-Reg */ HI);
3189 NeedsRegSwap = !IsImm;
3190 break;
3191 case AArch64CC::HI:
3192 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ HI, /* Reg-Reg */ HI);
3193 break;
3194 case AArch64CC::LS:
3195 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ LO, /* Reg-Reg */ HS);
3196 NeedsRegSwap = !IsImm;
3197 NeedsImmInc = IsImm;
3198 break;
3199 case AArch64CC::GE:
3200 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ GT, /* Reg-Reg */ GE);
3201 NeedsImmDec = IsImm;
3202 break;
3203 case AArch64CC::LT:
3204 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ LT, /* Reg-Reg */ GT);
3205 NeedsRegSwap = !IsImm;
3206 break;
3207 case AArch64CC::GT:
3208 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ GT, /* Reg-Reg */ GT);
3209 break;
3210 case AArch64CC::LE:
3211 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ LT, /* Reg-Reg */ GE);
3212 NeedsRegSwap = !IsImm;
3213 NeedsImmInc = IsImm;
3214 break;
3215 }
3216#undef GET_CB_OPC
3217
3218 MCInst Inst;
3219 Inst.setOpcode(MCOpC);
3220
3221 MCOperand Lhs, Rhs, Trgt;
3222 lowerOperand(MO: MI->getOperand(i: 1), MCOp&: Lhs);
3223 lowerOperand(MO: MI->getOperand(i: 2), MCOp&: Rhs);
3224 lowerOperand(MO: MI->getOperand(i: 3), MCOp&: Trgt);
3225
3226 // Now swap, increment or decrement
3227 if (NeedsRegSwap) {
3228 assert(Lhs.isReg() && "Expected register operand for CB");
3229 assert(Rhs.isReg() && "Expected register operand for CB");
3230 Inst.addOperand(Op: Rhs);
3231 Inst.addOperand(Op: Lhs);
3232 } else if (NeedsImmDec) {
3233 Rhs.setImm(Rhs.getImm() - 1);
3234 Inst.addOperand(Op: Lhs);
3235 Inst.addOperand(Op: Rhs);
3236 } else if (NeedsImmInc) {
3237 Rhs.setImm(Rhs.getImm() + 1);
3238 Inst.addOperand(Op: Lhs);
3239 Inst.addOperand(Op: Rhs);
3240 } else {
3241 Inst.addOperand(Op: Lhs);
3242 Inst.addOperand(Op: Rhs);
3243 }
3244
3245 assert((!IsImm || (Rhs.getImm() >= 0 && Rhs.getImm() < 64)) &&
3246 "CB immediate operand out-of-bounds");
3247
3248 Inst.addOperand(Op: Trgt);
3249 EmitToStreamer(S&: *OutStreamer, Inst);
3250}
3251
3252// Simple pseudo-instructions have their lowering (with expansion to real
3253// instructions) auto-generated.
3254#include "AArch64GenMCPseudoLowering.inc"
3255
3256void AArch64AsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) {
3257 S.emitInstruction(Inst, STI: *STI);
3258#ifndef NDEBUG
3259 ++InstsEmitted;
3260#endif
3261}
3262
3263void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
3264 AArch64_MC::verifyInstructionPredicates(Opcode: MI->getOpcode(), Features: STI->getFeatureBits());
3265
3266#ifndef NDEBUG
3267 InstsEmitted = 0;
3268 llvm::scope_exit CheckMISize([&]() {
3269 assert(STI->getInstrInfo()->getInstSizeInBytes(*MI) >= InstsEmitted * 4);
3270 });
3271#endif
3272
3273 // Do any auto-generated pseudo lowerings.
3274 if (MCInst OutInst; lowerPseudoInstExpansion(MI, Inst&: OutInst)) {
3275 EmitToStreamer(S&: *OutStreamer, Inst: OutInst);
3276 return;
3277 }
3278
3279 if (MI->getOpcode() == AArch64::ADRP) {
3280 for (auto &Opd : MI->operands()) {
3281 if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) ==
3282 "swift_async_extendedFramePointerFlags") {
3283 ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true;
3284 }
3285 }
3286 }
3287
3288 if (AArch64FI->getLOHRelated().count(Ptr: MI)) {
3289 // Generate a label for LOH related instruction
3290 MCSymbol *LOHLabel = createTempSymbol(Name: "loh");
3291 // Associate the instruction with the label
3292 LOHInstToLabel[MI] = LOHLabel;
3293 OutStreamer->emitLabel(Symbol: LOHLabel);
3294 }
3295
3296 AArch64TargetStreamer *TS =
3297 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
3298 // Do any manual lowerings.
3299 switch (MI->getOpcode()) {
3300 default:
3301 assert(!AArch64InstrInfo::isTailCallReturnInst(*MI) &&
3302 "Unhandled tail call instruction");
3303 break;
3304 case AArch64::READ_REGISTER_GPR64:
3305 // Read of a named GPR: emit "mov Xt, Xn" (ORR Xt, XZR, Xn). The source
3306 // register is encoded as an immediate operand so that earlier passes do not
3307 // see a use of an undefined physical register.
3308 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ORRXrs)
3309 .addReg(Reg: MI->getOperand(i: 0).getReg())
3310 .addReg(Reg: AArch64::XZR)
3311 .addReg(Reg: MI->getOperand(i: 1).getImm())
3312 .addImm(Val: 0));
3313 return;
3314 case AArch64::READ_REGISTER_FPR64:
3315 // Read of a named FP/SIMD d-register: emit "fmov Dt, Dn".
3316 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::FMOVDr)
3317 .addReg(Reg: MI->getOperand(i: 0).getReg())
3318 .addReg(Reg: MI->getOperand(i: 1).getImm()));
3319 return;
3320 case AArch64::HINT: {
3321 // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
3322 // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
3323 // non-empty. If MI is the initial BTI, place the
3324 // __patchable_function_entries label after BTI.
3325 if (CurrentPatchableFunctionEntrySym &&
3326 CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
3327 MI == &MF->front().front()) {
3328 int64_t Imm = MI->getOperand(i: 0).getImm();
3329 if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38) {
3330 MCInst Inst;
3331 MCInstLowering.Lower(MI, OutMI&: Inst);
3332 EmitToStreamer(S&: *OutStreamer, Inst);
3333 CurrentPatchableFunctionEntrySym = createTempSymbol(Name: "patch");
3334 OutStreamer->emitLabel(Symbol: CurrentPatchableFunctionEntrySym);
3335 return;
3336 }
3337 }
3338 break;
3339 }
3340 case AArch64::MOVMCSym: {
3341 Register DestReg = MI->getOperand(i: 0).getReg();
3342 const MachineOperand &MO_Sym = MI->getOperand(i: 1);
3343 MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
3344 MCOperand Hi_MCSym, Lo_MCSym;
3345
3346 Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
3347 Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
3348
3349 MCInstLowering.lowerOperand(MO: Hi_MOSym, MCOp&: Hi_MCSym);
3350 MCInstLowering.lowerOperand(MO: Lo_MOSym, MCOp&: Lo_MCSym);
3351
3352 MCInst MovZ;
3353 MovZ.setOpcode(AArch64::MOVZXi);
3354 MovZ.addOperand(Op: MCOperand::createReg(Reg: DestReg));
3355 MovZ.addOperand(Op: Hi_MCSym);
3356 MovZ.addOperand(Op: MCOperand::createImm(Val: 16));
3357 EmitToStreamer(S&: *OutStreamer, Inst: MovZ);
3358
3359 MCInst MovK;
3360 MovK.setOpcode(AArch64::MOVKXi);
3361 MovK.addOperand(Op: MCOperand::createReg(Reg: DestReg));
3362 MovK.addOperand(Op: MCOperand::createReg(Reg: DestReg));
3363 MovK.addOperand(Op: Lo_MCSym);
3364 MovK.addOperand(Op: MCOperand::createImm(Val: 0));
3365 EmitToStreamer(S&: *OutStreamer, Inst: MovK);
3366 return;
3367 }
3368 case AArch64::MOVIv2d_ns:
3369 // It is generally beneficial to rewrite "fmov s0, wzr" to "movi d0, #0".
3370 // as movi is more efficient across all cores. Newer cores can eliminate
3371 // fmovs early and there is no difference with movi, but this not true for
3372 // all implementations.
3373 //
3374 // The floating-point version doesn't quite work in rare cases on older
3375 // CPUs, so on those targets we lower this instruction to movi.16b instead.
3376 if (STI->hasZeroCycleZeroingFPWorkaround() &&
3377 MI->getOperand(i: 1).getImm() == 0) {
3378 MCInst TmpInst;
3379 TmpInst.setOpcode(AArch64::MOVIv16b_ns);
3380 TmpInst.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg()));
3381 TmpInst.addOperand(Op: MCOperand::createImm(Val: 0));
3382 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
3383 return;
3384 }
3385 break;
3386
3387 case AArch64::DBG_VALUE:
3388 case AArch64::DBG_VALUE_LIST:
3389 if (isVerbose() && OutStreamer->hasRawTextSupport()) {
3390 SmallString<128> TmpStr;
3391 raw_svector_ostream OS(TmpStr);
3392 PrintDebugValueComment(MI, OS);
3393 OutStreamer->emitRawText(String: StringRef(OS.str()));
3394 }
3395 return;
3396
3397 case AArch64::EMITBKEY: {
3398 ExceptionHandling ExceptionHandlingType = MAI.getExceptionHandlingType();
3399 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
3400 ExceptionHandlingType != ExceptionHandling::ARM)
3401 return;
3402
3403 if (getFunctionCFISectionType(MF: *MF) == CFISection::None)
3404 return;
3405
3406 OutStreamer->emitCFIBKeyFrame();
3407 return;
3408 }
3409
3410 case AArch64::EMITMTETAGGED: {
3411 ExceptionHandling ExceptionHandlingType = MAI.getExceptionHandlingType();
3412 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
3413 ExceptionHandlingType != ExceptionHandling::ARM)
3414 return;
3415
3416 if (getFunctionCFISectionType(MF: *MF) != CFISection::None)
3417 OutStreamer->emitCFIMTETaggedFrame();
3418 return;
3419 }
3420
3421 case AArch64::AUTx16x17: {
3422 const Register Pointer = AArch64::X16;
3423 const Register Scratch = AArch64::X17;
3424
3425 auto AuthSchema = PtrAuthSchema::CreateImmReg(
3426 Key: (AArch64PACKey::ID)MI->getOperand(i: 0).getImm(),
3427 IntDisc: MI->getOperand(i: 1).getImm(), AddrDiscOp: MI->getOperand(i: 2));
3428
3429 emitPtrauthAuthResign(Pointer, Scratch, AuthSchema, SignSchema: std::nullopt,
3430 Addend: std::nullopt, DS: MI->getDeactivationSymbol());
3431 return;
3432 }
3433
3434 case AArch64::AUTxMxN: {
3435 const Register Pointer = MI->getOperand(i: 0).getReg();
3436 const Register Scratch = MI->getOperand(i: 1).getReg();
3437
3438 auto AuthSchema = PtrAuthSchema::CreateImmReg(
3439 Key: (AArch64PACKey::ID)MI->getOperand(i: 3).getImm(),
3440 IntDisc: MI->getOperand(i: 4).getImm(), AddrDiscOp: MI->getOperand(i: 5));
3441
3442 emitPtrauthAuthResign(Pointer, Scratch, AuthSchema, SignSchema: std::nullopt,
3443 Addend: std::nullopt, DS: MI->getDeactivationSymbol());
3444 return;
3445 }
3446
3447 case AArch64::AUTPAC: {
3448 const Register Pointer = AArch64::X16;
3449 const Register Scratch = AArch64::X17;
3450
3451 auto AuthSchema = PtrAuthSchema::CreateImmReg(
3452 Key: (AArch64PACKey::ID)MI->getOperand(i: 0).getImm(),
3453 IntDisc: MI->getOperand(i: 1).getImm(), AddrDiscOp: MI->getOperand(i: 2));
3454
3455 auto SignSchema = PtrAuthSchema::CreateImmReg(
3456 Key: (AArch64PACKey::ID)MI->getOperand(i: 3).getImm(),
3457 IntDisc: MI->getOperand(i: 4).getImm(), AddrDiscOp: MI->getOperand(i: 5));
3458
3459 emitPtrauthAuthResign(Pointer, Scratch, AuthSchema, SignSchema,
3460 Addend: std::nullopt, DS: MI->getDeactivationSymbol());
3461 return;
3462 }
3463
3464 case AArch64::AUTPCPAC: {
3465 auto AuthSchema = PtrAuthSchema::CreateRegReg(
3466 Key: (AArch64PACKey::ID)MI->getOperand(i: 0).getImm(), AddrDisc: AArch64::X16,
3467 PCDisc: AArch64::X15);
3468
3469 auto SignSchema = PtrAuthSchema::CreateImmReg(
3470 Key: (AArch64PACKey::ID)MI->getOperand(i: 1).getImm(),
3471 IntDisc: MI->getOperand(i: 2).getImm(), AddrDiscOp: MI->getOperand(i: 3));
3472
3473 emitPtrauthAuthResign(/*Pointer=*/AArch64::X17, /*Scratch=*/AArch64::X16,
3474 AuthSchema, SignSchema, Addend: std::nullopt,
3475 DS: MI->getDeactivationSymbol());
3476 return;
3477 }
3478
3479 case AArch64::AUTRELLOADPAC: {
3480 const Register Pointer = AArch64::X16;
3481 const Register Scratch = AArch64::X17;
3482
3483 auto AuthSchema = PtrAuthSchema::CreateImmReg(
3484 Key: (AArch64PACKey::ID)MI->getOperand(i: 0).getImm(),
3485 IntDisc: MI->getOperand(i: 1).getImm(), AddrDiscOp: MI->getOperand(i: 2));
3486
3487 auto SignSchema = PtrAuthSchema::CreateImmReg(
3488 Key: (AArch64PACKey::ID)MI->getOperand(i: 3).getImm(),
3489 IntDisc: MI->getOperand(i: 4).getImm(), AddrDiscOp: MI->getOperand(i: 5));
3490
3491 emitPtrauthAuthResign(Pointer, Scratch, AuthSchema, SignSchema,
3492 Addend: MI->getOperand(i: 6).getImm(),
3493 DS: MI->getDeactivationSymbol());
3494
3495 return;
3496 }
3497
3498 case AArch64::PAC:
3499 emitPtrauthSign(MI);
3500 return;
3501
3502 case AArch64::LOADauthptrstatic:
3503 LowerLOADauthptrstatic(MI: *MI);
3504 return;
3505
3506 case AArch64::LOADgotPAC:
3507 case AArch64::MOVaddrPAC:
3508 LowerMOVaddrPAC(MI: *MI);
3509 return;
3510
3511 case AArch64::LOADgotAUTH:
3512 LowerLOADgotAUTH(MI: *MI);
3513 return;
3514
3515 case AArch64::BRA:
3516 case AArch64::BLRA:
3517 emitPtrauthBranch(MI);
3518 return;
3519
3520 // Tail calls use pseudo instructions so they have the proper code-gen
3521 // attributes (isCall, isReturn, etc.). We lower them to the real
3522 // instruction here.
3523 case AArch64::AUTH_TCRETURN:
3524 case AArch64::AUTH_TCRETURN_BTI: {
3525 Register Callee = MI->getOperand(i: 0).getReg();
3526 const auto Key = (AArch64PACKey::ID)MI->getOperand(i: 2).getImm();
3527 const uint64_t Disc = MI->getOperand(i: 3).getImm();
3528
3529 Register AddrDisc = MI->getOperand(i: 4).getReg();
3530
3531 Register ScratchReg = Callee == AArch64::X16 ? AArch64::X17 : AArch64::X16;
3532
3533 emitPtrauthTailCallHardening(TC: MI);
3534
3535 // See the comments in emitPtrauthBranch.
3536 if (Callee == AddrDisc)
3537 report_fatal_error(reason: "Call target is signed with its own value");
3538
3539 // After isX16X17Safer predicate was introduced, emitPtrauthDiscriminator is
3540 // no longer restricted to only reusing AddrDisc when it is X16 or X17
3541 // (which are implicit-def'ed by AUTH_TCRETURN pseudos), thus impose this
3542 // restriction manually not to clobber an unexpected register.
3543 bool AddrDiscIsImplicitDef =
3544 AddrDisc == AArch64::X16 || AddrDisc == AArch64::X17;
3545 Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, ScratchReg,
3546 MayClobberAddrDisc: AddrDiscIsImplicitDef);
3547 emitBLRA(/*IsCall*/ false, Key, Target: Callee, Disc: DiscReg);
3548 return;
3549 }
3550
3551 case AArch64::TCRETURNri:
3552 case AArch64::TCRETURNrix16x17:
3553 case AArch64::TCRETURNrix17:
3554 case AArch64::TCRETURNrinotx16:
3555 case AArch64::TCRETURNriALL: {
3556 emitPtrauthTailCallHardening(TC: MI);
3557
3558 recordIfImportCall(BranchInst: MI);
3559 MCInst TmpInst;
3560 TmpInst.setOpcode(AArch64::BR);
3561 TmpInst.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg()));
3562 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
3563 return;
3564 }
3565 case AArch64::TCRETURNdi: {
3566 emitPtrauthTailCallHardening(TC: MI);
3567
3568 MCOperand Dest;
3569 MCInstLowering.lowerOperand(MO: MI->getOperand(i: 0), MCOp&: Dest);
3570 recordIfImportCall(BranchInst: MI);
3571 MCInst TmpInst;
3572 TmpInst.setOpcode(AArch64::B);
3573 TmpInst.addOperand(Op: Dest);
3574 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
3575 return;
3576 }
3577 case AArch64::SpeculationBarrierISBDSBEndBB: {
3578 // Print DSB SYS + ISB
3579 MCInst TmpInstDSB;
3580 TmpInstDSB.setOpcode(AArch64::DSB);
3581 TmpInstDSB.addOperand(Op: MCOperand::createImm(Val: 0xf));
3582 EmitToStreamer(S&: *OutStreamer, Inst: TmpInstDSB);
3583 MCInst TmpInstISB;
3584 TmpInstISB.setOpcode(AArch64::ISB);
3585 TmpInstISB.addOperand(Op: MCOperand::createImm(Val: 0xf));
3586 EmitToStreamer(S&: *OutStreamer, Inst: TmpInstISB);
3587 return;
3588 }
3589 case AArch64::SpeculationBarrierSBEndBB: {
3590 // Print SB
3591 MCInst TmpInstSB;
3592 TmpInstSB.setOpcode(AArch64::SB);
3593 EmitToStreamer(S&: *OutStreamer, Inst: TmpInstSB);
3594 return;
3595 }
3596 case AArch64::TLSDESC_AUTH_CALLSEQ: {
3597 /// lower this to:
3598 /// adrp x0, :tlsdesc_auth:var
3599 /// ldr x16, [x0, #:tlsdesc_auth_lo12:var]
3600 /// add x0, x0, #:tlsdesc_auth_lo12:var
3601 /// blraa x16, x0
3602 /// (TPIDR_EL0 offset now in x0)
3603 const MachineOperand &MO_Sym = MI->getOperand(i: 0);
3604 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
3605 MCOperand SymTLSDescLo12, SymTLSDesc;
3606 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
3607 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
3608 MCInstLowering.lowerOperand(MO: MO_TLSDESC_LO12, MCOp&: SymTLSDescLo12);
3609 MCInstLowering.lowerOperand(MO: MO_TLSDESC, MCOp&: SymTLSDesc);
3610
3611 MCInst Adrp;
3612 Adrp.setOpcode(AArch64::ADRP);
3613 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3614 Adrp.addOperand(Op: SymTLSDesc);
3615 EmitToStreamer(S&: *OutStreamer, Inst: Adrp);
3616
3617 MCInst Ldr;
3618 Ldr.setOpcode(AArch64::LDRXui);
3619 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
3620 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3621 Ldr.addOperand(Op: SymTLSDescLo12);
3622 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
3623 EmitToStreamer(S&: *OutStreamer, Inst: Ldr);
3624
3625 MCInst Add;
3626 Add.setOpcode(AArch64::ADDXri);
3627 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3628 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3629 Add.addOperand(Op: SymTLSDescLo12);
3630 Add.addOperand(Op: MCOperand::createImm(Val: AArch64_AM::getShiftValue(Imm: 0)));
3631 EmitToStreamer(S&: *OutStreamer, Inst: Add);
3632
3633 // Authenticated TLSDESC accesses are not relaxed.
3634 // Thus, do not emit .tlsdesccall for AUTH TLSDESC.
3635
3636 MCInst Blraa;
3637 Blraa.setOpcode(AArch64::BLRAA);
3638 Blraa.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
3639 Blraa.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3640 EmitToStreamer(S&: *OutStreamer, Inst: Blraa);
3641
3642 return;
3643 }
3644 case AArch64::TLSDESC_CALLSEQ: {
3645 /// lower this to:
3646 /// adrp x0, :tlsdesc:var
3647 /// ldr x1, [x0, #:tlsdesc_lo12:var]
3648 /// add x0, x0, #:tlsdesc_lo12:var
3649 /// .tlsdesccall var
3650 /// blr x1
3651 /// (TPIDR_EL0 offset now in x0)
3652 const MachineOperand &MO_Sym = MI->getOperand(i: 0);
3653 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
3654 MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
3655 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
3656 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
3657 MCInstLowering.lowerOperand(MO: MO_Sym, MCOp&: Sym);
3658 MCInstLowering.lowerOperand(MO: MO_TLSDESC_LO12, MCOp&: SymTLSDescLo12);
3659 MCInstLowering.lowerOperand(MO: MO_TLSDESC, MCOp&: SymTLSDesc);
3660
3661 MCInst Adrp;
3662 Adrp.setOpcode(AArch64::ADRP);
3663 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3664 Adrp.addOperand(Op: SymTLSDesc);
3665 EmitToStreamer(S&: *OutStreamer, Inst: Adrp);
3666
3667 MCInst Ldr;
3668 if (STI->isTargetILP32()) {
3669 Ldr.setOpcode(AArch64::LDRWui);
3670 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::W1));
3671 } else {
3672 Ldr.setOpcode(AArch64::LDRXui);
3673 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X1));
3674 }
3675 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3676 Ldr.addOperand(Op: SymTLSDescLo12);
3677 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
3678 EmitToStreamer(S&: *OutStreamer, Inst: Ldr);
3679
3680 MCInst Add;
3681 if (STI->isTargetILP32()) {
3682 Add.setOpcode(AArch64::ADDWri);
3683 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::W0));
3684 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::W0));
3685 } else {
3686 Add.setOpcode(AArch64::ADDXri);
3687 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3688 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
3689 }
3690 Add.addOperand(Op: SymTLSDescLo12);
3691 Add.addOperand(Op: MCOperand::createImm(Val: AArch64_AM::getShiftValue(Imm: 0)));
3692 EmitToStreamer(S&: *OutStreamer, Inst: Add);
3693
3694 // Emit a relocation-annotation. This expands to no code, but requests
3695 // the following instruction gets an R_AARCH64_TLSDESC_CALL.
3696 MCInst TLSDescCall;
3697 TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
3698 TLSDescCall.addOperand(Op: Sym);
3699 EmitToStreamer(S&: *OutStreamer, Inst: TLSDescCall);
3700#ifndef NDEBUG
3701 --InstsEmitted; // no code emitted
3702#endif
3703
3704 MCInst Blr;
3705 Blr.setOpcode(AArch64::BLR);
3706 Blr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X1));
3707 EmitToStreamer(S&: *OutStreamer, Inst: Blr);
3708
3709 return;
3710 }
3711
3712 case AArch64::JumpTableDest32:
3713 case AArch64::JumpTableDest16:
3714 case AArch64::JumpTableDest8:
3715 LowerJumpTableDest(OutStreamer&: *OutStreamer, MI: *MI);
3716 return;
3717
3718 case AArch64::BR_JumpTable:
3719 LowerHardenedBRJumpTable(MI: *MI);
3720 return;
3721
3722 case AArch64::FMOVH0:
3723 case AArch64::FMOVS0:
3724 case AArch64::FMOVD0:
3725 emitFMov0(MI: *MI);
3726 return;
3727
3728 case AArch64::MOPSMemoryCopyPseudo:
3729 case AArch64::MOPSMemoryMovePseudo:
3730 case AArch64::MOPSMemorySetPseudo:
3731 case AArch64::MOPSMemorySetTaggingPseudo:
3732 LowerMOPS(OutStreamer&: *OutStreamer, MI: *MI);
3733 return;
3734
3735 case TargetOpcode::STACKMAP:
3736 return LowerSTACKMAP(OutStreamer&: *OutStreamer, SM, MI: *MI);
3737
3738 case TargetOpcode::PATCHPOINT:
3739 return LowerPATCHPOINT(OutStreamer&: *OutStreamer, SM, MI: *MI);
3740
3741 case TargetOpcode::STATEPOINT:
3742 return LowerSTATEPOINT(OutStreamer&: *OutStreamer, SM, MI: *MI);
3743
3744 case TargetOpcode::FAULTING_OP:
3745 return LowerFAULTING_OP(FaultingMI: *MI);
3746
3747 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
3748 LowerPATCHABLE_FUNCTION_ENTER(MI: *MI);
3749 return;
3750
3751 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
3752 LowerPATCHABLE_FUNCTION_EXIT(MI: *MI);
3753 return;
3754
3755 case TargetOpcode::PATCHABLE_TAIL_CALL:
3756 LowerPATCHABLE_TAIL_CALL(MI: *MI);
3757 return;
3758 case TargetOpcode::PATCHABLE_EVENT_CALL:
3759 return LowerPATCHABLE_EVENT_CALL(MI: *MI, Typed: false);
3760 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
3761 return LowerPATCHABLE_EVENT_CALL(MI: *MI, Typed: true);
3762
3763 case AArch64::KCFI_CHECK:
3764 LowerKCFI_CHECK(MI: *MI);
3765 return;
3766
3767 case AArch64::HWASAN_CHECK_MEMACCESS:
3768 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
3769 case AArch64::HWASAN_CHECK_MEMACCESS_FIXEDSHADOW:
3770 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW:
3771 LowerHWASAN_CHECK_MEMACCESS(MI: *MI);
3772 return;
3773
3774 case AArch64::SEH_StackAlloc:
3775 TS->emitARM64WinCFIAllocStack(Size: MI->getOperand(i: 0).getImm());
3776 return;
3777
3778 case AArch64::SEH_SaveFPLR:
3779 TS->emitARM64WinCFISaveFPLR(Offset: MI->getOperand(i: 0).getImm());
3780 return;
3781
3782 case AArch64::SEH_SaveFPLR_X:
3783 assert(MI->getOperand(0).getImm() < 0 &&
3784 "Pre increment SEH opcode must have a negative offset");
3785 TS->emitARM64WinCFISaveFPLRX(Offset: -MI->getOperand(i: 0).getImm());
3786 return;
3787
3788 case AArch64::SEH_SaveReg:
3789 TS->emitARM64WinCFISaveReg(Reg: MI->getOperand(i: 0).getImm(),
3790 Offset: MI->getOperand(i: 1).getImm());
3791 return;
3792
3793 case AArch64::SEH_SaveReg_X:
3794 assert(MI->getOperand(1).getImm() < 0 &&
3795 "Pre increment SEH opcode must have a negative offset");
3796 TS->emitARM64WinCFISaveRegX(Reg: MI->getOperand(i: 0).getImm(),
3797 Offset: -MI->getOperand(i: 1).getImm());
3798 return;
3799
3800 case AArch64::SEH_SaveRegP:
3801 if (MI->getOperand(i: 1).getImm() == 30 && MI->getOperand(i: 0).getImm() >= 19 &&
3802 MI->getOperand(i: 0).getImm() <= 28) {
3803 assert((MI->getOperand(0).getImm() - 19) % 2 == 0 &&
3804 "Register paired with LR must be odd");
3805 TS->emitARM64WinCFISaveLRPair(Reg: MI->getOperand(i: 0).getImm(),
3806 Offset: MI->getOperand(i: 2).getImm());
3807 return;
3808 }
3809 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
3810 "Non-consecutive registers not allowed for save_regp");
3811 TS->emitARM64WinCFISaveRegP(Reg: MI->getOperand(i: 0).getImm(),
3812 Offset: MI->getOperand(i: 2).getImm());
3813 return;
3814
3815 case AArch64::SEH_SaveRegP_X:
3816 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
3817 "Non-consecutive registers not allowed for save_regp_x");
3818 assert(MI->getOperand(2).getImm() < 0 &&
3819 "Pre increment SEH opcode must have a negative offset");
3820 TS->emitARM64WinCFISaveRegPX(Reg: MI->getOperand(i: 0).getImm(),
3821 Offset: -MI->getOperand(i: 2).getImm());
3822 return;
3823
3824 case AArch64::SEH_SaveFReg:
3825 TS->emitARM64WinCFISaveFReg(Reg: MI->getOperand(i: 0).getImm(),
3826 Offset: MI->getOperand(i: 1).getImm());
3827 return;
3828
3829 case AArch64::SEH_SaveFReg_X:
3830 assert(MI->getOperand(1).getImm() < 0 &&
3831 "Pre increment SEH opcode must have a negative offset");
3832 TS->emitARM64WinCFISaveFRegX(Reg: MI->getOperand(i: 0).getImm(),
3833 Offset: -MI->getOperand(i: 1).getImm());
3834 return;
3835
3836 case AArch64::SEH_SaveFRegP:
3837 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
3838 "Non-consecutive registers not allowed for save_regp");
3839 TS->emitARM64WinCFISaveFRegP(Reg: MI->getOperand(i: 0).getImm(),
3840 Offset: MI->getOperand(i: 2).getImm());
3841 return;
3842
3843 case AArch64::SEH_SaveFRegP_X:
3844 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
3845 "Non-consecutive registers not allowed for save_regp_x");
3846 assert(MI->getOperand(2).getImm() < 0 &&
3847 "Pre increment SEH opcode must have a negative offset");
3848 TS->emitARM64WinCFISaveFRegPX(Reg: MI->getOperand(i: 0).getImm(),
3849 Offset: -MI->getOperand(i: 2).getImm());
3850 return;
3851
3852 case AArch64::SEH_SetFP:
3853 TS->emitARM64WinCFISetFP();
3854 return;
3855
3856 case AArch64::SEH_AddFP:
3857 TS->emitARM64WinCFIAddFP(Size: MI->getOperand(i: 0).getImm());
3858 return;
3859
3860 case AArch64::SEH_Nop:
3861 TS->emitARM64WinCFINop();
3862 return;
3863
3864 case AArch64::SEH_PrologEnd:
3865 TS->emitARM64WinCFIPrologEnd();
3866 return;
3867
3868 case AArch64::SEH_EpilogStart:
3869 TS->emitARM64WinCFIEpilogStart();
3870 return;
3871
3872 case AArch64::SEH_EpilogEnd:
3873 TS->emitARM64WinCFIEpilogEnd();
3874 return;
3875
3876 case AArch64::SEH_PACSignLR:
3877 TS->emitARM64WinCFIPACSignLR();
3878 return;
3879
3880 case AArch64::SEH_SaveAnyRegI:
3881 assert(MI->getOperand(1).getImm() <= 1008 &&
3882 "SaveAnyRegQP SEH opcode offset must fit into 6 bits");
3883 TS->emitARM64WinCFISaveAnyRegI(Reg: MI->getOperand(i: 0).getImm(),
3884 Offset: MI->getOperand(i: 1).getImm());
3885 return;
3886
3887 case AArch64::SEH_SaveAnyRegIP:
3888 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
3889 "Non-consecutive registers not allowed for save_any_reg");
3890 assert(MI->getOperand(2).getImm() <= 1008 &&
3891 "SaveAnyRegQP SEH opcode offset must fit into 6 bits");
3892 TS->emitARM64WinCFISaveAnyRegIP(Reg: MI->getOperand(i: 0).getImm(),
3893 Offset: MI->getOperand(i: 2).getImm());
3894 return;
3895
3896 case AArch64::SEH_SaveAnyRegQP:
3897 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
3898 "Non-consecutive registers not allowed for save_any_reg");
3899 assert(MI->getOperand(2).getImm() >= 0 &&
3900 "SaveAnyRegQP SEH opcode offset must be non-negative");
3901 assert(MI->getOperand(2).getImm() <= 1008 &&
3902 "SaveAnyRegQP SEH opcode offset must fit into 6 bits");
3903 TS->emitARM64WinCFISaveAnyRegQP(Reg: MI->getOperand(i: 0).getImm(),
3904 Offset: MI->getOperand(i: 2).getImm());
3905 return;
3906
3907 case AArch64::SEH_SaveAnyRegQPX:
3908 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
3909 "Non-consecutive registers not allowed for save_any_reg");
3910 assert(MI->getOperand(2).getImm() < 0 &&
3911 "SaveAnyRegQPX SEH opcode offset must be negative");
3912 assert(MI->getOperand(2).getImm() >= -1008 &&
3913 "SaveAnyRegQPX SEH opcode offset must fit into 6 bits");
3914 TS->emitARM64WinCFISaveAnyRegQPX(Reg: MI->getOperand(i: 0).getImm(),
3915 Offset: -MI->getOperand(i: 2).getImm());
3916 return;
3917
3918 case AArch64::SEH_AllocZ:
3919 assert(MI->getOperand(0).getImm() >= 0 &&
3920 "AllocZ SEH opcode offset must be non-negative");
3921 assert(MI->getOperand(0).getImm() <= 255 &&
3922 "AllocZ SEH opcode offset must fit into 8 bits");
3923 TS->emitARM64WinCFIAllocZ(Offset: MI->getOperand(i: 0).getImm());
3924 return;
3925
3926 case AArch64::SEH_SaveZReg:
3927 assert(MI->getOperand(1).getImm() >= 0 &&
3928 "SaveZReg SEH opcode offset must be non-negative");
3929 assert(MI->getOperand(1).getImm() <= 255 &&
3930 "SaveZReg SEH opcode offset must fit into 8 bits");
3931 TS->emitARM64WinCFISaveZReg(Reg: MI->getOperand(i: 0).getImm(),
3932 Offset: MI->getOperand(i: 1).getImm());
3933 return;
3934
3935 case AArch64::SEH_SavePReg:
3936 assert(MI->getOperand(1).getImm() >= 0 &&
3937 "SavePReg SEH opcode offset must be non-negative");
3938 assert(MI->getOperand(1).getImm() <= 255 &&
3939 "SavePReg SEH opcode offset must fit into 8 bits");
3940 TS->emitARM64WinCFISavePReg(Reg: MI->getOperand(i: 0).getImm(),
3941 Offset: MI->getOperand(i: 1).getImm());
3942 return;
3943
3944 case AArch64::BLR:
3945 case AArch64::BR: {
3946 recordIfImportCall(BranchInst: MI);
3947 MCInst TmpInst;
3948 MCInstLowering.Lower(MI, OutMI&: TmpInst);
3949 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
3950 return;
3951 }
3952 case AArch64::CBWPri:
3953 case AArch64::CBXPri:
3954 case AArch64::CBBAssertExt:
3955 case AArch64::CBHAssertExt:
3956 case AArch64::CBWPrr:
3957 case AArch64::CBXPrr:
3958 emitCBPseudoExpansion(MI);
3959 return;
3960 }
3961
3962 if (emitDeactivationSymbolRelocation(DS: MI->getDeactivationSymbol()))
3963 return;
3964
3965 // Finally, do the automated lowerings for everything else.
3966 MCInst TmpInst;
3967 MCInstLowering.Lower(MI, OutMI&: TmpInst);
3968 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
3969}
3970
3971void AArch64AsmPrinter::recordIfImportCall(
3972 const llvm::MachineInstr *BranchInst) {
3973 if (!EnableImportCallOptimization)
3974 return;
3975
3976 auto [GV, OpFlags] = BranchInst->getMF()->tryGetCalledGlobal(MI: BranchInst);
3977 if (GV && GV->hasDLLImportStorageClass()) {
3978 auto *CallSiteSymbol = MMI->getContext().createNamedTempSymbol(Name: "impcall");
3979 OutStreamer->emitLabel(Symbol: CallSiteSymbol);
3980
3981 auto *CalledSymbol = MCInstLowering.GetGlobalValueSymbol(GV, TargetFlags: OpFlags);
3982 SectionToImportedFunctionCalls[OutStreamer->getCurrentSectionOnly()]
3983 .push_back(x: {CallSiteSymbol, CalledSymbol});
3984 }
3985}
3986
3987void AArch64AsmPrinter::emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI,
3988 MCSymbol *LazyPointer) {
3989 // _ifunc:
3990 // adrp x16, lazy_pointer@GOTPAGE
3991 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF]
3992 // ldr x16, [x16]
3993 // br x16
3994
3995 {
3996 MCInst Adrp;
3997 Adrp.setOpcode(AArch64::ADRP);
3998 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
3999 MCOperand SymPage;
4000 MCInstLowering.lowerOperand(
4001 MO: MachineOperand::CreateMCSymbol(Sym: LazyPointer,
4002 TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGE),
4003 MCOp&: SymPage);
4004 Adrp.addOperand(Op: SymPage);
4005 EmitToStreamer(Inst: Adrp);
4006 }
4007
4008 {
4009 MCInst Ldr;
4010 Ldr.setOpcode(AArch64::LDRXui);
4011 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
4012 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
4013 MCOperand SymPageOff;
4014 MCInstLowering.lowerOperand(
4015 MO: MachineOperand::CreateMCSymbol(Sym: LazyPointer, TargetFlags: AArch64II::MO_GOT |
4016 AArch64II::MO_PAGEOFF),
4017 MCOp&: SymPageOff);
4018 Ldr.addOperand(Op: SymPageOff);
4019 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
4020 EmitToStreamer(Inst: Ldr);
4021 }
4022
4023 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDRXui)
4024 .addReg(Reg: AArch64::X16)
4025 .addReg(Reg: AArch64::X16)
4026 .addImm(Val: 0));
4027
4028 EmitToStreamer(Inst: MCInstBuilder(TM.getTargetTriple().isArm64e() ? AArch64::BRAAZ
4029 : AArch64::BR)
4030 .addReg(Reg: AArch64::X16));
4031}
4032
4033void AArch64AsmPrinter::emitMachOIFuncStubHelperBody(Module &M,
4034 const GlobalIFunc &GI,
4035 MCSymbol *LazyPointer) {
4036 // These stub helpers are only ever called once, so here we're optimizing for
4037 // minimum size by using the pre-indexed store variants, which saves a few
4038 // bytes of instructions to bump & restore sp.
4039
4040 // _ifunc.stub_helper:
4041 // stp fp, lr, [sp, #-16]!
4042 // mov fp, sp
4043 // stp x1, x0, [sp, #-16]!
4044 // stp x3, x2, [sp, #-16]!
4045 // stp x5, x4, [sp, #-16]!
4046 // stp x7, x6, [sp, #-16]!
4047 // stp d1, d0, [sp, #-16]!
4048 // stp d3, d2, [sp, #-16]!
4049 // stp d5, d4, [sp, #-16]!
4050 // stp d7, d6, [sp, #-16]!
4051 // bl _resolver
4052 // adrp x16, lazy_pointer@GOTPAGE
4053 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF]
4054 // str x0, [x16]
4055 // mov x16, x0
4056 // ldp d7, d6, [sp], #16
4057 // ldp d5, d4, [sp], #16
4058 // ldp d3, d2, [sp], #16
4059 // ldp d1, d0, [sp], #16
4060 // ldp x7, x6, [sp], #16
4061 // ldp x5, x4, [sp], #16
4062 // ldp x3, x2, [sp], #16
4063 // ldp x1, x0, [sp], #16
4064 // ldp fp, lr, [sp], #16
4065 // br x16
4066
4067 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPXpre)
4068 .addReg(Reg: AArch64::SP)
4069 .addReg(Reg: AArch64::FP)
4070 .addReg(Reg: AArch64::LR)
4071 .addReg(Reg: AArch64::SP)
4072 .addImm(Val: -2));
4073
4074 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
4075 .addReg(Reg: AArch64::FP)
4076 .addReg(Reg: AArch64::SP)
4077 .addImm(Val: 0)
4078 .addImm(Val: 0));
4079
4080 for (int I = 0; I != 4; ++I)
4081 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPXpre)
4082 .addReg(Reg: AArch64::SP)
4083 .addReg(Reg: AArch64::X1 + 2 * I)
4084 .addReg(Reg: AArch64::X0 + 2 * I)
4085 .addReg(Reg: AArch64::SP)
4086 .addImm(Val: -2));
4087
4088 for (int I = 0; I != 4; ++I)
4089 EmitToStreamer(Inst: MCInstBuilder(AArch64::STPDpre)
4090 .addReg(Reg: AArch64::SP)
4091 .addReg(Reg: AArch64::D1 + 2 * I)
4092 .addReg(Reg: AArch64::D0 + 2 * I)
4093 .addReg(Reg: AArch64::SP)
4094 .addImm(Val: -2));
4095
4096 EmitToStreamer(
4097 Inst: MCInstBuilder(AArch64::BL)
4098 .addOperand(Op: MCOperand::createExpr(Val: lowerConstant(CV: GI.getResolver()))));
4099
4100 {
4101 MCInst Adrp;
4102 Adrp.setOpcode(AArch64::ADRP);
4103 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
4104 MCOperand SymPage;
4105 MCInstLowering.lowerOperand(
4106 MO: MachineOperand::CreateES(SymName: LazyPointer->getName().data() + 1,
4107 TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGE),
4108 MCOp&: SymPage);
4109 Adrp.addOperand(Op: SymPage);
4110 EmitToStreamer(Inst: Adrp);
4111 }
4112
4113 {
4114 MCInst Ldr;
4115 Ldr.setOpcode(AArch64::LDRXui);
4116 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
4117 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
4118 MCOperand SymPageOff;
4119 MCInstLowering.lowerOperand(
4120 MO: MachineOperand::CreateES(SymName: LazyPointer->getName().data() + 1,
4121 TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGEOFF),
4122 MCOp&: SymPageOff);
4123 Ldr.addOperand(Op: SymPageOff);
4124 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
4125 EmitToStreamer(Inst: Ldr);
4126 }
4127
4128 EmitToStreamer(Inst: MCInstBuilder(AArch64::STRXui)
4129 .addReg(Reg: AArch64::X0)
4130 .addReg(Reg: AArch64::X16)
4131 .addImm(Val: 0));
4132
4133 EmitToStreamer(Inst: MCInstBuilder(AArch64::ADDXri)
4134 .addReg(Reg: AArch64::X16)
4135 .addReg(Reg: AArch64::X0)
4136 .addImm(Val: 0)
4137 .addImm(Val: 0));
4138
4139 for (int I = 3; I != -1; --I)
4140 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDPDpost)
4141 .addReg(Reg: AArch64::SP)
4142 .addReg(Reg: AArch64::D1 + 2 * I)
4143 .addReg(Reg: AArch64::D0 + 2 * I)
4144 .addReg(Reg: AArch64::SP)
4145 .addImm(Val: 2));
4146
4147 for (int I = 3; I != -1; --I)
4148 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDPXpost)
4149 .addReg(Reg: AArch64::SP)
4150 .addReg(Reg: AArch64::X1 + 2 * I)
4151 .addReg(Reg: AArch64::X0 + 2 * I)
4152 .addReg(Reg: AArch64::SP)
4153 .addImm(Val: 2));
4154
4155 EmitToStreamer(Inst: MCInstBuilder(AArch64::LDPXpost)
4156 .addReg(Reg: AArch64::SP)
4157 .addReg(Reg: AArch64::FP)
4158 .addReg(Reg: AArch64::LR)
4159 .addReg(Reg: AArch64::SP)
4160 .addImm(Val: 2));
4161
4162 EmitToStreamer(Inst: MCInstBuilder(TM.getTargetTriple().isArm64e() ? AArch64::BRAAZ
4163 : AArch64::BR)
4164 .addReg(Reg: AArch64::X16));
4165}
4166
4167const MCExpr *AArch64AsmPrinter::lowerConstant(const Constant *CV,
4168 const Constant *BaseCV,
4169 uint64_t Offset) {
4170 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: CV)) {
4171 return MCSymbolRefExpr::create(Symbol: MCInstLowering.GetGlobalValueSymbol(GV, TargetFlags: 0),
4172 Ctx&: OutContext);
4173 }
4174
4175 return AsmPrinter::lowerConstant(CV, BaseCV, Offset);
4176}
4177
4178char AArch64AsmPrinter::ID = 0;
4179
4180INITIALIZE_PASS(AArch64AsmPrinter, "aarch64-asm-printer",
4181 "AArch64 Assembly Printer", false, false)
4182
4183// Force static initialization.
4184extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
4185LLVMInitializeAArch64AsmPrinter() {
4186 RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
4187 RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
4188 RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
4189 RegisterAsmPrinter<AArch64AsmPrinter> W(getTheARM64_32Target());
4190 RegisterAsmPrinter<AArch64AsmPrinter> V(getTheAArch64_32Target());
4191}
4192
4193PreservedAnalyses AArch64AsmPrinterBeginPass::run(Module &M,
4194 ModuleAnalysisManager &MAM) {
4195 AArch64AsmPrinter &AsmPrinter = static_cast<AArch64AsmPrinter &>(
4196 MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter());
4197 setupModuleAsmPrinter(M, MAM, AsmPrinter);
4198 AsmPrinter.doInitialization(M);
4199 return PreservedAnalyses::all();
4200}
4201
4202PreservedAnalyses
4203AArch64AsmPrinterPass::run(MachineFunction &MF,
4204 MachineFunctionAnalysisManager &MFAM) {
4205 AArch64AsmPrinter &AsmPrinter = static_cast<AArch64AsmPrinter &>(
4206 MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(IR&: MF)
4207 .getCachedResult<AsmPrinterAnalysis>(IR&: *MF.getFunction().getParent())
4208 ->getPrinter());
4209 setupMachineFunctionAsmPrinter(MFAM, MF, AsmPrinter);
4210 AsmPrinter.runOnMachineFunction(MF);
4211 return PreservedAnalyses::all();
4212}
4213
4214PreservedAnalyses AArch64AsmPrinterEndPass::run(Module &M,
4215 ModuleAnalysisManager &MAM) {
4216 AArch64AsmPrinter &AsmPrinter = static_cast<AArch64AsmPrinter &>(
4217 MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter());
4218 setupModuleAsmPrinter(M, MAM, AsmPrinter);
4219 AsmPrinter.doFinalization(M);
4220 return PreservedAnalyses::all();
4221}
4222