1//===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a printer that converts from our internal representation
10// of machine-dependent LLVM code to the AArch64 assembly language.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AArch64.h"
15#include "AArch64MCInstLower.h"
16#include "AArch64MachineFunctionInfo.h"
17#include "AArch64RegisterInfo.h"
18#include "AArch64Subtarget.h"
19#include "AArch64TargetObjectFile.h"
20#include "MCTargetDesc/AArch64AddressingModes.h"
21#include "MCTargetDesc/AArch64InstPrinter.h"
22#include "MCTargetDesc/AArch64MCExpr.h"
23#include "MCTargetDesc/AArch64MCTargetDesc.h"
24#include "MCTargetDesc/AArch64TargetStreamer.h"
25#include "TargetInfo/AArch64TargetInfo.h"
26#include "Utils/AArch64BaseInfo.h"
27#include "llvm/ADT/SmallString.h"
28#include "llvm/ADT/SmallVector.h"
29#include "llvm/ADT/StringRef.h"
30#include "llvm/ADT/Twine.h"
31#include "llvm/BinaryFormat/COFF.h"
32#include "llvm/BinaryFormat/ELF.h"
33#include "llvm/BinaryFormat/MachO.h"
34#include "llvm/CodeGen/AsmPrinter.h"
35#include "llvm/CodeGen/FaultMaps.h"
36#include "llvm/CodeGen/MachineBasicBlock.h"
37#include "llvm/CodeGen/MachineFunction.h"
38#include "llvm/CodeGen/MachineInstr.h"
39#include "llvm/CodeGen/MachineJumpTableInfo.h"
40#include "llvm/CodeGen/MachineModuleInfoImpls.h"
41#include "llvm/CodeGen/MachineOperand.h"
42#include "llvm/CodeGen/StackMaps.h"
43#include "llvm/CodeGen/TargetRegisterInfo.h"
44#include "llvm/IR/DataLayout.h"
45#include "llvm/IR/DebugInfoMetadata.h"
46#include "llvm/IR/Module.h"
47#include "llvm/MC/MCAsmInfo.h"
48#include "llvm/MC/MCContext.h"
49#include "llvm/MC/MCInst.h"
50#include "llvm/MC/MCInstBuilder.h"
51#include "llvm/MC/MCSectionELF.h"
52#include "llvm/MC/MCSectionMachO.h"
53#include "llvm/MC/MCStreamer.h"
54#include "llvm/MC/MCSymbol.h"
55#include "llvm/MC/TargetRegistry.h"
56#include "llvm/Support/Casting.h"
57#include "llvm/Support/CommandLine.h"
58#include "llvm/Support/ErrorHandling.h"
59#include "llvm/Support/raw_ostream.h"
60#include "llvm/Target/TargetMachine.h"
61#include "llvm/TargetParser/Triple.h"
62#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
63#include <algorithm>
64#include <cassert>
65#include <cstdint>
66#include <map>
67#include <memory>
68
69using namespace llvm;
70
71enum PtrauthCheckMode { Default, Unchecked, Poison, Trap };
72static cl::opt<PtrauthCheckMode> PtrauthAuthChecks(
73 "aarch64-ptrauth-auth-checks", cl::Hidden,
74 cl::values(clEnumValN(Unchecked, "none", "don't test for failure"),
75 clEnumValN(Poison, "poison", "poison on failure"),
76 clEnumValN(Trap, "trap", "trap on failure")),
77 cl::desc("Check pointer authentication auth/resign failures"),
78 cl::init(Val: Default));
79
80#define DEBUG_TYPE "asm-printer"
81
82namespace {
83
84class AArch64AsmPrinter : public AsmPrinter {
85 AArch64MCInstLower MCInstLowering;
86 FaultMaps FM;
87 const AArch64Subtarget *STI;
88 bool ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = false;
89
90public:
91 AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
92 : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this),
93 FM(*this) {}
94
95 StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
96
97 /// Wrapper for MCInstLowering.lowerOperand() for the
98 /// tblgen'erated pseudo lowering.
99 bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
100 return MCInstLowering.lowerOperand(MO, MCOp);
101 }
102
103 const MCExpr *lowerConstantPtrAuth(const ConstantPtrAuth &CPA) override;
104
105 const MCExpr *lowerBlockAddressConstant(const BlockAddress &BA) override;
106
107 void emitStartOfAsmFile(Module &M) override;
108 void emitJumpTableInfo() override;
109 std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
110 codeview::JumpTableEntrySize>
111 getCodeViewJumpTableInfo(int JTI, const MachineInstr *BranchInstr,
112 const MCSymbol *BranchLabel) const override;
113
114 void emitFunctionEntryLabel() override;
115
116 void LowerJumpTableDest(MCStreamer &OutStreamer, const MachineInstr &MI);
117
118 void LowerHardenedBRJumpTable(const MachineInstr &MI);
119
120 void LowerMOPS(MCStreamer &OutStreamer, const MachineInstr &MI);
121
122 void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
123 const MachineInstr &MI);
124 void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
125 const MachineInstr &MI);
126 void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
127 const MachineInstr &MI);
128 void LowerFAULTING_OP(const MachineInstr &MI);
129
130 void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
131 void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
132 void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
133 void LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI, bool Typed);
134
135 typedef std::tuple<unsigned, bool, uint32_t, bool, uint64_t>
136 HwasanMemaccessTuple;
137 std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
138 void LowerKCFI_CHECK(const MachineInstr &MI);
139 void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
140 void emitHwasanMemaccessSymbols(Module &M);
141
142 void emitSled(const MachineInstr &MI, SledKind Kind);
143
144 // Emit the sequence for BRA/BLRA (authenticate + branch/call).
145 void emitPtrauthBranch(const MachineInstr *MI);
146
147 // Emit the sequence for AUT or AUTPAC.
148 void emitPtrauthAuthResign(const MachineInstr *MI);
149
150 // Emit the sequence to compute a discriminator into x17, or reuse AddrDisc.
151 unsigned emitPtrauthDiscriminator(uint16_t Disc, unsigned AddrDisc,
152 unsigned &InstsEmitted);
153
154 // Emit the sequence for LOADauthptrstatic
155 void LowerLOADauthptrstatic(const MachineInstr &MI);
156
157 // Emit the sequence for LOADgotPAC/MOVaddrPAC (either GOT adrp-ldr or
158 // adrp-add followed by PAC sign)
159 void LowerMOVaddrPAC(const MachineInstr &MI);
160
161 /// tblgen'erated driver function for lowering simple MI->MC
162 /// pseudo instructions.
163 bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
164 const MachineInstr *MI);
165
166 void emitInstruction(const MachineInstr *MI) override;
167
168 void emitFunctionHeaderComment() override;
169
170 void getAnalysisUsage(AnalysisUsage &AU) const override {
171 AsmPrinter::getAnalysisUsage(AU);
172 AU.setPreservesAll();
173 }
174
175 bool runOnMachineFunction(MachineFunction &MF) override {
176 AArch64FI = MF.getInfo<AArch64FunctionInfo>();
177 STI = &MF.getSubtarget<AArch64Subtarget>();
178
179 SetupMachineFunction(MF);
180
181 if (STI->isTargetCOFF()) {
182 bool Local = MF.getFunction().hasLocalLinkage();
183 COFF::SymbolStorageClass Scl =
184 Local ? COFF::IMAGE_SYM_CLASS_STATIC : COFF::IMAGE_SYM_CLASS_EXTERNAL;
185 int Type =
186 COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
187
188 OutStreamer->beginCOFFSymbolDef(Symbol: CurrentFnSym);
189 OutStreamer->emitCOFFSymbolStorageClass(StorageClass: Scl);
190 OutStreamer->emitCOFFSymbolType(Type);
191 OutStreamer->endCOFFSymbolDef();
192 }
193
194 // Emit the rest of the function body.
195 emitFunctionBody();
196
197 // Emit the XRay table for this function.
198 emitXRayTable();
199
200 // We didn't modify anything.
201 return false;
202 }
203
204 const MCExpr *lowerConstant(const Constant *CV) override;
205
206private:
207 void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
208 bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
209 bool printAsmRegInClass(const MachineOperand &MO,
210 const TargetRegisterClass *RC, unsigned AltName,
211 raw_ostream &O);
212
213 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
214 const char *ExtraCode, raw_ostream &O) override;
215 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
216 const char *ExtraCode, raw_ostream &O) override;
217
218 void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
219
220 void emitFunctionBodyEnd() override;
221 void emitGlobalAlias(const Module &M, const GlobalAlias &GA) override;
222
223 MCSymbol *GetCPISymbol(unsigned CPID) const override;
224 void emitEndOfAsmFile(Module &M) override;
225
226 AArch64FunctionInfo *AArch64FI = nullptr;
227
228 /// Emit the LOHs contained in AArch64FI.
229 void emitLOHs();
230
231 /// Emit instruction to set float register to zero.
232 void emitFMov0(const MachineInstr &MI);
233
234 using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
235
236 MInstToMCSymbol LOHInstToLabel;
237
238 bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const override {
239 return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags;
240 }
241
242 const MCSubtargetInfo *getIFuncMCSubtargetInfo() const override {
243 assert(STI);
244 return STI;
245 }
246 void emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI,
247 MCSymbol *LazyPointer) override;
248 void emitMachOIFuncStubHelperBody(Module &M, const GlobalIFunc &GI,
249 MCSymbol *LazyPointer) override;
250};
251
252} // end anonymous namespace
253
254void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
255 const Triple &TT = TM.getTargetTriple();
256
257 if (TT.isOSBinFormatCOFF()) {
258 // Emit an absolute @feat.00 symbol
259 MCSymbol *S = MMI->getContext().getOrCreateSymbol(Name: StringRef("@feat.00"));
260 OutStreamer->beginCOFFSymbolDef(Symbol: S);
261 OutStreamer->emitCOFFSymbolStorageClass(StorageClass: COFF::IMAGE_SYM_CLASS_STATIC);
262 OutStreamer->emitCOFFSymbolType(Type: COFF::IMAGE_SYM_DTYPE_NULL);
263 OutStreamer->endCOFFSymbolDef();
264 int64_t Feat00Value = 0;
265
266 if (M.getModuleFlag(Key: "cfguard")) {
267 // Object is CFG-aware.
268 Feat00Value |= COFF::Feat00Flags::GuardCF;
269 }
270
271 if (M.getModuleFlag(Key: "ehcontguard")) {
272 // Object also has EHCont.
273 Feat00Value |= COFF::Feat00Flags::GuardEHCont;
274 }
275
276 if (M.getModuleFlag(Key: "ms-kernel")) {
277 // Object is compiled with /kernel.
278 Feat00Value |= COFF::Feat00Flags::Kernel;
279 }
280
281 OutStreamer->emitSymbolAttribute(Symbol: S, Attribute: MCSA_Global);
282 OutStreamer->emitAssignment(
283 Symbol: S, Value: MCConstantExpr::create(Value: Feat00Value, Ctx&: MMI->getContext()));
284 }
285
286 if (!TT.isOSBinFormatELF())
287 return;
288
289 // Assemble feature flags that may require creation of a note section.
290 unsigned Flags = 0;
291 if (const auto *BTE = mdconst::extract_or_null<ConstantInt>(
292 MD: M.getModuleFlag(Key: "branch-target-enforcement")))
293 if (!BTE->isZero())
294 Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
295
296 if (const auto *GCS = mdconst::extract_or_null<ConstantInt>(
297 MD: M.getModuleFlag(Key: "guarded-control-stack")))
298 if (!GCS->isZero())
299 Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_GCS;
300
301 if (const auto *Sign = mdconst::extract_or_null<ConstantInt>(
302 MD: M.getModuleFlag(Key: "sign-return-address")))
303 if (!Sign->isZero())
304 Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
305
306 uint64_t PAuthABIPlatform = -1;
307 if (const auto *PAP = mdconst::extract_or_null<ConstantInt>(
308 MD: M.getModuleFlag(Key: "aarch64-elf-pauthabi-platform")))
309 PAuthABIPlatform = PAP->getZExtValue();
310 uint64_t PAuthABIVersion = -1;
311 if (const auto *PAV = mdconst::extract_or_null<ConstantInt>(
312 MD: M.getModuleFlag(Key: "aarch64-elf-pauthabi-version")))
313 PAuthABIVersion = PAV->getZExtValue();
314
315 // Emit a .note.gnu.property section with the flags.
316 auto *TS =
317 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
318 TS->emitNoteSection(Flags, PAuthABIPlatform, PAuthABIVersion);
319}
320
321void AArch64AsmPrinter::emitFunctionHeaderComment() {
322 const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>();
323 std::optional<std::string> OutlinerString = FI->getOutliningStyle();
324 if (OutlinerString != std::nullopt)
325 OutStreamer->getCommentOS() << ' ' << OutlinerString;
326}
327
328void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
329{
330 const Function &F = MF->getFunction();
331 if (F.hasFnAttribute(Kind: "patchable-function-entry")) {
332 unsigned Num;
333 if (F.getFnAttribute(Kind: "patchable-function-entry")
334 .getValueAsString()
335 .getAsInteger(Radix: 10, Result&: Num))
336 return;
337 emitNops(N: Num);
338 return;
339 }
340
341 emitSled(MI, Kind: SledKind::FUNCTION_ENTER);
342}
343
344void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI) {
345 emitSled(MI, Kind: SledKind::FUNCTION_EXIT);
346}
347
348void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) {
349 emitSled(MI, Kind: SledKind::TAIL_CALL);
350}
351
352void AArch64AsmPrinter::emitSled(const MachineInstr &MI, SledKind Kind) {
353 static const int8_t NoopsInSledCount = 7;
354 // We want to emit the following pattern:
355 //
356 // .Lxray_sled_N:
357 // ALIGN
358 // B #32
359 // ; 7 NOP instructions (28 bytes)
360 // .tmpN
361 //
362 // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
363 // over the full 32 bytes (8 instructions) with the following pattern:
364 //
365 // STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
366 // LDR W17, #12 ; W17 := function ID
367 // LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
368 // BLR X16 ; call the tracing trampoline
369 // ;DATA: 32 bits of function ID
370 // ;DATA: lower 32 bits of the address of the trampoline
371 // ;DATA: higher 32 bits of the address of the trampoline
372 // LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
373 //
374 OutStreamer->emitCodeAlignment(Alignment: Align(4), STI: &getSubtargetInfo());
375 auto CurSled = OutContext.createTempSymbol(Name: "xray_sled_", AlwaysAddSuffix: true);
376 OutStreamer->emitLabel(Symbol: CurSled);
377 auto Target = OutContext.createTempSymbol();
378
379 // Emit "B #32" instruction, which jumps over the next 28 bytes.
380 // The operand has to be the number of 4-byte instructions to jump over,
381 // including the current instruction.
382 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::B).addImm(Val: 8));
383
384 for (int8_t I = 0; I < NoopsInSledCount; I++)
385 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::HINT).addImm(Val: 0));
386
387 OutStreamer->emitLabel(Symbol: Target);
388 recordSled(Sled: CurSled, MI, Kind, Version: 2);
389}
390
391// Emit the following code for Intrinsic::{xray_customevent,xray_typedevent}
392// (built-in functions __xray_customevent/__xray_typedevent).
393//
394// .Lxray_event_sled_N:
395// b 1f
396// save x0 and x1 (and also x2 for TYPED_EVENT_CALL)
397// set up x0 and x1 (and also x2 for TYPED_EVENT_CALL)
398// bl __xray_CustomEvent or __xray_TypedEvent
399// restore x0 and x1 (and also x2 for TYPED_EVENT_CALL)
400// 1:
401//
402// There are 6 instructions for EVENT_CALL and 9 for TYPED_EVENT_CALL.
403//
404// Then record a sled of kind CUSTOM_EVENT or TYPED_EVENT.
405// After patching, b .+N will become a nop.
406void AArch64AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI,
407 bool Typed) {
408 auto &O = *OutStreamer;
409 MCSymbol *CurSled = OutContext.createTempSymbol(Name: "xray_sled_", AlwaysAddSuffix: true);
410 O.emitLabel(Symbol: CurSled);
411 MCInst MovX0Op0 = MCInstBuilder(AArch64::ORRXrs)
412 .addReg(Reg: AArch64::X0)
413 .addReg(Reg: AArch64::XZR)
414 .addReg(Reg: MI.getOperand(i: 0).getReg())
415 .addImm(Val: 0);
416 MCInst MovX1Op1 = MCInstBuilder(AArch64::ORRXrs)
417 .addReg(Reg: AArch64::X1)
418 .addReg(Reg: AArch64::XZR)
419 .addReg(Reg: MI.getOperand(i: 1).getReg())
420 .addImm(Val: 0);
421 bool MachO = TM.getTargetTriple().isOSBinFormatMachO();
422 auto *Sym = MCSymbolRefExpr::create(
423 Symbol: OutContext.getOrCreateSymbol(
424 Name: Twine(MachO ? "_" : "") +
425 (Typed ? "__xray_TypedEvent" : "__xray_CustomEvent")),
426 Ctx&: OutContext);
427 if (Typed) {
428 O.AddComment(T: "Begin XRay typed event");
429 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::B).addImm(Val: 9));
430 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::STPXpre)
431 .addReg(Reg: AArch64::SP)
432 .addReg(Reg: AArch64::X0)
433 .addReg(Reg: AArch64::X1)
434 .addReg(Reg: AArch64::SP)
435 .addImm(Val: -4));
436 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::STRXui)
437 .addReg(Reg: AArch64::X2)
438 .addReg(Reg: AArch64::SP)
439 .addImm(Val: 2));
440 EmitToStreamer(S&: O, Inst: MovX0Op0);
441 EmitToStreamer(S&: O, Inst: MovX1Op1);
442 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::ORRXrs)
443 .addReg(Reg: AArch64::X2)
444 .addReg(Reg: AArch64::XZR)
445 .addReg(Reg: MI.getOperand(i: 2).getReg())
446 .addImm(Val: 0));
447 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::BL).addExpr(Val: Sym));
448 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::LDRXui)
449 .addReg(Reg: AArch64::X2)
450 .addReg(Reg: AArch64::SP)
451 .addImm(Val: 2));
452 O.AddComment(T: "End XRay typed event");
453 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::LDPXpost)
454 .addReg(Reg: AArch64::SP)
455 .addReg(Reg: AArch64::X0)
456 .addReg(Reg: AArch64::X1)
457 .addReg(Reg: AArch64::SP)
458 .addImm(Val: 4));
459
460 recordSled(Sled: CurSled, MI, Kind: SledKind::TYPED_EVENT, Version: 2);
461 } else {
462 O.AddComment(T: "Begin XRay custom event");
463 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::B).addImm(Val: 6));
464 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::STPXpre)
465 .addReg(Reg: AArch64::SP)
466 .addReg(Reg: AArch64::X0)
467 .addReg(Reg: AArch64::X1)
468 .addReg(Reg: AArch64::SP)
469 .addImm(Val: -2));
470 EmitToStreamer(S&: O, Inst: MovX0Op0);
471 EmitToStreamer(S&: O, Inst: MovX1Op1);
472 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::BL).addExpr(Val: Sym));
473 O.AddComment(T: "End XRay custom event");
474 EmitToStreamer(S&: O, Inst: MCInstBuilder(AArch64::LDPXpost)
475 .addReg(Reg: AArch64::SP)
476 .addReg(Reg: AArch64::X0)
477 .addReg(Reg: AArch64::X1)
478 .addReg(Reg: AArch64::SP)
479 .addImm(Val: 2));
480
481 recordSled(Sled: CurSled, MI, Kind: SledKind::CUSTOM_EVENT, Version: 2);
482 }
483}
484
485void AArch64AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
486 Register AddrReg = MI.getOperand(i: 0).getReg();
487 assert(std::next(MI.getIterator())->isCall() &&
488 "KCFI_CHECK not followed by a call instruction");
489 assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg &&
490 "KCFI_CHECK call target doesn't match call operand");
491
492 // Default to using the intra-procedure-call temporary registers for
493 // comparing the hashes.
494 unsigned ScratchRegs[] = {AArch64::W16, AArch64::W17};
495 if (AddrReg == AArch64::XZR) {
496 // Checking XZR makes no sense. Instead of emitting a load, zero
497 // ScratchRegs[0] and use it for the ESR AddrIndex below.
498 AddrReg = getXRegFromWReg(Reg: ScratchRegs[0]);
499 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ORRXrs)
500 .addReg(Reg: AddrReg)
501 .addReg(Reg: AArch64::XZR)
502 .addReg(Reg: AArch64::XZR)
503 .addImm(Val: 0));
504 } else {
505 // If one of the scratch registers is used for the call target (e.g.
506 // with AArch64::TCRETURNriBTI), we can clobber another caller-saved
507 // temporary register instead (in this case, AArch64::W9) as the check
508 // is immediately followed by the call instruction.
509 for (auto &Reg : ScratchRegs) {
510 if (Reg == getWRegFromXReg(Reg: AddrReg)) {
511 Reg = AArch64::W9;
512 break;
513 }
514 }
515 assert(ScratchRegs[0] != AddrReg && ScratchRegs[1] != AddrReg &&
516 "Invalid scratch registers for KCFI_CHECK");
517
518 // Adjust the offset for patchable-function-prefix. This assumes that
519 // patchable-function-prefix is the same for all functions.
520 int64_t PrefixNops = 0;
521 (void)MI.getMF()
522 ->getFunction()
523 .getFnAttribute(Kind: "patchable-function-prefix")
524 .getValueAsString()
525 .getAsInteger(Radix: 10, Result&: PrefixNops);
526
527 // Load the target function type hash.
528 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::LDURWi)
529 .addReg(Reg: ScratchRegs[0])
530 .addReg(Reg: AddrReg)
531 .addImm(Val: -(PrefixNops * 4 + 4)));
532 }
533
534 // Load the expected type hash.
535 const int64_t Type = MI.getOperand(i: 1).getImm();
536 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::MOVKWi)
537 .addReg(Reg: ScratchRegs[1])
538 .addReg(Reg: ScratchRegs[1])
539 .addImm(Val: Type & 0xFFFF)
540 .addImm(Val: 0));
541 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::MOVKWi)
542 .addReg(Reg: ScratchRegs[1])
543 .addReg(Reg: ScratchRegs[1])
544 .addImm(Val: (Type >> 16) & 0xFFFF)
545 .addImm(Val: 16));
546
547 // Compare the hashes and trap if there's a mismatch.
548 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::SUBSWrs)
549 .addReg(Reg: AArch64::WZR)
550 .addReg(Reg: ScratchRegs[0])
551 .addReg(Reg: ScratchRegs[1])
552 .addImm(Val: 0));
553
554 MCSymbol *Pass = OutContext.createTempSymbol();
555 EmitToStreamer(S&: *OutStreamer,
556 Inst: MCInstBuilder(AArch64::Bcc)
557 .addImm(Val: AArch64CC::EQ)
558 .addExpr(Val: MCSymbolRefExpr::create(Symbol: Pass, Ctx&: OutContext)));
559
560 // The base ESR is 0x8000 and the register information is encoded in bits
561 // 0-9 as follows:
562 // - 0-4: n, where the register Xn contains the target address
563 // - 5-9: m, where the register Wm contains the expected type hash
564 // Where n, m are in [0, 30].
565 unsigned TypeIndex = ScratchRegs[1] - AArch64::W0;
566 unsigned AddrIndex;
567 switch (AddrReg) {
568 default:
569 AddrIndex = AddrReg - AArch64::X0;
570 break;
571 case AArch64::FP:
572 AddrIndex = 29;
573 break;
574 case AArch64::LR:
575 AddrIndex = 30;
576 break;
577 }
578
579 assert(AddrIndex < 31 && TypeIndex < 31);
580
581 unsigned ESR = 0x8000 | ((TypeIndex & 31) << 5) | (AddrIndex & 31);
582 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::BRK).addImm(Val: ESR));
583 OutStreamer->emitLabel(Symbol: Pass);
584}
585
586void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
587 Register Reg = MI.getOperand(i: 0).getReg();
588 bool IsShort =
589 ((MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES) ||
590 (MI.getOpcode() ==
591 AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW));
592 uint32_t AccessInfo = MI.getOperand(i: 1).getImm();
593 bool IsFixedShadow =
594 ((MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_FIXEDSHADOW) ||
595 (MI.getOpcode() ==
596 AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW));
597 uint64_t FixedShadowOffset = IsFixedShadow ? MI.getOperand(i: 2).getImm() : 0;
598
599 MCSymbol *&Sym = HwasanMemaccessSymbols[HwasanMemaccessTuple(
600 Reg, IsShort, AccessInfo, IsFixedShadow, FixedShadowOffset)];
601 if (!Sym) {
602 // FIXME: Make this work on non-ELF.
603 if (!TM.getTargetTriple().isOSBinFormatELF())
604 report_fatal_error(reason: "llvm.hwasan.check.memaccess only supported on ELF");
605
606 std::string SymName = "__hwasan_check_x" + utostr(X: Reg - AArch64::X0) + "_" +
607 utostr(X: AccessInfo);
608 if (IsFixedShadow)
609 SymName += "_fixed_" + utostr(X: FixedShadowOffset);
610 if (IsShort)
611 SymName += "_short_v2";
612 Sym = OutContext.getOrCreateSymbol(Name: SymName);
613 }
614
615 EmitToStreamer(S&: *OutStreamer,
616 Inst: MCInstBuilder(AArch64::BL)
617 .addExpr(Val: MCSymbolRefExpr::create(Symbol: Sym, Ctx&: OutContext)));
618}
619
620void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) {
621 if (HwasanMemaccessSymbols.empty())
622 return;
623
624 const Triple &TT = TM.getTargetTriple();
625 assert(TT.isOSBinFormatELF());
626 std::unique_ptr<MCSubtargetInfo> STI(
627 TM.getTarget().createMCSubtargetInfo(TheTriple: TT.str(), CPU: "", Features: ""));
628 assert(STI && "Unable to create subtarget info");
629
630 MCSymbol *HwasanTagMismatchV1Sym =
631 OutContext.getOrCreateSymbol(Name: "__hwasan_tag_mismatch");
632 MCSymbol *HwasanTagMismatchV2Sym =
633 OutContext.getOrCreateSymbol(Name: "__hwasan_tag_mismatch_v2");
634
635 const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
636 MCSymbolRefExpr::create(Symbol: HwasanTagMismatchV1Sym, Ctx&: OutContext);
637 const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
638 MCSymbolRefExpr::create(Symbol: HwasanTagMismatchV2Sym, Ctx&: OutContext);
639
640 for (auto &P : HwasanMemaccessSymbols) {
641 unsigned Reg = std::get<0>(t: P.first);
642 bool IsShort = std::get<1>(t: P.first);
643 uint32_t AccessInfo = std::get<2>(t: P.first);
644 bool IsFixedShadow = std::get<3>(t: P.first);
645 uint64_t FixedShadowOffset = std::get<4>(t: P.first);
646 const MCSymbolRefExpr *HwasanTagMismatchRef =
647 IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
648 MCSymbol *Sym = P.second;
649
650 bool HasMatchAllTag =
651 (AccessInfo >> HWASanAccessInfo::HasMatchAllShift) & 1;
652 uint8_t MatchAllTag =
653 (AccessInfo >> HWASanAccessInfo::MatchAllShift) & 0xff;
654 unsigned Size =
655 1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf);
656 bool CompileKernel =
657 (AccessInfo >> HWASanAccessInfo::CompileKernelShift) & 1;
658
659 OutStreamer->switchSection(Section: OutContext.getELFSection(
660 Section: ".text.hot", Type: ELF::SHT_PROGBITS,
661 Flags: ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, EntrySize: 0, Group: Sym->getName(),
662 /*IsComdat=*/true));
663
664 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_ELF_TypeFunction);
665 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Weak);
666 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Hidden);
667 OutStreamer->emitLabel(Symbol: Sym);
668
669 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::SBFMXri)
670 .addReg(Reg: AArch64::X16)
671 .addReg(Reg)
672 .addImm(Val: 4)
673 .addImm(Val: 55),
674 STI: *STI);
675
676 if (IsFixedShadow) {
677 // Aarch64 makes it difficult to embed large constants in the code.
678 // Fortuitously, kShadowBaseAlignment == 32, so we use the 32-bit
679 // left-shift option in the MOV instruction. Combined with the 16-bit
680 // immediate, this is enough to represent any offset up to 2**48.
681 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::MOVZXi)
682 .addReg(Reg: AArch64::X17)
683 .addImm(Val: FixedShadowOffset >> 32)
684 .addImm(Val: 32),
685 STI: *STI);
686 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::LDRBBroX)
687 .addReg(Reg: AArch64::W16)
688 .addReg(Reg: AArch64::X17)
689 .addReg(Reg: AArch64::X16)
690 .addImm(Val: 0)
691 .addImm(Val: 0),
692 STI: *STI);
693 } else {
694 OutStreamer->emitInstruction(
695 Inst: MCInstBuilder(AArch64::LDRBBroX)
696 .addReg(Reg: AArch64::W16)
697 .addReg(Reg: IsShort ? AArch64::X20 : AArch64::X9)
698 .addReg(Reg: AArch64::X16)
699 .addImm(Val: 0)
700 .addImm(Val: 0),
701 STI: *STI);
702 }
703
704 OutStreamer->emitInstruction(
705 Inst: MCInstBuilder(AArch64::SUBSXrs)
706 .addReg(Reg: AArch64::XZR)
707 .addReg(Reg: AArch64::X16)
708 .addReg(Reg)
709 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSR, Imm: 56)),
710 STI: *STI);
711 MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
712 OutStreamer->emitInstruction(
713 Inst: MCInstBuilder(AArch64::Bcc)
714 .addImm(Val: AArch64CC::NE)
715 .addExpr(Val: MCSymbolRefExpr::create(Symbol: HandleMismatchOrPartialSym,
716 Ctx&: OutContext)),
717 STI: *STI);
718 MCSymbol *ReturnSym = OutContext.createTempSymbol();
719 OutStreamer->emitLabel(Symbol: ReturnSym);
720 OutStreamer->emitInstruction(
721 Inst: MCInstBuilder(AArch64::RET).addReg(Reg: AArch64::LR), STI: *STI);
722 OutStreamer->emitLabel(Symbol: HandleMismatchOrPartialSym);
723
724 if (HasMatchAllTag) {
725 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::UBFMXri)
726 .addReg(Reg: AArch64::X17)
727 .addReg(Reg)
728 .addImm(Val: 56)
729 .addImm(Val: 63),
730 STI: *STI);
731 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::SUBSXri)
732 .addReg(Reg: AArch64::XZR)
733 .addReg(Reg: AArch64::X17)
734 .addImm(Val: MatchAllTag)
735 .addImm(Val: 0),
736 STI: *STI);
737 OutStreamer->emitInstruction(
738 Inst: MCInstBuilder(AArch64::Bcc)
739 .addImm(Val: AArch64CC::EQ)
740 .addExpr(Val: MCSymbolRefExpr::create(Symbol: ReturnSym, Ctx&: OutContext)),
741 STI: *STI);
742 }
743
744 if (IsShort) {
745 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::SUBSWri)
746 .addReg(Reg: AArch64::WZR)
747 .addReg(Reg: AArch64::W16)
748 .addImm(Val: 15)
749 .addImm(Val: 0),
750 STI: *STI);
751 MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
752 OutStreamer->emitInstruction(
753 Inst: MCInstBuilder(AArch64::Bcc)
754 .addImm(Val: AArch64CC::HI)
755 .addExpr(Val: MCSymbolRefExpr::create(Symbol: HandleMismatchSym, Ctx&: OutContext)),
756 STI: *STI);
757
758 OutStreamer->emitInstruction(
759 Inst: MCInstBuilder(AArch64::ANDXri)
760 .addReg(Reg: AArch64::X17)
761 .addReg(Reg)
762 .addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: 0xf, regSize: 64)),
763 STI: *STI);
764 if (Size != 1)
765 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::ADDXri)
766 .addReg(Reg: AArch64::X17)
767 .addReg(Reg: AArch64::X17)
768 .addImm(Val: Size - 1)
769 .addImm(Val: 0),
770 STI: *STI);
771 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::SUBSWrs)
772 .addReg(Reg: AArch64::WZR)
773 .addReg(Reg: AArch64::W16)
774 .addReg(Reg: AArch64::W17)
775 .addImm(Val: 0),
776 STI: *STI);
777 OutStreamer->emitInstruction(
778 Inst: MCInstBuilder(AArch64::Bcc)
779 .addImm(Val: AArch64CC::LS)
780 .addExpr(Val: MCSymbolRefExpr::create(Symbol: HandleMismatchSym, Ctx&: OutContext)),
781 STI: *STI);
782
783 OutStreamer->emitInstruction(
784 Inst: MCInstBuilder(AArch64::ORRXri)
785 .addReg(Reg: AArch64::X16)
786 .addReg(Reg)
787 .addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: 0xf, regSize: 64)),
788 STI: *STI);
789 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::LDRBBui)
790 .addReg(Reg: AArch64::W16)
791 .addReg(Reg: AArch64::X16)
792 .addImm(Val: 0),
793 STI: *STI);
794 OutStreamer->emitInstruction(
795 Inst: MCInstBuilder(AArch64::SUBSXrs)
796 .addReg(Reg: AArch64::XZR)
797 .addReg(Reg: AArch64::X16)
798 .addReg(Reg)
799 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSR, Imm: 56)),
800 STI: *STI);
801 OutStreamer->emitInstruction(
802 Inst: MCInstBuilder(AArch64::Bcc)
803 .addImm(Val: AArch64CC::EQ)
804 .addExpr(Val: MCSymbolRefExpr::create(Symbol: ReturnSym, Ctx&: OutContext)),
805 STI: *STI);
806
807 OutStreamer->emitLabel(Symbol: HandleMismatchSym);
808 }
809
810 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::STPXpre)
811 .addReg(Reg: AArch64::SP)
812 .addReg(Reg: AArch64::X0)
813 .addReg(Reg: AArch64::X1)
814 .addReg(Reg: AArch64::SP)
815 .addImm(Val: -32),
816 STI: *STI);
817 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::STPXi)
818 .addReg(Reg: AArch64::FP)
819 .addReg(Reg: AArch64::LR)
820 .addReg(Reg: AArch64::SP)
821 .addImm(Val: 29),
822 STI: *STI);
823
824 if (Reg != AArch64::X0)
825 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::ORRXrs)
826 .addReg(Reg: AArch64::X0)
827 .addReg(Reg: AArch64::XZR)
828 .addReg(Reg)
829 .addImm(Val: 0),
830 STI: *STI);
831 OutStreamer->emitInstruction(
832 Inst: MCInstBuilder(AArch64::MOVZXi)
833 .addReg(Reg: AArch64::X1)
834 .addImm(Val: AccessInfo & HWASanAccessInfo::RuntimeMask)
835 .addImm(Val: 0),
836 STI: *STI);
837
838 if (CompileKernel) {
839 // The Linux kernel's dynamic loader doesn't support GOT relative
840 // relocations, but it doesn't support late binding either, so just call
841 // the function directly.
842 OutStreamer->emitInstruction(
843 Inst: MCInstBuilder(AArch64::B).addExpr(Val: HwasanTagMismatchRef), STI: *STI);
844 } else {
845 // Intentionally load the GOT entry and branch to it, rather than possibly
846 // late binding the function, which may clobber the registers before we
847 // have a chance to save them.
848 OutStreamer->emitInstruction(
849 Inst: MCInstBuilder(AArch64::ADRP)
850 .addReg(Reg: AArch64::X16)
851 .addExpr(Val: AArch64MCExpr::create(
852 Expr: HwasanTagMismatchRef, Kind: AArch64MCExpr::VariantKind::VK_GOT_PAGE,
853 Ctx&: OutContext)),
854 STI: *STI);
855 OutStreamer->emitInstruction(
856 Inst: MCInstBuilder(AArch64::LDRXui)
857 .addReg(Reg: AArch64::X16)
858 .addReg(Reg: AArch64::X16)
859 .addExpr(Val: AArch64MCExpr::create(
860 Expr: HwasanTagMismatchRef, Kind: AArch64MCExpr::VariantKind::VK_GOT_LO12,
861 Ctx&: OutContext)),
862 STI: *STI);
863 OutStreamer->emitInstruction(
864 Inst: MCInstBuilder(AArch64::BR).addReg(Reg: AArch64::X16), STI: *STI);
865 }
866 }
867}
868
869static void emitAuthenticatedPointer(MCStreamer &OutStreamer,
870 MCSymbol *StubLabel,
871 const MCExpr *StubAuthPtrRef) {
872 // sym$auth_ptr$key$disc:
873 OutStreamer.emitLabel(Symbol: StubLabel);
874 OutStreamer.emitValue(Value: StubAuthPtrRef, /*size=*/Size: 8);
875}
876
877void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
878 emitHwasanMemaccessSymbols(M);
879
880 const Triple &TT = TM.getTargetTriple();
881 if (TT.isOSBinFormatMachO()) {
882 // Output authenticated pointers as indirect symbols, if we have any.
883 MachineModuleInfoMachO &MMIMacho =
884 MMI->getObjFileInfo<MachineModuleInfoMachO>();
885
886 auto Stubs = MMIMacho.getAuthGVStubList();
887
888 if (!Stubs.empty()) {
889 // Switch to the "__auth_ptr" section.
890 OutStreamer->switchSection(
891 Section: OutContext.getMachOSection(Segment: "__DATA", Section: "__auth_ptr", TypeAndAttributes: MachO::S_REGULAR,
892 K: SectionKind::getMetadata()));
893 emitAlignment(Alignment: Align(8));
894
895 for (const auto &Stub : Stubs)
896 emitAuthenticatedPointer(OutStreamer&: *OutStreamer, StubLabel: Stub.first, StubAuthPtrRef: Stub.second);
897
898 OutStreamer->addBlankLine();
899 }
900
901 // Funny Darwin hack: This flag tells the linker that no global symbols
902 // contain code that falls through to other global symbols (e.g. the obvious
903 // implementation of multiple entry points). If this doesn't occur, the
904 // linker can safely perform dead code stripping. Since LLVM never
905 // generates code that does this, it is always safe to set.
906 OutStreamer->emitAssemblerFlag(Flag: MCAF_SubsectionsViaSymbols);
907 }
908
909 if (TT.isOSBinFormatELF()) {
910 // Output authenticated pointers as indirect symbols, if we have any.
911 MachineModuleInfoELF &MMIELF = MMI->getObjFileInfo<MachineModuleInfoELF>();
912
913 auto Stubs = MMIELF.getAuthGVStubList();
914
915 if (!Stubs.empty()) {
916 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
917 OutStreamer->switchSection(Section: TLOF.getDataSection());
918 emitAlignment(Alignment: Align(8));
919
920 for (const auto &Stub : Stubs)
921 emitAuthenticatedPointer(OutStreamer&: *OutStreamer, StubLabel: Stub.first, StubAuthPtrRef: Stub.second);
922
923 OutStreamer->addBlankLine();
924 }
925 }
926
927 // Emit stack and fault map information.
928 FM.serializeToFaultMapSection();
929
930}
931
932void AArch64AsmPrinter::emitLOHs() {
933 SmallVector<MCSymbol *, 3> MCArgs;
934
935 for (const auto &D : AArch64FI->getLOHContainer()) {
936 for (const MachineInstr *MI : D.getArgs()) {
937 MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(x: MI);
938 assert(LabelIt != LOHInstToLabel.end() &&
939 "Label hasn't been inserted for LOH related instruction");
940 MCArgs.push_back(Elt: LabelIt->second);
941 }
942 OutStreamer->emitLOHDirective(Kind: D.getKind(), Args: MCArgs);
943 MCArgs.clear();
944 }
945}
946
947void AArch64AsmPrinter::emitFunctionBodyEnd() {
948 if (!AArch64FI->getLOHRelated().empty())
949 emitLOHs();
950}
951
952/// GetCPISymbol - Return the symbol for the specified constant pool entry.
953MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
954 // Darwin uses a linker-private symbol name for constant-pools (to
955 // avoid addends on the relocation?), ELF has no such concept and
956 // uses a normal private symbol.
957 if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
958 return OutContext.getOrCreateSymbol(
959 Name: Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
960 Twine(getFunctionNumber()) + "_" + Twine(CPID));
961
962 return AsmPrinter::GetCPISymbol(CPID);
963}
964
965void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
966 raw_ostream &O) {
967 const MachineOperand &MO = MI->getOperand(i: OpNum);
968 switch (MO.getType()) {
969 default:
970 llvm_unreachable("<unknown operand type>");
971 case MachineOperand::MO_Register: {
972 Register Reg = MO.getReg();
973 assert(Reg.isPhysical());
974 assert(!MO.getSubReg() && "Subregs should be eliminated!");
975 O << AArch64InstPrinter::getRegisterName(Reg);
976 break;
977 }
978 case MachineOperand::MO_Immediate: {
979 O << MO.getImm();
980 break;
981 }
982 case MachineOperand::MO_GlobalAddress: {
983 PrintSymbolOperand(MO, OS&: O);
984 break;
985 }
986 case MachineOperand::MO_BlockAddress: {
987 MCSymbol *Sym = GetBlockAddressSymbol(BA: MO.getBlockAddress());
988 Sym->print(OS&: O, MAI);
989 break;
990 }
991 }
992}
993
994bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
995 raw_ostream &O) {
996 Register Reg = MO.getReg();
997 switch (Mode) {
998 default:
999 return true; // Unknown mode.
1000 case 'w':
1001 Reg = getWRegFromXReg(Reg);
1002 break;
1003 case 'x':
1004 Reg = getXRegFromWReg(Reg);
1005 break;
1006 case 't':
1007 Reg = getXRegFromXRegTuple(RegTuple: Reg);
1008 break;
1009 }
1010
1011 O << AArch64InstPrinter::getRegisterName(Reg);
1012 return false;
1013}
1014
1015// Prints the register in MO using class RC using the offset in the
1016// new register class. This should not be used for cross class
1017// printing.
1018bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
1019 const TargetRegisterClass *RC,
1020 unsigned AltName, raw_ostream &O) {
1021 assert(MO.isReg() && "Should only get here with a register!");
1022 const TargetRegisterInfo *RI = STI->getRegisterInfo();
1023 Register Reg = MO.getReg();
1024 unsigned RegToPrint = RC->getRegister(i: RI->getEncodingValue(RegNo: Reg));
1025 if (!RI->regsOverlap(RegA: RegToPrint, RegB: Reg))
1026 return true;
1027 O << AArch64InstPrinter::getRegisterName(Reg: RegToPrint, AltIdx: AltName);
1028 return false;
1029}
1030
1031bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
1032 const char *ExtraCode, raw_ostream &O) {
1033 const MachineOperand &MO = MI->getOperand(i: OpNum);
1034
1035 // First try the generic code, which knows about modifiers like 'c' and 'n'.
1036 if (!AsmPrinter::PrintAsmOperand(MI, OpNo: OpNum, ExtraCode, OS&: O))
1037 return false;
1038
1039 // Does this asm operand have a single letter operand modifier?
1040 if (ExtraCode && ExtraCode[0]) {
1041 if (ExtraCode[1] != 0)
1042 return true; // Unknown modifier.
1043
1044 switch (ExtraCode[0]) {
1045 default:
1046 return true; // Unknown modifier.
1047 case 'w': // Print W register
1048 case 'x': // Print X register
1049 if (MO.isReg())
1050 return printAsmMRegister(MO, Mode: ExtraCode[0], O);
1051 if (MO.isImm() && MO.getImm() == 0) {
1052 unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
1053 O << AArch64InstPrinter::getRegisterName(Reg);
1054 return false;
1055 }
1056 printOperand(MI, OpNum, O);
1057 return false;
1058 case 'b': // Print B register.
1059 case 'h': // Print H register.
1060 case 's': // Print S register.
1061 case 'd': // Print D register.
1062 case 'q': // Print Q register.
1063 case 'z': // Print Z register.
1064 if (MO.isReg()) {
1065 const TargetRegisterClass *RC;
1066 switch (ExtraCode[0]) {
1067 case 'b':
1068 RC = &AArch64::FPR8RegClass;
1069 break;
1070 case 'h':
1071 RC = &AArch64::FPR16RegClass;
1072 break;
1073 case 's':
1074 RC = &AArch64::FPR32RegClass;
1075 break;
1076 case 'd':
1077 RC = &AArch64::FPR64RegClass;
1078 break;
1079 case 'q':
1080 RC = &AArch64::FPR128RegClass;
1081 break;
1082 case 'z':
1083 RC = &AArch64::ZPRRegClass;
1084 break;
1085 default:
1086 return true;
1087 }
1088 return printAsmRegInClass(MO, RC, AltName: AArch64::NoRegAltName, O);
1089 }
1090 printOperand(MI, OpNum, O);
1091 return false;
1092 }
1093 }
1094
1095 // According to ARM, we should emit x and v registers unless we have a
1096 // modifier.
1097 if (MO.isReg()) {
1098 Register Reg = MO.getReg();
1099
1100 // If this is a w or x register, print an x register.
1101 if (AArch64::GPR32allRegClass.contains(Reg) ||
1102 AArch64::GPR64allRegClass.contains(Reg))
1103 return printAsmMRegister(MO, Mode: 'x', O);
1104
1105 // If this is an x register tuple, print an x register.
1106 if (AArch64::GPR64x8ClassRegClass.contains(Reg))
1107 return printAsmMRegister(MO, Mode: 't', O);
1108
1109 unsigned AltName = AArch64::NoRegAltName;
1110 const TargetRegisterClass *RegClass;
1111 if (AArch64::ZPRRegClass.contains(Reg)) {
1112 RegClass = &AArch64::ZPRRegClass;
1113 } else if (AArch64::PPRRegClass.contains(Reg)) {
1114 RegClass = &AArch64::PPRRegClass;
1115 } else if (AArch64::PNRRegClass.contains(Reg)) {
1116 RegClass = &AArch64::PNRRegClass;
1117 } else {
1118 RegClass = &AArch64::FPR128RegClass;
1119 AltName = AArch64::vreg;
1120 }
1121
1122 // If this is a b, h, s, d, or q register, print it as a v register.
1123 return printAsmRegInClass(MO, RC: RegClass, AltName, O);
1124 }
1125
1126 printOperand(MI, OpNum, O);
1127 return false;
1128}
1129
1130bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
1131 unsigned OpNum,
1132 const char *ExtraCode,
1133 raw_ostream &O) {
1134 if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
1135 return true; // Unknown modifier.
1136
1137 const MachineOperand &MO = MI->getOperand(i: OpNum);
1138 assert(MO.isReg() && "unexpected inline asm memory operand");
1139 O << "[" << AArch64InstPrinter::getRegisterName(Reg: MO.getReg()) << "]";
1140 return false;
1141}
1142
1143void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
1144 raw_ostream &OS) {
1145 unsigned NOps = MI->getNumOperands();
1146 assert(NOps == 4);
1147 OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
1148 // cast away const; DIetc do not take const operands for some reason.
1149 OS << MI->getDebugVariable()->getName();
1150 OS << " <- ";
1151 // Frame address. Currently handles register +- offset only.
1152 assert(MI->isIndirectDebugValue());
1153 OS << '[';
1154 for (unsigned I = 0, E = std::distance(first: MI->debug_operands().begin(),
1155 last: MI->debug_operands().end());
1156 I < E; ++I) {
1157 if (I != 0)
1158 OS << ", ";
1159 printOperand(MI, OpNum: I, O&: OS);
1160 }
1161 OS << ']';
1162 OS << "+";
1163 printOperand(MI, OpNum: NOps - 2, O&: OS);
1164}
1165
1166void AArch64AsmPrinter::emitJumpTableInfo() {
1167 const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
1168 if (!MJTI) return;
1169
1170 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
1171 if (JT.empty()) return;
1172
1173 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
1174 MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(F: MF->getFunction(), TM);
1175 OutStreamer->switchSection(Section: ReadOnlySec);
1176
1177 auto AFI = MF->getInfo<AArch64FunctionInfo>();
1178 for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
1179 const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
1180
1181 // If this jump table was deleted, ignore it.
1182 if (JTBBs.empty()) continue;
1183
1184 unsigned Size = AFI->getJumpTableEntrySize(Idx: JTI);
1185 emitAlignment(Alignment: Align(Size));
1186 OutStreamer->emitLabel(Symbol: GetJTISymbol(JTID: JTI));
1187
1188 const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(Idx: JTI);
1189 const MCExpr *Base = MCSymbolRefExpr::create(Symbol: BaseSym, Ctx&: OutContext);
1190
1191 for (auto *JTBB : JTBBs) {
1192 const MCExpr *Value =
1193 MCSymbolRefExpr::create(Symbol: JTBB->getSymbol(), Ctx&: OutContext);
1194
1195 // Each entry is:
1196 // .byte/.hword (LBB - Lbase)>>2
1197 // or plain:
1198 // .word LBB - Lbase
1199 Value = MCBinaryExpr::createSub(LHS: Value, RHS: Base, Ctx&: OutContext);
1200 if (Size != 4)
1201 Value = MCBinaryExpr::createLShr(
1202 LHS: Value, RHS: MCConstantExpr::create(Value: 2, Ctx&: OutContext), Ctx&: OutContext);
1203
1204 OutStreamer->emitValue(Value, Size);
1205 }
1206 }
1207}
1208
1209std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
1210 codeview::JumpTableEntrySize>
1211AArch64AsmPrinter::getCodeViewJumpTableInfo(int JTI,
1212 const MachineInstr *BranchInstr,
1213 const MCSymbol *BranchLabel) const {
1214 const auto AFI = MF->getInfo<AArch64FunctionInfo>();
1215 const auto Base = AArch64FI->getJumpTableEntryPCRelSymbol(Idx: JTI);
1216 codeview::JumpTableEntrySize EntrySize;
1217 switch (AFI->getJumpTableEntrySize(Idx: JTI)) {
1218 case 1:
1219 EntrySize = codeview::JumpTableEntrySize::UInt8ShiftLeft;
1220 break;
1221 case 2:
1222 EntrySize = codeview::JumpTableEntrySize::UInt16ShiftLeft;
1223 break;
1224 case 4:
1225 EntrySize = codeview::JumpTableEntrySize::Int32;
1226 break;
1227 default:
1228 llvm_unreachable("Unexpected jump table entry size");
1229 }
1230 return std::make_tuple(args: Base, args: 0, args&: BranchLabel, args&: EntrySize);
1231}
1232
1233void AArch64AsmPrinter::emitFunctionEntryLabel() {
1234 if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall ||
1235 MF->getFunction().getCallingConv() ==
1236 CallingConv::AArch64_SVE_VectorCall ||
1237 MF->getInfo<AArch64FunctionInfo>()->isSVECC()) {
1238 auto *TS =
1239 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
1240 TS->emitDirectiveVariantPCS(Symbol: CurrentFnSym);
1241 }
1242
1243 AsmPrinter::emitFunctionEntryLabel();
1244
1245 if (TM.getTargetTriple().isWindowsArm64EC() &&
1246 !MF->getFunction().hasLocalLinkage()) {
1247 // For ARM64EC targets, a function definition's name is mangled differently
1248 // from the normal symbol, emit required aliases here.
1249 auto emitFunctionAlias = [&](MCSymbol *Src, MCSymbol *Dst) {
1250 OutStreamer->emitSymbolAttribute(Symbol: Src, Attribute: MCSA_WeakAntiDep);
1251 OutStreamer->emitAssignment(
1252 Symbol: Src, Value: MCSymbolRefExpr::create(Symbol: Dst, Kind: MCSymbolRefExpr::VK_None,
1253 Ctx&: MMI->getContext()));
1254 };
1255
1256 auto getSymbolFromMetadata = [&](StringRef Name) {
1257 MCSymbol *Sym = nullptr;
1258 if (MDNode *Node = MF->getFunction().getMetadata(Kind: Name)) {
1259 StringRef NameStr = cast<MDString>(Val: Node->getOperand(I: 0))->getString();
1260 Sym = MMI->getContext().getOrCreateSymbol(Name: NameStr);
1261 }
1262 return Sym;
1263 };
1264
1265 if (MCSymbol *UnmangledSym =
1266 getSymbolFromMetadata("arm64ec_unmangled_name")) {
1267 MCSymbol *ECMangledSym = getSymbolFromMetadata("arm64ec_ecmangled_name");
1268
1269 if (ECMangledSym) {
1270 // An external function, emit the alias from the unmangled symbol to
1271 // mangled symbol name and the alias from the mangled symbol to guest
1272 // exit thunk.
1273 emitFunctionAlias(UnmangledSym, ECMangledSym);
1274 emitFunctionAlias(ECMangledSym, CurrentFnSym);
1275 } else {
1276 // A function implementation, emit the alias from the unmangled symbol
1277 // to mangled symbol name.
1278 emitFunctionAlias(UnmangledSym, CurrentFnSym);
1279 }
1280 }
1281 }
1282}
1283
1284void AArch64AsmPrinter::emitGlobalAlias(const Module &M,
1285 const GlobalAlias &GA) {
1286 if (auto F = dyn_cast_or_null<Function>(Val: GA.getAliasee())) {
1287 // Global aliases must point to a definition, but unmangled patchable
1288 // symbols are special and need to point to an undefined symbol with "EXP+"
1289 // prefix. Such undefined symbol is resolved by the linker by creating
1290 // x86 thunk that jumps back to the actual EC target.
1291 if (MDNode *Node = F->getMetadata(Kind: "arm64ec_exp_name")) {
1292 StringRef ExpStr = cast<MDString>(Val: Node->getOperand(I: 0))->getString();
1293 MCSymbol *ExpSym = MMI->getContext().getOrCreateSymbol(Name: ExpStr);
1294 MCSymbol *Sym = MMI->getContext().getOrCreateSymbol(Name: GA.getName());
1295
1296 OutStreamer->beginCOFFSymbolDef(Symbol: ExpSym);
1297 OutStreamer->emitCOFFSymbolStorageClass(StorageClass: COFF::IMAGE_SYM_CLASS_EXTERNAL);
1298 OutStreamer->emitCOFFSymbolType(Type: COFF::IMAGE_SYM_DTYPE_FUNCTION
1299 << COFF::SCT_COMPLEX_TYPE_SHIFT);
1300 OutStreamer->endCOFFSymbolDef();
1301
1302 OutStreamer->beginCOFFSymbolDef(Symbol: Sym);
1303 OutStreamer->emitCOFFSymbolStorageClass(StorageClass: COFF::IMAGE_SYM_CLASS_EXTERNAL);
1304 OutStreamer->emitCOFFSymbolType(Type: COFF::IMAGE_SYM_DTYPE_FUNCTION
1305 << COFF::SCT_COMPLEX_TYPE_SHIFT);
1306 OutStreamer->endCOFFSymbolDef();
1307 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Weak);
1308 OutStreamer->emitAssignment(
1309 Symbol: Sym, Value: MCSymbolRefExpr::create(Symbol: ExpSym, Kind: MCSymbolRefExpr::VK_None,
1310 Ctx&: MMI->getContext()));
1311 return;
1312 }
1313 }
1314 AsmPrinter::emitGlobalAlias(M, GA);
1315}
1316
1317/// Small jump tables contain an unsigned byte or half, representing the offset
1318/// from the lowest-addressed possible destination to the desired basic
1319/// block. Since all instructions are 4-byte aligned, this is further compressed
1320/// by counting in instructions rather than bytes (i.e. divided by 4). So, to
1321/// materialize the correct destination we need:
1322///
1323/// adr xDest, .LBB0_0
1324/// ldrb wScratch, [xTable, xEntry] (with "lsl #1" for ldrh).
1325/// add xDest, xDest, xScratch (with "lsl #2" for smaller entries)
1326void AArch64AsmPrinter::LowerJumpTableDest(llvm::MCStreamer &OutStreamer,
1327 const llvm::MachineInstr &MI) {
1328 Register DestReg = MI.getOperand(i: 0).getReg();
1329 Register ScratchReg = MI.getOperand(i: 1).getReg();
1330 Register ScratchRegW =
1331 STI->getRegisterInfo()->getSubReg(Reg: ScratchReg, Idx: AArch64::sub_32);
1332 Register TableReg = MI.getOperand(i: 2).getReg();
1333 Register EntryReg = MI.getOperand(i: 3).getReg();
1334 int JTIdx = MI.getOperand(i: 4).getIndex();
1335 int Size = AArch64FI->getJumpTableEntrySize(Idx: JTIdx);
1336
1337 // This has to be first because the compression pass based its reachability
1338 // calculations on the start of the JumpTableDest instruction.
1339 auto Label =
1340 MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(Idx: JTIdx);
1341
1342 // If we don't already have a symbol to use as the base, use the ADR
1343 // instruction itself.
1344 if (!Label) {
1345 Label = MF->getContext().createTempSymbol();
1346 AArch64FI->setJumpTableEntryInfo(Idx: JTIdx, Size, PCRelSym: Label);
1347 OutStreamer.emitLabel(Symbol: Label);
1348 }
1349
1350 auto LabelExpr = MCSymbolRefExpr::create(Symbol: Label, Ctx&: MF->getContext());
1351 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::ADR)
1352 .addReg(Reg: DestReg)
1353 .addExpr(Val: LabelExpr));
1354
1355 // Load the number of instruction-steps to offset from the label.
1356 unsigned LdrOpcode;
1357 switch (Size) {
1358 case 1: LdrOpcode = AArch64::LDRBBroX; break;
1359 case 2: LdrOpcode = AArch64::LDRHHroX; break;
1360 case 4: LdrOpcode = AArch64::LDRSWroX; break;
1361 default:
1362 llvm_unreachable("Unknown jump table size");
1363 }
1364
1365 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(LdrOpcode)
1366 .addReg(Reg: Size == 4 ? ScratchReg : ScratchRegW)
1367 .addReg(Reg: TableReg)
1368 .addReg(Reg: EntryReg)
1369 .addImm(Val: 0)
1370 .addImm(Val: Size == 1 ? 0 : 1));
1371
1372 // Add to the already materialized base label address, multiplying by 4 if
1373 // compressed.
1374 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::ADDXrs)
1375 .addReg(Reg: DestReg)
1376 .addReg(Reg: DestReg)
1377 .addReg(Reg: ScratchReg)
1378 .addImm(Val: Size == 4 ? 0 : 2));
1379}
1380
1381void AArch64AsmPrinter::LowerHardenedBRJumpTable(const MachineInstr &MI) {
1382 unsigned InstsEmitted = 0;
1383
1384 const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
1385 assert(MJTI && "Can't lower jump-table dispatch without JTI");
1386
1387 const std::vector<MachineJumpTableEntry> &JTs = MJTI->getJumpTables();
1388 assert(!JTs.empty() && "Invalid JT index for jump-table dispatch");
1389
1390 // Emit:
1391 // mov x17, #<size of table> ; depending on table size, with MOVKs
1392 // cmp x16, x17 ; or #imm if table size fits in 12-bit
1393 // csel x16, x16, xzr, ls ; check for index overflow
1394 //
1395 // adrp x17, Ltable@PAGE ; materialize table address
1396 // add x17, Ltable@PAGEOFF
1397 // ldrsw x16, [x17, x16, lsl #2] ; load table entry
1398 //
1399 // Lanchor:
1400 // adr x17, Lanchor ; compute target address
1401 // add x16, x17, x16
1402 // br x16 ; branch to target
1403
1404 MachineOperand JTOp = MI.getOperand(i: 0);
1405
1406 unsigned JTI = JTOp.getIndex();
1407 assert(!AArch64FI->getJumpTableEntryPCRelSymbol(JTI) &&
1408 "unsupported compressed jump table");
1409
1410 const uint64_t NumTableEntries = JTs[JTI].MBBs.size();
1411
1412 // cmp only supports a 12-bit immediate. If we need more, materialize the
1413 // immediate, using x17 as a scratch register.
1414 uint64_t MaxTableEntry = NumTableEntries - 1;
1415 if (isUInt<12>(x: MaxTableEntry)) {
1416 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::SUBSXri)
1417 .addReg(Reg: AArch64::XZR)
1418 .addReg(Reg: AArch64::X16)
1419 .addImm(Val: MaxTableEntry)
1420 .addImm(Val: 0));
1421 ++InstsEmitted;
1422 } else {
1423 EmitToStreamer(S&: *OutStreamer,
1424 Inst: MCInstBuilder(AArch64::MOVZXi)
1425 .addReg(Reg: AArch64::X17)
1426 .addImm(Val: static_cast<uint16_t>(MaxTableEntry))
1427 .addImm(Val: 0));
1428 ++InstsEmitted;
1429 // It's sad that we have to manually materialize instructions, but we can't
1430 // trivially reuse the main pseudo expansion logic.
1431 // A MOVK sequence is easy enough to generate and handles the general case.
1432 for (int Offset = 16; Offset < 64; Offset += 16) {
1433 if ((MaxTableEntry >> Offset) == 0)
1434 break;
1435 EmitToStreamer(S&: *OutStreamer,
1436 Inst: MCInstBuilder(AArch64::MOVKXi)
1437 .addReg(Reg: AArch64::X17)
1438 .addReg(Reg: AArch64::X17)
1439 .addImm(Val: static_cast<uint16_t>(MaxTableEntry >> Offset))
1440 .addImm(Val: Offset));
1441 ++InstsEmitted;
1442 }
1443 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::SUBSXrs)
1444 .addReg(Reg: AArch64::XZR)
1445 .addReg(Reg: AArch64::X16)
1446 .addReg(Reg: AArch64::X17)
1447 .addImm(Val: 0));
1448 ++InstsEmitted;
1449 }
1450
1451 // This picks entry #0 on failure.
1452 // We might want to trap instead.
1453 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::CSELXr)
1454 .addReg(Reg: AArch64::X16)
1455 .addReg(Reg: AArch64::X16)
1456 .addReg(Reg: AArch64::XZR)
1457 .addImm(Val: AArch64CC::LS));
1458 ++InstsEmitted;
1459
1460 // Prepare the @PAGE/@PAGEOFF low/high operands.
1461 MachineOperand JTMOHi(JTOp), JTMOLo(JTOp);
1462 MCOperand JTMCHi, JTMCLo;
1463
1464 JTMOHi.setTargetFlags(AArch64II::MO_PAGE);
1465 JTMOLo.setTargetFlags(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
1466
1467 MCInstLowering.lowerOperand(MO: JTMOHi, MCOp&: JTMCHi);
1468 MCInstLowering.lowerOperand(MO: JTMOLo, MCOp&: JTMCLo);
1469
1470 EmitToStreamer(
1471 S&: *OutStreamer,
1472 Inst: MCInstBuilder(AArch64::ADRP).addReg(Reg: AArch64::X17).addOperand(Op: JTMCHi));
1473 ++InstsEmitted;
1474
1475 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ADDXri)
1476 .addReg(Reg: AArch64::X17)
1477 .addReg(Reg: AArch64::X17)
1478 .addOperand(Op: JTMCLo)
1479 .addImm(Val: 0));
1480 ++InstsEmitted;
1481
1482 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::LDRSWroX)
1483 .addReg(Reg: AArch64::X16)
1484 .addReg(Reg: AArch64::X17)
1485 .addReg(Reg: AArch64::X16)
1486 .addImm(Val: 0)
1487 .addImm(Val: 1));
1488 ++InstsEmitted;
1489
1490 MCSymbol *AdrLabel = MF->getContext().createTempSymbol();
1491 const auto *AdrLabelE = MCSymbolRefExpr::create(Symbol: AdrLabel, Ctx&: MF->getContext());
1492 AArch64FI->setJumpTableEntryInfo(Idx: JTI, Size: 4, PCRelSym: AdrLabel);
1493
1494 OutStreamer->emitLabel(Symbol: AdrLabel);
1495 EmitToStreamer(
1496 S&: *OutStreamer,
1497 Inst: MCInstBuilder(AArch64::ADR).addReg(Reg: AArch64::X17).addExpr(Val: AdrLabelE));
1498 ++InstsEmitted;
1499
1500 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ADDXrs)
1501 .addReg(Reg: AArch64::X16)
1502 .addReg(Reg: AArch64::X17)
1503 .addReg(Reg: AArch64::X16)
1504 .addImm(Val: 0));
1505 ++InstsEmitted;
1506
1507 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::BR).addReg(Reg: AArch64::X16));
1508 ++InstsEmitted;
1509
1510 (void)InstsEmitted;
1511 assert(STI->getInstrInfo()->getInstSizeInBytes(MI) >= InstsEmitted * 4);
1512}
1513
1514void AArch64AsmPrinter::LowerMOPS(llvm::MCStreamer &OutStreamer,
1515 const llvm::MachineInstr &MI) {
1516 unsigned Opcode = MI.getOpcode();
1517 assert(STI->hasMOPS());
1518 assert(STI->hasMTE() || Opcode != AArch64::MOPSMemorySetTaggingPseudo);
1519
1520 const auto Ops = [Opcode]() -> std::array<unsigned, 3> {
1521 if (Opcode == AArch64::MOPSMemoryCopyPseudo)
1522 return {AArch64::CPYFP, AArch64::CPYFM, AArch64::CPYFE};
1523 if (Opcode == AArch64::MOPSMemoryMovePseudo)
1524 return {AArch64::CPYP, AArch64::CPYM, AArch64::CPYE};
1525 if (Opcode == AArch64::MOPSMemorySetPseudo)
1526 return {AArch64::SETP, AArch64::SETM, AArch64::SETE};
1527 if (Opcode == AArch64::MOPSMemorySetTaggingPseudo)
1528 return {AArch64::SETGP, AArch64::SETGM, AArch64::MOPSSETGE};
1529 llvm_unreachable("Unhandled memory operation pseudo");
1530 }();
1531 const bool IsSet = Opcode == AArch64::MOPSMemorySetPseudo ||
1532 Opcode == AArch64::MOPSMemorySetTaggingPseudo;
1533
1534 for (auto Op : Ops) {
1535 int i = 0;
1536 auto MCIB = MCInstBuilder(Op);
1537 // Destination registers
1538 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1539 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1540 if (!IsSet)
1541 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1542 // Input registers
1543 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1544 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1545 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1546
1547 EmitToStreamer(S&: OutStreamer, Inst: MCIB);
1548 }
1549}
1550
1551void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
1552 const MachineInstr &MI) {
1553 unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
1554
1555 auto &Ctx = OutStreamer.getContext();
1556 MCSymbol *MILabel = Ctx.createTempSymbol();
1557 OutStreamer.emitLabel(Symbol: MILabel);
1558
1559 SM.recordStackMap(L: *MILabel, MI);
1560 assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1561
1562 // Scan ahead to trim the shadow.
1563 const MachineBasicBlock &MBB = *MI.getParent();
1564 MachineBasicBlock::const_iterator MII(MI);
1565 ++MII;
1566 while (NumNOPBytes > 0) {
1567 if (MII == MBB.end() || MII->isCall() ||
1568 MII->getOpcode() == AArch64::DBG_VALUE ||
1569 MII->getOpcode() == TargetOpcode::PATCHPOINT ||
1570 MII->getOpcode() == TargetOpcode::STACKMAP)
1571 break;
1572 ++MII;
1573 NumNOPBytes -= 4;
1574 }
1575
1576 // Emit nops.
1577 for (unsigned i = 0; i < NumNOPBytes; i += 4)
1578 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::HINT).addImm(Val: 0));
1579}
1580
1581// Lower a patchpoint of the form:
1582// [<def>], <id>, <numBytes>, <target>, <numArgs>
1583void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1584 const MachineInstr &MI) {
1585 auto &Ctx = OutStreamer.getContext();
1586 MCSymbol *MILabel = Ctx.createTempSymbol();
1587 OutStreamer.emitLabel(Symbol: MILabel);
1588 SM.recordPatchPoint(L: *MILabel, MI);
1589
1590 PatchPointOpers Opers(&MI);
1591
1592 int64_t CallTarget = Opers.getCallTarget().getImm();
1593 unsigned EncodedBytes = 0;
1594 if (CallTarget) {
1595 assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
1596 "High 16 bits of call target should be zero.");
1597 Register ScratchReg = MI.getOperand(i: Opers.getNextScratchIdx()).getReg();
1598 EncodedBytes = 16;
1599 // Materialize the jump address:
1600 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::MOVZXi)
1601 .addReg(Reg: ScratchReg)
1602 .addImm(Val: (CallTarget >> 32) & 0xFFFF)
1603 .addImm(Val: 32));
1604 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::MOVKXi)
1605 .addReg(Reg: ScratchReg)
1606 .addReg(Reg: ScratchReg)
1607 .addImm(Val: (CallTarget >> 16) & 0xFFFF)
1608 .addImm(Val: 16));
1609 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::MOVKXi)
1610 .addReg(Reg: ScratchReg)
1611 .addReg(Reg: ScratchReg)
1612 .addImm(Val: CallTarget & 0xFFFF)
1613 .addImm(Val: 0));
1614 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::BLR).addReg(Reg: ScratchReg));
1615 }
1616 // Emit padding.
1617 unsigned NumBytes = Opers.getNumPatchBytes();
1618 assert(NumBytes >= EncodedBytes &&
1619 "Patchpoint can't request size less than the length of a call.");
1620 assert((NumBytes - EncodedBytes) % 4 == 0 &&
1621 "Invalid number of NOP bytes requested!");
1622 for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
1623 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::HINT).addImm(Val: 0));
1624}
1625
1626void AArch64AsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1627 const MachineInstr &MI) {
1628 StatepointOpers SOpers(&MI);
1629 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
1630 assert(PatchBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1631 for (unsigned i = 0; i < PatchBytes; i += 4)
1632 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(AArch64::HINT).addImm(Val: 0));
1633 } else {
1634 // Lower call target and choose correct opcode
1635 const MachineOperand &CallTarget = SOpers.getCallTarget();
1636 MCOperand CallTargetMCOp;
1637 unsigned CallOpcode;
1638 switch (CallTarget.getType()) {
1639 case MachineOperand::MO_GlobalAddress:
1640 case MachineOperand::MO_ExternalSymbol:
1641 MCInstLowering.lowerOperand(MO: CallTarget, MCOp&: CallTargetMCOp);
1642 CallOpcode = AArch64::BL;
1643 break;
1644 case MachineOperand::MO_Immediate:
1645 CallTargetMCOp = MCOperand::createImm(Val: CallTarget.getImm());
1646 CallOpcode = AArch64::BL;
1647 break;
1648 case MachineOperand::MO_Register:
1649 CallTargetMCOp = MCOperand::createReg(Reg: CallTarget.getReg());
1650 CallOpcode = AArch64::BLR;
1651 break;
1652 default:
1653 llvm_unreachable("Unsupported operand type in statepoint call target");
1654 break;
1655 }
1656
1657 EmitToStreamer(S&: OutStreamer,
1658 Inst: MCInstBuilder(CallOpcode).addOperand(Op: CallTargetMCOp));
1659 }
1660
1661 auto &Ctx = OutStreamer.getContext();
1662 MCSymbol *MILabel = Ctx.createTempSymbol();
1663 OutStreamer.emitLabel(Symbol: MILabel);
1664 SM.recordStatepoint(L: *MILabel, MI);
1665}
1666
1667void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) {
1668 // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
1669 // <opcode>, <operands>
1670
1671 Register DefRegister = FaultingMI.getOperand(i: 0).getReg();
1672 FaultMaps::FaultKind FK =
1673 static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(i: 1).getImm());
1674 MCSymbol *HandlerLabel = FaultingMI.getOperand(i: 2).getMBB()->getSymbol();
1675 unsigned Opcode = FaultingMI.getOperand(i: 3).getImm();
1676 unsigned OperandsBeginIdx = 4;
1677
1678 auto &Ctx = OutStreamer->getContext();
1679 MCSymbol *FaultingLabel = Ctx.createTempSymbol();
1680 OutStreamer->emitLabel(Symbol: FaultingLabel);
1681
1682 assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
1683 FM.recordFaultingOp(FaultTy: FK, FaultingLabel, HandlerLabel);
1684
1685 MCInst MI;
1686 MI.setOpcode(Opcode);
1687
1688 if (DefRegister != (Register)0)
1689 MI.addOperand(Op: MCOperand::createReg(Reg: DefRegister));
1690
1691 for (const MachineOperand &MO :
1692 llvm::drop_begin(RangeOrContainer: FaultingMI.operands(), N: OperandsBeginIdx)) {
1693 MCOperand Dest;
1694 lowerOperand(MO, MCOp&: Dest);
1695 MI.addOperand(Op: Dest);
1696 }
1697
1698 OutStreamer->AddComment(T: "on-fault: " + HandlerLabel->getName());
1699 OutStreamer->emitInstruction(Inst: MI, STI: getSubtargetInfo());
1700}
1701
1702void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) {
1703 Register DestReg = MI.getOperand(i: 0).getReg();
1704 if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround() &&
1705 STI->isNeonAvailable()) {
1706 // Convert H/S register to corresponding D register
1707 if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31)
1708 DestReg = AArch64::D0 + (DestReg - AArch64::H0);
1709 else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31)
1710 DestReg = AArch64::D0 + (DestReg - AArch64::S0);
1711 else
1712 assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
1713
1714 MCInst MOVI;
1715 MOVI.setOpcode(AArch64::MOVID);
1716 MOVI.addOperand(Op: MCOperand::createReg(Reg: DestReg));
1717 MOVI.addOperand(Op: MCOperand::createImm(Val: 0));
1718 EmitToStreamer(S&: *OutStreamer, Inst: MOVI);
1719 } else {
1720 MCInst FMov;
1721 switch (MI.getOpcode()) {
1722 default: llvm_unreachable("Unexpected opcode");
1723 case AArch64::FMOVH0:
1724 FMov.setOpcode(STI->hasFullFP16() ? AArch64::FMOVWHr : AArch64::FMOVWSr);
1725 if (!STI->hasFullFP16())
1726 DestReg = (AArch64::S0 + (DestReg - AArch64::H0));
1727 FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg));
1728 FMov.addOperand(Op: MCOperand::createReg(Reg: AArch64::WZR));
1729 break;
1730 case AArch64::FMOVS0:
1731 FMov.setOpcode(AArch64::FMOVWSr);
1732 FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg));
1733 FMov.addOperand(Op: MCOperand::createReg(Reg: AArch64::WZR));
1734 break;
1735 case AArch64::FMOVD0:
1736 FMov.setOpcode(AArch64::FMOVXDr);
1737 FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg));
1738 FMov.addOperand(Op: MCOperand::createReg(Reg: AArch64::XZR));
1739 break;
1740 }
1741 EmitToStreamer(S&: *OutStreamer, Inst: FMov);
1742 }
1743}
1744
1745unsigned AArch64AsmPrinter::emitPtrauthDiscriminator(uint16_t Disc,
1746 unsigned AddrDisc,
1747 unsigned &InstsEmitted) {
1748 // So far we've used NoRegister in pseudos. Now we need real encodings.
1749 if (AddrDisc == AArch64::NoRegister)
1750 AddrDisc = AArch64::XZR;
1751
1752 // If there is no constant discriminator, there's no blend involved:
1753 // just use the address discriminator register as-is (XZR or not).
1754 if (!Disc)
1755 return AddrDisc;
1756
1757 // If there's only a constant discriminator, MOV it into x17.
1758 if (AddrDisc == AArch64::XZR) {
1759 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::MOVZXi)
1760 .addReg(Reg: AArch64::X17)
1761 .addImm(Val: Disc)
1762 .addImm(/*shift=*/Val: 0));
1763 ++InstsEmitted;
1764 return AArch64::X17;
1765 }
1766
1767 // If there are both, emit a blend into x17.
1768 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ORRXrs)
1769 .addReg(Reg: AArch64::X17)
1770 .addReg(Reg: AArch64::XZR)
1771 .addReg(Reg: AddrDisc)
1772 .addImm(Val: 0));
1773 ++InstsEmitted;
1774 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::MOVKXi)
1775 .addReg(Reg: AArch64::X17)
1776 .addReg(Reg: AArch64::X17)
1777 .addImm(Val: Disc)
1778 .addImm(/*shift=*/Val: 48));
1779 ++InstsEmitted;
1780 return AArch64::X17;
1781}
1782
1783void AArch64AsmPrinter::emitPtrauthAuthResign(const MachineInstr *MI) {
1784 unsigned InstsEmitted = 0;
1785 const bool IsAUTPAC = MI->getOpcode() == AArch64::AUTPAC;
1786
1787 // We can expand AUT/AUTPAC into 3 possible sequences:
1788 // - unchecked:
1789 // autia x16, x0
1790 // pacib x16, x1 ; if AUTPAC
1791 //
1792 // - checked and clearing:
1793 // mov x17, x0
1794 // movk x17, #disc, lsl #48
1795 // autia x16, x17
1796 // mov x17, x16
1797 // xpaci x17
1798 // cmp x16, x17
1799 // b.eq Lsuccess
1800 // mov x16, x17
1801 // b Lend
1802 // Lsuccess:
1803 // mov x17, x1
1804 // movk x17, #disc, lsl #48
1805 // pacib x16, x17
1806 // Lend:
1807 // Where we only emit the AUT if we started with an AUT.
1808 //
1809 // - checked and trapping:
1810 // mov x17, x0
1811 // movk x17, #disc, lsl #48
1812 // autia x16, x0
1813 // mov x17, x16
1814 // xpaci x17
1815 // cmp x16, x17
1816 // b.eq Lsuccess
1817 // brk #<0xc470 + aut key>
1818 // Lsuccess:
1819 // mov x17, x1
1820 // movk x17, #disc, lsl #48
1821 // pacib x16, x17 ; if AUTPAC
1822 // Where the b.eq skips over the trap if the PAC is valid.
1823 //
1824 // This sequence is expensive, but we need more information to be able to
1825 // do better.
1826 //
1827 // We can't TBZ the poison bit because EnhancedPAC2 XORs the PAC bits
1828 // on failure.
1829 // We can't TST the PAC bits because we don't always know how the address
1830 // space is setup for the target environment (and the bottom PAC bit is
1831 // based on that).
1832 // Either way, we also don't always know whether TBI is enabled or not for
1833 // the specific target environment.
1834
1835 // By default, auth/resign sequences check for auth failures.
1836 bool ShouldCheck = true;
1837 // In the checked sequence, we only trap if explicitly requested.
1838 bool ShouldTrap = MF->getFunction().hasFnAttribute(Kind: "ptrauth-auth-traps");
1839
1840 // On an FPAC CPU, you get traps whether you want them or not: there's
1841 // no point in emitting checks or traps.
1842 if (STI->hasFPAC())
1843 ShouldCheck = ShouldTrap = false;
1844
1845 // However, command-line flags can override this, for experimentation.
1846 switch (PtrauthAuthChecks) {
1847 case PtrauthCheckMode::Default:
1848 break;
1849 case PtrauthCheckMode::Unchecked:
1850 ShouldCheck = ShouldTrap = false;
1851 break;
1852 case PtrauthCheckMode::Poison:
1853 ShouldCheck = true;
1854 ShouldTrap = false;
1855 break;
1856 case PtrauthCheckMode::Trap:
1857 ShouldCheck = ShouldTrap = true;
1858 break;
1859 }
1860
1861 auto AUTKey = (AArch64PACKey::ID)MI->getOperand(i: 0).getImm();
1862 uint64_t AUTDisc = MI->getOperand(i: 1).getImm();
1863 unsigned AUTAddrDisc = MI->getOperand(i: 2).getReg();
1864
1865 unsigned XPACOpc = getXPACOpcodeForKey(K: AUTKey);
1866
1867 // Compute aut discriminator into x17
1868 assert(isUInt<16>(AUTDisc));
1869 unsigned AUTDiscReg =
1870 emitPtrauthDiscriminator(Disc: AUTDisc, AddrDisc: AUTAddrDisc, InstsEmitted);
1871 bool AUTZero = AUTDiscReg == AArch64::XZR;
1872 unsigned AUTOpc = getAUTOpcodeForKey(K: AUTKey, Zero: AUTZero);
1873
1874 // autiza x16 ; if AUTZero
1875 // autia x16, x17 ; if !AUTZero
1876 MCInst AUTInst;
1877 AUTInst.setOpcode(AUTOpc);
1878 AUTInst.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
1879 AUTInst.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
1880 if (!AUTZero)
1881 AUTInst.addOperand(Op: MCOperand::createReg(Reg: AUTDiscReg));
1882 EmitToStreamer(S&: *OutStreamer, Inst: AUTInst);
1883 ++InstsEmitted;
1884
1885 // Unchecked or checked-but-non-trapping AUT is just an "AUT": we're done.
1886 if (!IsAUTPAC && (!ShouldCheck || !ShouldTrap)) {
1887 assert(STI->getInstrInfo()->getInstSizeInBytes(*MI) >= InstsEmitted * 4);
1888 return;
1889 }
1890
1891 MCSymbol *EndSym = nullptr;
1892
1893 // Checked sequences do an additional strip-and-compare.
1894 if (ShouldCheck) {
1895 MCSymbol *SuccessSym = createTempSymbol(Name: "auth_success_");
1896
1897 // XPAC has tied src/dst: use x17 as a temporary copy.
1898 // mov x17, x16
1899 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ORRXrs)
1900 .addReg(Reg: AArch64::X17)
1901 .addReg(Reg: AArch64::XZR)
1902 .addReg(Reg: AArch64::X16)
1903 .addImm(Val: 0));
1904 ++InstsEmitted;
1905
1906 // xpaci x17
1907 EmitToStreamer(
1908 S&: *OutStreamer,
1909 Inst: MCInstBuilder(XPACOpc).addReg(Reg: AArch64::X17).addReg(Reg: AArch64::X17));
1910 ++InstsEmitted;
1911
1912 // cmp x16, x17
1913 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::SUBSXrs)
1914 .addReg(Reg: AArch64::XZR)
1915 .addReg(Reg: AArch64::X16)
1916 .addReg(Reg: AArch64::X17)
1917 .addImm(Val: 0));
1918 ++InstsEmitted;
1919
1920 // b.eq Lsuccess
1921 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::Bcc)
1922 .addImm(Val: AArch64CC::EQ)
1923 .addExpr(Val: MCSymbolRefExpr::create(
1924 Symbol: SuccessSym, Ctx&: OutContext)));
1925 ++InstsEmitted;
1926
1927 if (ShouldTrap) {
1928 // Trapping sequences do a 'brk'.
1929 // brk #<0xc470 + aut key>
1930 EmitToStreamer(S&: *OutStreamer,
1931 Inst: MCInstBuilder(AArch64::BRK).addImm(Val: 0xc470 | AUTKey));
1932 ++InstsEmitted;
1933 } else {
1934 // Non-trapping checked sequences return the stripped result in x16,
1935 // skipping over the PAC if there is one.
1936
1937 // FIXME: can we simply return the AUT result, already in x16? without..
1938 // ..traps this is usable as an oracle anyway, based on high bits
1939 // mov x17, x16
1940 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ORRXrs)
1941 .addReg(Reg: AArch64::X16)
1942 .addReg(Reg: AArch64::XZR)
1943 .addReg(Reg: AArch64::X17)
1944 .addImm(Val: 0));
1945 ++InstsEmitted;
1946
1947 if (IsAUTPAC) {
1948 EndSym = createTempSymbol(Name: "resign_end_");
1949
1950 // b Lend
1951 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::B)
1952 .addExpr(Val: MCSymbolRefExpr::create(
1953 Symbol: EndSym, Ctx&: OutContext)));
1954 ++InstsEmitted;
1955 }
1956 }
1957
1958 // If the auth check succeeds, we can continue.
1959 // Lsuccess:
1960 OutStreamer->emitLabel(Symbol: SuccessSym);
1961 }
1962
1963 // We already emitted unchecked and checked-but-non-trapping AUTs.
1964 // That left us with trapping AUTs, and AUTPACs.
1965 // Trapping AUTs don't need PAC: we're done.
1966 if (!IsAUTPAC) {
1967 assert(STI->getInstrInfo()->getInstSizeInBytes(*MI) >= InstsEmitted * 4);
1968 return;
1969 }
1970
1971 auto PACKey = (AArch64PACKey::ID)MI->getOperand(i: 3).getImm();
1972 uint64_t PACDisc = MI->getOperand(i: 4).getImm();
1973 unsigned PACAddrDisc = MI->getOperand(i: 5).getReg();
1974
1975 // Compute pac discriminator into x17
1976 assert(isUInt<16>(PACDisc));
1977 unsigned PACDiscReg =
1978 emitPtrauthDiscriminator(Disc: PACDisc, AddrDisc: PACAddrDisc, InstsEmitted);
1979 bool PACZero = PACDiscReg == AArch64::XZR;
1980 unsigned PACOpc = getPACOpcodeForKey(K: PACKey, Zero: PACZero);
1981
1982 // pacizb x16 ; if PACZero
1983 // pacib x16, x17 ; if !PACZero
1984 MCInst PACInst;
1985 PACInst.setOpcode(PACOpc);
1986 PACInst.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
1987 PACInst.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
1988 if (!PACZero)
1989 PACInst.addOperand(Op: MCOperand::createReg(Reg: PACDiscReg));
1990 EmitToStreamer(S&: *OutStreamer, Inst: PACInst);
1991 ++InstsEmitted;
1992
1993 assert(STI->getInstrInfo()->getInstSizeInBytes(*MI) >= InstsEmitted * 4);
1994 // Lend:
1995 if (EndSym)
1996 OutStreamer->emitLabel(Symbol: EndSym);
1997}
1998
1999void AArch64AsmPrinter::emitPtrauthBranch(const MachineInstr *MI) {
2000 unsigned InstsEmitted = 0;
2001 bool IsCall = MI->getOpcode() == AArch64::BLRA;
2002 unsigned BrTarget = MI->getOperand(i: 0).getReg();
2003
2004 auto Key = (AArch64PACKey::ID)MI->getOperand(i: 1).getImm();
2005 assert((Key == AArch64PACKey::IA || Key == AArch64PACKey::IB) &&
2006 "Invalid auth call key");
2007
2008 uint64_t Disc = MI->getOperand(i: 2).getImm();
2009 assert(isUInt<16>(Disc));
2010
2011 unsigned AddrDisc = MI->getOperand(i: 3).getReg();
2012
2013 // Compute discriminator into x17
2014 unsigned DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, InstsEmitted);
2015 bool IsZeroDisc = DiscReg == AArch64::XZR;
2016
2017 unsigned Opc;
2018 if (IsCall) {
2019 if (Key == AArch64PACKey::IA)
2020 Opc = IsZeroDisc ? AArch64::BLRAAZ : AArch64::BLRAA;
2021 else
2022 Opc = IsZeroDisc ? AArch64::BLRABZ : AArch64::BLRAB;
2023 } else {
2024 if (Key == AArch64PACKey::IA)
2025 Opc = IsZeroDisc ? AArch64::BRAAZ : AArch64::BRAA;
2026 else
2027 Opc = IsZeroDisc ? AArch64::BRABZ : AArch64::BRAB;
2028 }
2029
2030 MCInst BRInst;
2031 BRInst.setOpcode(Opc);
2032 BRInst.addOperand(Op: MCOperand::createReg(Reg: BrTarget));
2033 if (!IsZeroDisc)
2034 BRInst.addOperand(Op: MCOperand::createReg(Reg: DiscReg));
2035 EmitToStreamer(S&: *OutStreamer, Inst: BRInst);
2036 ++InstsEmitted;
2037
2038 assert(STI->getInstrInfo()->getInstSizeInBytes(*MI) >= InstsEmitted * 4);
2039}
2040
2041const MCExpr *
2042AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) {
2043 MCContext &Ctx = OutContext;
2044
2045 // Figure out the base symbol and the addend, if any.
2046 APInt Offset(64, 0);
2047 const Value *BaseGV = CPA.getPointer()->stripAndAccumulateConstantOffsets(
2048 DL: getDataLayout(), Offset, /*AllowNonInbounds=*/true);
2049
2050 auto *BaseGVB = dyn_cast<GlobalValue>(Val: BaseGV);
2051
2052 // If we can't understand the referenced ConstantExpr, there's nothing
2053 // else we can do: emit an error.
2054 if (!BaseGVB) {
2055 BaseGV->getContext().emitError(
2056 ErrorStr: "cannot resolve target base/addend of ptrauth constant");
2057 return nullptr;
2058 }
2059
2060 // If there is an addend, turn that into the appropriate MCExpr.
2061 const MCExpr *Sym = MCSymbolRefExpr::create(Symbol: getSymbol(GV: BaseGVB), Ctx);
2062 if (Offset.sgt(RHS: 0))
2063 Sym = MCBinaryExpr::createAdd(
2064 LHS: Sym, RHS: MCConstantExpr::create(Value: Offset.getSExtValue(), Ctx), Ctx);
2065 else if (Offset.slt(RHS: 0))
2066 Sym = MCBinaryExpr::createSub(
2067 LHS: Sym, RHS: MCConstantExpr::create(Value: (-Offset).getSExtValue(), Ctx), Ctx);
2068
2069 uint64_t KeyID = CPA.getKey()->getZExtValue();
2070 // We later rely on valid KeyID value in AArch64PACKeyIDToString call from
2071 // AArch64AuthMCExpr::printImpl, so fail fast.
2072 if (KeyID > AArch64PACKey::LAST)
2073 report_fatal_error(reason: "AArch64 PAC Key ID '" + Twine(KeyID) +
2074 "' out of range [0, " +
2075 Twine((unsigned)AArch64PACKey::LAST) + "]");
2076
2077 uint64_t Disc = CPA.getDiscriminator()->getZExtValue();
2078 if (!isUInt<16>(x: Disc))
2079 report_fatal_error(reason: "AArch64 PAC Discriminator '" + Twine(Disc) +
2080 "' out of range [0, 0xFFFF]");
2081
2082 // Finally build the complete @AUTH expr.
2083 return AArch64AuthMCExpr::create(Expr: Sym, Discriminator: Disc, Key: AArch64PACKey::ID(KeyID),
2084 HasAddressDiversity: CPA.hasAddressDiscriminator(), Ctx);
2085}
2086
2087void AArch64AsmPrinter::LowerLOADauthptrstatic(const MachineInstr &MI) {
2088 unsigned DstReg = MI.getOperand(i: 0).getReg();
2089 const MachineOperand &GAOp = MI.getOperand(i: 1);
2090 const uint64_t KeyC = MI.getOperand(i: 2).getImm();
2091 assert(KeyC <= AArch64PACKey::LAST &&
2092 "key is out of range [0, AArch64PACKey::LAST]");
2093 const auto Key = (AArch64PACKey::ID)KeyC;
2094 const uint64_t Disc = MI.getOperand(i: 3).getImm();
2095 assert(isUInt<16>(Disc) &&
2096 "constant discriminator is out of range [0, 0xffff]");
2097
2098 // Emit instruction sequence like the following:
2099 // ADRP x16, symbol$auth_ptr$key$disc
2100 // LDR x16, [x16, :lo12:symbol$auth_ptr$key$disc]
2101 //
2102 // Where the $auth_ptr$ symbol is the stub slot containing the signed pointer
2103 // to symbol.
2104 MCSymbol *AuthPtrStubSym;
2105 if (TM.getTargetTriple().isOSBinFormatELF()) {
2106 const auto &TLOF =
2107 static_cast<const AArch64_ELFTargetObjectFile &>(getObjFileLowering());
2108
2109 assert(GAOp.getOffset() == 0 &&
2110 "non-zero offset for $auth_ptr$ stub slots is not supported");
2111 const MCSymbol *GASym = TM.getSymbol(GV: GAOp.getGlobal());
2112 AuthPtrStubSym = TLOF.getAuthPtrSlotSymbol(TM, MMI, RawSym: GASym, Key, Discriminator: Disc);
2113 } else {
2114 assert(TM.getTargetTriple().isOSBinFormatMachO() &&
2115 "LOADauthptrstatic is implemented only for MachO/ELF");
2116
2117 const auto &TLOF = static_cast<const AArch64_MachoTargetObjectFile &>(
2118 getObjFileLowering());
2119
2120 assert(GAOp.getOffset() == 0 &&
2121 "non-zero offset for $auth_ptr$ stub slots is not supported");
2122 const MCSymbol *GASym = TM.getSymbol(GV: GAOp.getGlobal());
2123 AuthPtrStubSym = TLOF.getAuthPtrSlotSymbol(TM, MMI, RawSym: GASym, Key, Discriminator: Disc);
2124 }
2125
2126 MachineOperand StubMOHi =
2127 MachineOperand::CreateMCSymbol(Sym: AuthPtrStubSym, TargetFlags: AArch64II::MO_PAGE);
2128 MachineOperand StubMOLo = MachineOperand::CreateMCSymbol(
2129 Sym: AuthPtrStubSym, TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2130 MCOperand StubMCHi, StubMCLo;
2131
2132 MCInstLowering.lowerOperand(MO: StubMOHi, MCOp&: StubMCHi);
2133 MCInstLowering.lowerOperand(MO: StubMOLo, MCOp&: StubMCLo);
2134
2135 EmitToStreamer(
2136 S&: *OutStreamer,
2137 Inst: MCInstBuilder(AArch64::ADRP).addReg(Reg: DstReg).addOperand(Op: StubMCHi));
2138
2139 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::LDRXui)
2140 .addReg(Reg: DstReg)
2141 .addReg(Reg: DstReg)
2142 .addOperand(Op: StubMCLo));
2143}
2144
2145void AArch64AsmPrinter::LowerMOVaddrPAC(const MachineInstr &MI) {
2146 unsigned InstsEmitted = 0;
2147 auto EmitAndIncrement = [this, &InstsEmitted](const MCInst &Inst) {
2148 EmitToStreamer(S&: *OutStreamer, Inst);
2149 ++InstsEmitted;
2150 };
2151
2152 const bool IsGOTLoad = MI.getOpcode() == AArch64::LOADgotPAC;
2153 MachineOperand GAOp = MI.getOperand(i: 0);
2154 const uint64_t KeyC = MI.getOperand(i: 1).getImm();
2155 assert(KeyC <= AArch64PACKey::LAST &&
2156 "key is out of range [0, AArch64PACKey::LAST]");
2157 const auto Key = (AArch64PACKey::ID)KeyC;
2158 const unsigned AddrDisc = MI.getOperand(i: 2).getReg();
2159 const uint64_t Disc = MI.getOperand(i: 3).getImm();
2160 assert(isUInt<16>(Disc) &&
2161 "constant discriminator is out of range [0, 0xffff]");
2162
2163 const int64_t Offset = GAOp.getOffset();
2164 GAOp.setOffset(0);
2165
2166 // Emit:
2167 // target materialization:
2168 // - via GOT:
2169 // adrp x16, :got:target
2170 // ldr x16, [x16, :got_lo12:target]
2171 // add offset to x16 if offset != 0
2172 //
2173 // - direct:
2174 // adrp x16, target
2175 // add x16, x16, :lo12:target
2176 // add offset to x16 if offset != 0
2177 //
2178 // add offset to x16:
2179 // - abs(offset) fits 24 bits:
2180 // add/sub x16, x16, #<offset>[, #lsl 12] (up to 2 instructions)
2181 // - abs(offset) does not fit 24 bits:
2182 // - offset < 0:
2183 // movn+movk sequence filling x17 register with the offset (up to 4
2184 // instructions)
2185 // add x16, x16, x17
2186 // - offset > 0:
2187 // movz+movk sequence filling x17 register with the offset (up to 4
2188 // instructions)
2189 // add x16, x16, x17
2190 //
2191 // signing:
2192 // - 0 discriminator:
2193 // paciza x16
2194 // - Non-0 discriminator, no address discriminator:
2195 // mov x17, #Disc
2196 // pacia x16, x17
2197 // - address discriminator (with potentially folded immediate discriminator):
2198 // pacia x16, xAddrDisc
2199
2200 MachineOperand GAMOHi(GAOp), GAMOLo(GAOp);
2201 MCOperand GAMCHi, GAMCLo;
2202
2203 GAMOHi.setTargetFlags(AArch64II::MO_PAGE);
2204 GAMOLo.setTargetFlags(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2205 if (IsGOTLoad) {
2206 GAMOHi.addTargetFlag(F: AArch64II::MO_GOT);
2207 GAMOLo.addTargetFlag(F: AArch64II::MO_GOT);
2208 }
2209
2210 MCInstLowering.lowerOperand(MO: GAMOHi, MCOp&: GAMCHi);
2211 MCInstLowering.lowerOperand(MO: GAMOLo, MCOp&: GAMCLo);
2212
2213 EmitAndIncrement(
2214 MCInstBuilder(AArch64::ADRP).addReg(Reg: AArch64::X16).addOperand(Op: GAMCHi));
2215
2216 if (IsGOTLoad) {
2217 EmitAndIncrement(MCInstBuilder(AArch64::LDRXui)
2218 .addReg(Reg: AArch64::X16)
2219 .addReg(Reg: AArch64::X16)
2220 .addOperand(Op: GAMCLo));
2221 } else {
2222 EmitAndIncrement(MCInstBuilder(AArch64::ADDXri)
2223 .addReg(Reg: AArch64::X16)
2224 .addReg(Reg: AArch64::X16)
2225 .addOperand(Op: GAMCLo)
2226 .addImm(Val: 0));
2227 }
2228
2229 if (Offset != 0) {
2230 const uint64_t AbsOffset = (Offset > 0 ? Offset : -((uint64_t)Offset));
2231 const bool IsNeg = Offset < 0;
2232 if (isUInt<24>(x: AbsOffset)) {
2233 for (int BitPos = 0; BitPos != 24 && (AbsOffset >> BitPos);
2234 BitPos += 12) {
2235 EmitAndIncrement(
2236 MCInstBuilder(IsNeg ? AArch64::SUBXri : AArch64::ADDXri)
2237 .addReg(Reg: AArch64::X16)
2238 .addReg(Reg: AArch64::X16)
2239 .addImm(Val: (AbsOffset >> BitPos) & 0xfff)
2240 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: BitPos)));
2241 }
2242 } else {
2243 const uint64_t UOffset = Offset;
2244 EmitAndIncrement(MCInstBuilder(IsNeg ? AArch64::MOVNXi : AArch64::MOVZXi)
2245 .addReg(Reg: AArch64::X17)
2246 .addImm(Val: (IsNeg ? ~UOffset : UOffset) & 0xffff)
2247 .addImm(/*shift=*/Val: 0));
2248 auto NeedMovk = [IsNeg, UOffset](int BitPos) -> bool {
2249 assert(BitPos == 16 || BitPos == 32 || BitPos == 48);
2250 uint64_t Shifted = UOffset >> BitPos;
2251 if (!IsNeg)
2252 return Shifted != 0;
2253 for (int I = 0; I != 64 - BitPos; I += 16)
2254 if (((Shifted >> I) & 0xffff) != 0xffff)
2255 return true;
2256 return false;
2257 };
2258 for (int BitPos = 16; BitPos != 64 && NeedMovk(BitPos); BitPos += 16) {
2259 EmitAndIncrement(MCInstBuilder(AArch64::MOVKXi)
2260 .addReg(Reg: AArch64::X17)
2261 .addReg(Reg: AArch64::X17)
2262 .addImm(Val: (UOffset >> BitPos) & 0xffff)
2263 .addImm(/*shift=*/Val: BitPos));
2264 }
2265 EmitAndIncrement(MCInstBuilder(AArch64::ADDXrs)
2266 .addReg(Reg: AArch64::X16)
2267 .addReg(Reg: AArch64::X16)
2268 .addReg(Reg: AArch64::X17)
2269 .addImm(/*shift=*/Val: 0));
2270 }
2271 }
2272
2273 unsigned DiscReg = AddrDisc;
2274 if (Disc != 0) {
2275 if (AddrDisc != AArch64::XZR) {
2276 EmitAndIncrement(MCInstBuilder(AArch64::ORRXrs)
2277 .addReg(Reg: AArch64::X17)
2278 .addReg(Reg: AArch64::XZR)
2279 .addReg(Reg: AddrDisc)
2280 .addImm(Val: 0));
2281 EmitAndIncrement(MCInstBuilder(AArch64::MOVKXi)
2282 .addReg(Reg: AArch64::X17)
2283 .addReg(Reg: AArch64::X17)
2284 .addImm(Val: Disc)
2285 .addImm(/*shift=*/Val: 48));
2286 } else {
2287 EmitAndIncrement(MCInstBuilder(AArch64::MOVZXi)
2288 .addReg(Reg: AArch64::X17)
2289 .addImm(Val: Disc)
2290 .addImm(/*shift=*/Val: 0));
2291 }
2292 DiscReg = AArch64::X17;
2293 }
2294
2295 auto MIB = MCInstBuilder(getPACOpcodeForKey(K: Key, Zero: DiscReg == AArch64::XZR))
2296 .addReg(Reg: AArch64::X16)
2297 .addReg(Reg: AArch64::X16);
2298 if (DiscReg != AArch64::XZR)
2299 MIB.addReg(Reg: DiscReg);
2300 EmitAndIncrement(MIB);
2301
2302 assert(STI->getInstrInfo()->getInstSizeInBytes(MI) >= InstsEmitted * 4);
2303}
2304
2305const MCExpr *
2306AArch64AsmPrinter::lowerBlockAddressConstant(const BlockAddress &BA) {
2307 const MCExpr *BAE = AsmPrinter::lowerBlockAddressConstant(BA);
2308 const Function &Fn = *BA.getFunction();
2309
2310 if (std::optional<uint16_t> BADisc =
2311 STI->getPtrAuthBlockAddressDiscriminatorIfEnabled(ParentFn: Fn))
2312 return AArch64AuthMCExpr::create(Expr: BAE, Discriminator: *BADisc, Key: AArch64PACKey::IA,
2313 /*HasAddressDiversity=*/false, Ctx&: OutContext);
2314
2315 return BAE;
2316}
2317
2318// Simple pseudo-instructions have their lowering (with expansion to real
2319// instructions) auto-generated.
2320#include "AArch64GenMCPseudoLowering.inc"
2321
2322void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
2323 AArch64_MC::verifyInstructionPredicates(Opcode: MI->getOpcode(), Features: STI->getFeatureBits());
2324
2325 // Do any auto-generated pseudo lowerings.
2326 if (emitPseudoExpansionLowering(OutStreamer&: *OutStreamer, MI))
2327 return;
2328
2329 if (MI->getOpcode() == AArch64::ADRP) {
2330 for (auto &Opd : MI->operands()) {
2331 if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) ==
2332 "swift_async_extendedFramePointerFlags") {
2333 ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true;
2334 }
2335 }
2336 }
2337
2338 if (AArch64FI->getLOHRelated().count(Ptr: MI)) {
2339 // Generate a label for LOH related instruction
2340 MCSymbol *LOHLabel = createTempSymbol(Name: "loh");
2341 // Associate the instruction with the label
2342 LOHInstToLabel[MI] = LOHLabel;
2343 OutStreamer->emitLabel(Symbol: LOHLabel);
2344 }
2345
2346 AArch64TargetStreamer *TS =
2347 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
2348 // Do any manual lowerings.
2349 switch (MI->getOpcode()) {
2350 default:
2351 break;
2352 case AArch64::HINT: {
2353 // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
2354 // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
2355 // non-empty. If MI is the initial BTI, place the
2356 // __patchable_function_entries label after BTI.
2357 if (CurrentPatchableFunctionEntrySym &&
2358 CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
2359 MI == &MF->front().front()) {
2360 int64_t Imm = MI->getOperand(i: 0).getImm();
2361 if ((Imm & 32) && (Imm & 6)) {
2362 MCInst Inst;
2363 MCInstLowering.Lower(MI, OutMI&: Inst);
2364 EmitToStreamer(S&: *OutStreamer, Inst);
2365 CurrentPatchableFunctionEntrySym = createTempSymbol(Name: "patch");
2366 OutStreamer->emitLabel(Symbol: CurrentPatchableFunctionEntrySym);
2367 return;
2368 }
2369 }
2370 break;
2371 }
2372 case AArch64::MOVMCSym: {
2373 Register DestReg = MI->getOperand(i: 0).getReg();
2374 const MachineOperand &MO_Sym = MI->getOperand(i: 1);
2375 MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
2376 MCOperand Hi_MCSym, Lo_MCSym;
2377
2378 Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
2379 Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
2380
2381 MCInstLowering.lowerOperand(MO: Hi_MOSym, MCOp&: Hi_MCSym);
2382 MCInstLowering.lowerOperand(MO: Lo_MOSym, MCOp&: Lo_MCSym);
2383
2384 MCInst MovZ;
2385 MovZ.setOpcode(AArch64::MOVZXi);
2386 MovZ.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2387 MovZ.addOperand(Op: Hi_MCSym);
2388 MovZ.addOperand(Op: MCOperand::createImm(Val: 16));
2389 EmitToStreamer(S&: *OutStreamer, Inst: MovZ);
2390
2391 MCInst MovK;
2392 MovK.setOpcode(AArch64::MOVKXi);
2393 MovK.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2394 MovK.addOperand(Op: MCOperand::createReg(Reg: DestReg));
2395 MovK.addOperand(Op: Lo_MCSym);
2396 MovK.addOperand(Op: MCOperand::createImm(Val: 0));
2397 EmitToStreamer(S&: *OutStreamer, Inst: MovK);
2398 return;
2399 }
2400 case AArch64::MOVIv2d_ns:
2401 // It is generally beneficial to rewrite "fmov s0, wzr" to "movi d0, #0".
2402 // as movi is more efficient across all cores. Newer cores can eliminate
2403 // fmovs early and there is no difference with movi, but this not true for
2404 // all implementations.
2405 //
2406 // The floating-point version doesn't quite work in rare cases on older
2407 // CPUs, so on those targets we lower this instruction to movi.16b instead.
2408 if (STI->hasZeroCycleZeroingFPWorkaround() &&
2409 MI->getOperand(i: 1).getImm() == 0) {
2410 MCInst TmpInst;
2411 TmpInst.setOpcode(AArch64::MOVIv16b_ns);
2412 TmpInst.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg()));
2413 TmpInst.addOperand(Op: MCOperand::createImm(Val: MI->getOperand(i: 1).getImm()));
2414 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
2415 return;
2416 }
2417 break;
2418
2419 case AArch64::DBG_VALUE:
2420 case AArch64::DBG_VALUE_LIST:
2421 if (isVerbose() && OutStreamer->hasRawTextSupport()) {
2422 SmallString<128> TmpStr;
2423 raw_svector_ostream OS(TmpStr);
2424 PrintDebugValueComment(MI, OS);
2425 OutStreamer->emitRawText(String: StringRef(OS.str()));
2426 }
2427 return;
2428
2429 case AArch64::EMITBKEY: {
2430 ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
2431 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
2432 ExceptionHandlingType != ExceptionHandling::ARM)
2433 return;
2434
2435 if (getFunctionCFISectionType(MF: *MF) == CFISection::None)
2436 return;
2437
2438 OutStreamer->emitCFIBKeyFrame();
2439 return;
2440 }
2441
2442 case AArch64::EMITMTETAGGED: {
2443 ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
2444 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
2445 ExceptionHandlingType != ExceptionHandling::ARM)
2446 return;
2447
2448 if (getFunctionCFISectionType(MF: *MF) != CFISection::None)
2449 OutStreamer->emitCFIMTETaggedFrame();
2450 return;
2451 }
2452
2453 case AArch64::AUT:
2454 case AArch64::AUTPAC:
2455 emitPtrauthAuthResign(MI);
2456 return;
2457
2458 case AArch64::LOADauthptrstatic:
2459 LowerLOADauthptrstatic(MI: *MI);
2460 return;
2461
2462 case AArch64::LOADgotPAC:
2463 case AArch64::MOVaddrPAC:
2464 LowerMOVaddrPAC(MI: *MI);
2465 return;
2466
2467 case AArch64::BRA:
2468 case AArch64::BLRA:
2469 emitPtrauthBranch(MI);
2470 return;
2471
2472 // Tail calls use pseudo instructions so they have the proper code-gen
2473 // attributes (isCall, isReturn, etc.). We lower them to the real
2474 // instruction here.
2475 case AArch64::AUTH_TCRETURN:
2476 case AArch64::AUTH_TCRETURN_BTI: {
2477 const uint64_t Key = MI->getOperand(i: 2).getImm();
2478 assert((Key == AArch64PACKey::IA || Key == AArch64PACKey::IB) &&
2479 "Invalid auth key for tail-call return");
2480
2481 const uint64_t Disc = MI->getOperand(i: 3).getImm();
2482 assert(isUInt<16>(Disc) && "Integer discriminator is too wide");
2483
2484 Register AddrDisc = MI->getOperand(i: 4).getReg();
2485
2486 Register ScratchReg = MI->getOperand(i: 0).getReg() == AArch64::X16
2487 ? AArch64::X17
2488 : AArch64::X16;
2489
2490 unsigned DiscReg = AddrDisc;
2491 if (Disc) {
2492 if (AddrDisc != AArch64::NoRegister) {
2493 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::ORRXrs)
2494 .addReg(Reg: ScratchReg)
2495 .addReg(Reg: AArch64::XZR)
2496 .addReg(Reg: AddrDisc)
2497 .addImm(Val: 0));
2498 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::MOVKXi)
2499 .addReg(Reg: ScratchReg)
2500 .addReg(Reg: ScratchReg)
2501 .addImm(Val: Disc)
2502 .addImm(/*shift=*/Val: 48));
2503 } else {
2504 EmitToStreamer(S&: *OutStreamer, Inst: MCInstBuilder(AArch64::MOVZXi)
2505 .addReg(Reg: ScratchReg)
2506 .addImm(Val: Disc)
2507 .addImm(/*shift=*/Val: 0));
2508 }
2509 DiscReg = ScratchReg;
2510 }
2511
2512 const bool IsZero = DiscReg == AArch64::NoRegister;
2513 const unsigned Opcodes[2][2] = {{AArch64::BRAA, AArch64::BRAAZ},
2514 {AArch64::BRAB, AArch64::BRABZ}};
2515
2516 MCInst TmpInst;
2517 TmpInst.setOpcode(Opcodes[Key][IsZero]);
2518 TmpInst.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg()));
2519 if (!IsZero)
2520 TmpInst.addOperand(Op: MCOperand::createReg(Reg: DiscReg));
2521 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
2522 return;
2523 }
2524
2525 case AArch64::TCRETURNri:
2526 case AArch64::TCRETURNrix16x17:
2527 case AArch64::TCRETURNrix17:
2528 case AArch64::TCRETURNrinotx16:
2529 case AArch64::TCRETURNriALL: {
2530 MCInst TmpInst;
2531 TmpInst.setOpcode(AArch64::BR);
2532 TmpInst.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg()));
2533 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
2534 return;
2535 }
2536 case AArch64::TCRETURNdi: {
2537 MCOperand Dest;
2538 MCInstLowering.lowerOperand(MO: MI->getOperand(i: 0), MCOp&: Dest);
2539 MCInst TmpInst;
2540 TmpInst.setOpcode(AArch64::B);
2541 TmpInst.addOperand(Op: Dest);
2542 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
2543 return;
2544 }
2545 case AArch64::SpeculationBarrierISBDSBEndBB: {
2546 // Print DSB SYS + ISB
2547 MCInst TmpInstDSB;
2548 TmpInstDSB.setOpcode(AArch64::DSB);
2549 TmpInstDSB.addOperand(Op: MCOperand::createImm(Val: 0xf));
2550 EmitToStreamer(S&: *OutStreamer, Inst: TmpInstDSB);
2551 MCInst TmpInstISB;
2552 TmpInstISB.setOpcode(AArch64::ISB);
2553 TmpInstISB.addOperand(Op: MCOperand::createImm(Val: 0xf));
2554 EmitToStreamer(S&: *OutStreamer, Inst: TmpInstISB);
2555 return;
2556 }
2557 case AArch64::SpeculationBarrierSBEndBB: {
2558 // Print SB
2559 MCInst TmpInstSB;
2560 TmpInstSB.setOpcode(AArch64::SB);
2561 EmitToStreamer(S&: *OutStreamer, Inst: TmpInstSB);
2562 return;
2563 }
2564 case AArch64::TLSDESC_CALLSEQ: {
2565 /// lower this to:
2566 /// adrp x0, :tlsdesc:var
2567 /// ldr x1, [x0, #:tlsdesc_lo12:var]
2568 /// add x0, x0, #:tlsdesc_lo12:var
2569 /// .tlsdesccall var
2570 /// blr x1
2571 /// (TPIDR_EL0 offset now in x0)
2572 const MachineOperand &MO_Sym = MI->getOperand(i: 0);
2573 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
2574 MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
2575 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
2576 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
2577 MCInstLowering.lowerOperand(MO: MO_Sym, MCOp&: Sym);
2578 MCInstLowering.lowerOperand(MO: MO_TLSDESC_LO12, MCOp&: SymTLSDescLo12);
2579 MCInstLowering.lowerOperand(MO: MO_TLSDESC, MCOp&: SymTLSDesc);
2580
2581 MCInst Adrp;
2582 Adrp.setOpcode(AArch64::ADRP);
2583 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
2584 Adrp.addOperand(Op: SymTLSDesc);
2585 EmitToStreamer(S&: *OutStreamer, Inst: Adrp);
2586
2587 MCInst Ldr;
2588 if (STI->isTargetILP32()) {
2589 Ldr.setOpcode(AArch64::LDRWui);
2590 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::W1));
2591 } else {
2592 Ldr.setOpcode(AArch64::LDRXui);
2593 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X1));
2594 }
2595 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
2596 Ldr.addOperand(Op: SymTLSDescLo12);
2597 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
2598 EmitToStreamer(S&: *OutStreamer, Inst: Ldr);
2599
2600 MCInst Add;
2601 if (STI->isTargetILP32()) {
2602 Add.setOpcode(AArch64::ADDWri);
2603 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::W0));
2604 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::W0));
2605 } else {
2606 Add.setOpcode(AArch64::ADDXri);
2607 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
2608 Add.addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
2609 }
2610 Add.addOperand(Op: SymTLSDescLo12);
2611 Add.addOperand(Op: MCOperand::createImm(Val: AArch64_AM::getShiftValue(Imm: 0)));
2612 EmitToStreamer(S&: *OutStreamer, Inst: Add);
2613
2614 // Emit a relocation-annotation. This expands to no code, but requests
2615 // the following instruction gets an R_AARCH64_TLSDESC_CALL.
2616 MCInst TLSDescCall;
2617 TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
2618 TLSDescCall.addOperand(Op: Sym);
2619 EmitToStreamer(S&: *OutStreamer, Inst: TLSDescCall);
2620
2621 MCInst Blr;
2622 Blr.setOpcode(AArch64::BLR);
2623 Blr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X1));
2624 EmitToStreamer(S&: *OutStreamer, Inst: Blr);
2625
2626 return;
2627 }
2628
2629 case AArch64::JumpTableDest32:
2630 case AArch64::JumpTableDest16:
2631 case AArch64::JumpTableDest8:
2632 LowerJumpTableDest(OutStreamer&: *OutStreamer, MI: *MI);
2633 return;
2634
2635 case AArch64::BR_JumpTable:
2636 LowerHardenedBRJumpTable(MI: *MI);
2637 return;
2638
2639 case AArch64::FMOVH0:
2640 case AArch64::FMOVS0:
2641 case AArch64::FMOVD0:
2642 emitFMov0(MI: *MI);
2643 return;
2644
2645 case AArch64::MOPSMemoryCopyPseudo:
2646 case AArch64::MOPSMemoryMovePseudo:
2647 case AArch64::MOPSMemorySetPseudo:
2648 case AArch64::MOPSMemorySetTaggingPseudo:
2649 LowerMOPS(OutStreamer&: *OutStreamer, MI: *MI);
2650 return;
2651
2652 case TargetOpcode::STACKMAP:
2653 return LowerSTACKMAP(OutStreamer&: *OutStreamer, SM, MI: *MI);
2654
2655 case TargetOpcode::PATCHPOINT:
2656 return LowerPATCHPOINT(OutStreamer&: *OutStreamer, SM, MI: *MI);
2657
2658 case TargetOpcode::STATEPOINT:
2659 return LowerSTATEPOINT(OutStreamer&: *OutStreamer, SM, MI: *MI);
2660
2661 case TargetOpcode::FAULTING_OP:
2662 return LowerFAULTING_OP(FaultingMI: *MI);
2663
2664 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
2665 LowerPATCHABLE_FUNCTION_ENTER(MI: *MI);
2666 return;
2667
2668 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
2669 LowerPATCHABLE_FUNCTION_EXIT(MI: *MI);
2670 return;
2671
2672 case TargetOpcode::PATCHABLE_TAIL_CALL:
2673 LowerPATCHABLE_TAIL_CALL(MI: *MI);
2674 return;
2675 case TargetOpcode::PATCHABLE_EVENT_CALL:
2676 return LowerPATCHABLE_EVENT_CALL(MI: *MI, Typed: false);
2677 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
2678 return LowerPATCHABLE_EVENT_CALL(MI: *MI, Typed: true);
2679
2680 case AArch64::KCFI_CHECK:
2681 LowerKCFI_CHECK(MI: *MI);
2682 return;
2683
2684 case AArch64::HWASAN_CHECK_MEMACCESS:
2685 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
2686 case AArch64::HWASAN_CHECK_MEMACCESS_FIXEDSHADOW:
2687 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW:
2688 LowerHWASAN_CHECK_MEMACCESS(MI: *MI);
2689 return;
2690
2691 case AArch64::SEH_StackAlloc:
2692 TS->emitARM64WinCFIAllocStack(Size: MI->getOperand(i: 0).getImm());
2693 return;
2694
2695 case AArch64::SEH_SaveFPLR:
2696 TS->emitARM64WinCFISaveFPLR(Offset: MI->getOperand(i: 0).getImm());
2697 return;
2698
2699 case AArch64::SEH_SaveFPLR_X:
2700 assert(MI->getOperand(0).getImm() < 0 &&
2701 "Pre increment SEH opcode must have a negative offset");
2702 TS->emitARM64WinCFISaveFPLRX(Offset: -MI->getOperand(i: 0).getImm());
2703 return;
2704
2705 case AArch64::SEH_SaveReg:
2706 TS->emitARM64WinCFISaveReg(Reg: MI->getOperand(i: 0).getImm(),
2707 Offset: MI->getOperand(i: 1).getImm());
2708 return;
2709
2710 case AArch64::SEH_SaveReg_X:
2711 assert(MI->getOperand(1).getImm() < 0 &&
2712 "Pre increment SEH opcode must have a negative offset");
2713 TS->emitARM64WinCFISaveRegX(Reg: MI->getOperand(i: 0).getImm(),
2714 Offset: -MI->getOperand(i: 1).getImm());
2715 return;
2716
2717 case AArch64::SEH_SaveRegP:
2718 if (MI->getOperand(i: 1).getImm() == 30 && MI->getOperand(i: 0).getImm() >= 19 &&
2719 MI->getOperand(i: 0).getImm() <= 28) {
2720 assert((MI->getOperand(0).getImm() - 19) % 2 == 0 &&
2721 "Register paired with LR must be odd");
2722 TS->emitARM64WinCFISaveLRPair(Reg: MI->getOperand(i: 0).getImm(),
2723 Offset: MI->getOperand(i: 2).getImm());
2724 return;
2725 }
2726 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
2727 "Non-consecutive registers not allowed for save_regp");
2728 TS->emitARM64WinCFISaveRegP(Reg: MI->getOperand(i: 0).getImm(),
2729 Offset: MI->getOperand(i: 2).getImm());
2730 return;
2731
2732 case AArch64::SEH_SaveRegP_X:
2733 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
2734 "Non-consecutive registers not allowed for save_regp_x");
2735 assert(MI->getOperand(2).getImm() < 0 &&
2736 "Pre increment SEH opcode must have a negative offset");
2737 TS->emitARM64WinCFISaveRegPX(Reg: MI->getOperand(i: 0).getImm(),
2738 Offset: -MI->getOperand(i: 2).getImm());
2739 return;
2740
2741 case AArch64::SEH_SaveFReg:
2742 TS->emitARM64WinCFISaveFReg(Reg: MI->getOperand(i: 0).getImm(),
2743 Offset: MI->getOperand(i: 1).getImm());
2744 return;
2745
2746 case AArch64::SEH_SaveFReg_X:
2747 assert(MI->getOperand(1).getImm() < 0 &&
2748 "Pre increment SEH opcode must have a negative offset");
2749 TS->emitARM64WinCFISaveFRegX(Reg: MI->getOperand(i: 0).getImm(),
2750 Offset: -MI->getOperand(i: 1).getImm());
2751 return;
2752
2753 case AArch64::SEH_SaveFRegP:
2754 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
2755 "Non-consecutive registers not allowed for save_regp");
2756 TS->emitARM64WinCFISaveFRegP(Reg: MI->getOperand(i: 0).getImm(),
2757 Offset: MI->getOperand(i: 2).getImm());
2758 return;
2759
2760 case AArch64::SEH_SaveFRegP_X:
2761 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
2762 "Non-consecutive registers not allowed for save_regp_x");
2763 assert(MI->getOperand(2).getImm() < 0 &&
2764 "Pre increment SEH opcode must have a negative offset");
2765 TS->emitARM64WinCFISaveFRegPX(Reg: MI->getOperand(i: 0).getImm(),
2766 Offset: -MI->getOperand(i: 2).getImm());
2767 return;
2768
2769 case AArch64::SEH_SetFP:
2770 TS->emitARM64WinCFISetFP();
2771 return;
2772
2773 case AArch64::SEH_AddFP:
2774 TS->emitARM64WinCFIAddFP(Size: MI->getOperand(i: 0).getImm());
2775 return;
2776
2777 case AArch64::SEH_Nop:
2778 TS->emitARM64WinCFINop();
2779 return;
2780
2781 case AArch64::SEH_PrologEnd:
2782 TS->emitARM64WinCFIPrologEnd();
2783 return;
2784
2785 case AArch64::SEH_EpilogStart:
2786 TS->emitARM64WinCFIEpilogStart();
2787 return;
2788
2789 case AArch64::SEH_EpilogEnd:
2790 TS->emitARM64WinCFIEpilogEnd();
2791 return;
2792
2793 case AArch64::SEH_PACSignLR:
2794 TS->emitARM64WinCFIPACSignLR();
2795 return;
2796
2797 case AArch64::SEH_SaveAnyRegQP:
2798 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
2799 "Non-consecutive registers not allowed for save_any_reg");
2800 assert(MI->getOperand(2).getImm() >= 0 &&
2801 "SaveAnyRegQP SEH opcode offset must be non-negative");
2802 assert(MI->getOperand(2).getImm() <= 1008 &&
2803 "SaveAnyRegQP SEH opcode offset must fit into 6 bits");
2804 TS->emitARM64WinCFISaveAnyRegQP(Reg: MI->getOperand(i: 0).getImm(),
2805 Offset: MI->getOperand(i: 2).getImm());
2806 return;
2807
2808 case AArch64::SEH_SaveAnyRegQPX:
2809 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
2810 "Non-consecutive registers not allowed for save_any_reg");
2811 assert(MI->getOperand(2).getImm() < 0 &&
2812 "SaveAnyRegQPX SEH opcode offset must be negative");
2813 assert(MI->getOperand(2).getImm() >= -1008 &&
2814 "SaveAnyRegQPX SEH opcode offset must fit into 6 bits");
2815 TS->emitARM64WinCFISaveAnyRegQPX(Reg: MI->getOperand(i: 0).getImm(),
2816 Offset: -MI->getOperand(i: 2).getImm());
2817 return;
2818 }
2819
2820 // Finally, do the automated lowerings for everything else.
2821 MCInst TmpInst;
2822 MCInstLowering.Lower(MI, OutMI&: TmpInst);
2823 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
2824}
2825
2826void AArch64AsmPrinter::emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI,
2827 MCSymbol *LazyPointer) {
2828 // _ifunc:
2829 // adrp x16, lazy_pointer@GOTPAGE
2830 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF]
2831 // ldr x16, [x16]
2832 // br x16
2833
2834 {
2835 MCInst Adrp;
2836 Adrp.setOpcode(AArch64::ADRP);
2837 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
2838 MCOperand SymPage;
2839 MCInstLowering.lowerOperand(
2840 MO: MachineOperand::CreateMCSymbol(Sym: LazyPointer,
2841 TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGE),
2842 MCOp&: SymPage);
2843 Adrp.addOperand(Op: SymPage);
2844 OutStreamer->emitInstruction(Inst: Adrp, STI: *STI);
2845 }
2846
2847 {
2848 MCInst Ldr;
2849 Ldr.setOpcode(AArch64::LDRXui);
2850 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
2851 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
2852 MCOperand SymPageOff;
2853 MCInstLowering.lowerOperand(
2854 MO: MachineOperand::CreateMCSymbol(Sym: LazyPointer, TargetFlags: AArch64II::MO_GOT |
2855 AArch64II::MO_PAGEOFF),
2856 MCOp&: SymPageOff);
2857 Ldr.addOperand(Op: SymPageOff);
2858 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
2859 OutStreamer->emitInstruction(Inst: Ldr, STI: *STI);
2860 }
2861
2862 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::LDRXui)
2863 .addReg(Reg: AArch64::X16)
2864 .addReg(Reg: AArch64::X16)
2865 .addImm(Val: 0),
2866 STI: *STI);
2867
2868 OutStreamer->emitInstruction(Inst: MCInstBuilder(TM.getTargetTriple().isArm64e()
2869 ? AArch64::BRAAZ
2870 : AArch64::BR)
2871 .addReg(Reg: AArch64::X16),
2872 STI: *STI);
2873}
2874
2875void AArch64AsmPrinter::emitMachOIFuncStubHelperBody(Module &M,
2876 const GlobalIFunc &GI,
2877 MCSymbol *LazyPointer) {
2878 // These stub helpers are only ever called once, so here we're optimizing for
2879 // minimum size by using the pre-indexed store variants, which saves a few
2880 // bytes of instructions to bump & restore sp.
2881
2882 // _ifunc.stub_helper:
2883 // stp fp, lr, [sp, #-16]!
2884 // mov fp, sp
2885 // stp x1, x0, [sp, #-16]!
2886 // stp x3, x2, [sp, #-16]!
2887 // stp x5, x4, [sp, #-16]!
2888 // stp x7, x6, [sp, #-16]!
2889 // stp d1, d0, [sp, #-16]!
2890 // stp d3, d2, [sp, #-16]!
2891 // stp d5, d4, [sp, #-16]!
2892 // stp d7, d6, [sp, #-16]!
2893 // bl _resolver
2894 // adrp x16, lazy_pointer@GOTPAGE
2895 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF]
2896 // str x0, [x16]
2897 // mov x16, x0
2898 // ldp d7, d6, [sp], #16
2899 // ldp d5, d4, [sp], #16
2900 // ldp d3, d2, [sp], #16
2901 // ldp d1, d0, [sp], #16
2902 // ldp x7, x6, [sp], #16
2903 // ldp x5, x4, [sp], #16
2904 // ldp x3, x2, [sp], #16
2905 // ldp x1, x0, [sp], #16
2906 // ldp fp, lr, [sp], #16
2907 // br x16
2908
2909 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::STPXpre)
2910 .addReg(Reg: AArch64::SP)
2911 .addReg(Reg: AArch64::FP)
2912 .addReg(Reg: AArch64::LR)
2913 .addReg(Reg: AArch64::SP)
2914 .addImm(Val: -2),
2915 STI: *STI);
2916
2917 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::ADDXri)
2918 .addReg(Reg: AArch64::FP)
2919 .addReg(Reg: AArch64::SP)
2920 .addImm(Val: 0)
2921 .addImm(Val: 0),
2922 STI: *STI);
2923
2924 for (int I = 0; I != 4; ++I)
2925 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::STPXpre)
2926 .addReg(Reg: AArch64::SP)
2927 .addReg(Reg: AArch64::X1 + 2 * I)
2928 .addReg(Reg: AArch64::X0 + 2 * I)
2929 .addReg(Reg: AArch64::SP)
2930 .addImm(Val: -2),
2931 STI: *STI);
2932
2933 for (int I = 0; I != 4; ++I)
2934 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::STPDpre)
2935 .addReg(Reg: AArch64::SP)
2936 .addReg(Reg: AArch64::D1 + 2 * I)
2937 .addReg(Reg: AArch64::D0 + 2 * I)
2938 .addReg(Reg: AArch64::SP)
2939 .addImm(Val: -2),
2940 STI: *STI);
2941
2942 OutStreamer->emitInstruction(
2943 Inst: MCInstBuilder(AArch64::BL)
2944 .addOperand(Op: MCOperand::createExpr(Val: lowerConstant(CV: GI.getResolver()))),
2945 STI: *STI);
2946
2947 {
2948 MCInst Adrp;
2949 Adrp.setOpcode(AArch64::ADRP);
2950 Adrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
2951 MCOperand SymPage;
2952 MCInstLowering.lowerOperand(
2953 MO: MachineOperand::CreateES(SymName: LazyPointer->getName().data() + 1,
2954 TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGE),
2955 MCOp&: SymPage);
2956 Adrp.addOperand(Op: SymPage);
2957 OutStreamer->emitInstruction(Inst: Adrp, STI: *STI);
2958 }
2959
2960 {
2961 MCInst Ldr;
2962 Ldr.setOpcode(AArch64::LDRXui);
2963 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
2964 Ldr.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
2965 MCOperand SymPageOff;
2966 MCInstLowering.lowerOperand(
2967 MO: MachineOperand::CreateES(SymName: LazyPointer->getName().data() + 1,
2968 TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGEOFF),
2969 MCOp&: SymPageOff);
2970 Ldr.addOperand(Op: SymPageOff);
2971 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
2972 OutStreamer->emitInstruction(Inst: Ldr, STI: *STI);
2973 }
2974
2975 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::STRXui)
2976 .addReg(Reg: AArch64::X0)
2977 .addReg(Reg: AArch64::X16)
2978 .addImm(Val: 0),
2979 STI: *STI);
2980
2981 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::ADDXri)
2982 .addReg(Reg: AArch64::X16)
2983 .addReg(Reg: AArch64::X0)
2984 .addImm(Val: 0)
2985 .addImm(Val: 0),
2986 STI: *STI);
2987
2988 for (int I = 3; I != -1; --I)
2989 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::LDPDpost)
2990 .addReg(Reg: AArch64::SP)
2991 .addReg(Reg: AArch64::D1 + 2 * I)
2992 .addReg(Reg: AArch64::D0 + 2 * I)
2993 .addReg(Reg: AArch64::SP)
2994 .addImm(Val: 2),
2995 STI: *STI);
2996
2997 for (int I = 3; I != -1; --I)
2998 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::LDPXpost)
2999 .addReg(Reg: AArch64::SP)
3000 .addReg(Reg: AArch64::X1 + 2 * I)
3001 .addReg(Reg: AArch64::X0 + 2 * I)
3002 .addReg(Reg: AArch64::SP)
3003 .addImm(Val: 2),
3004 STI: *STI);
3005
3006 OutStreamer->emitInstruction(Inst: MCInstBuilder(AArch64::LDPXpost)
3007 .addReg(Reg: AArch64::SP)
3008 .addReg(Reg: AArch64::FP)
3009 .addReg(Reg: AArch64::LR)
3010 .addReg(Reg: AArch64::SP)
3011 .addImm(Val: 2),
3012 STI: *STI);
3013
3014 OutStreamer->emitInstruction(Inst: MCInstBuilder(TM.getTargetTriple().isArm64e()
3015 ? AArch64::BRAAZ
3016 : AArch64::BR)
3017 .addReg(Reg: AArch64::X16),
3018 STI: *STI);
3019}
3020
3021const MCExpr *AArch64AsmPrinter::lowerConstant(const Constant *CV) {
3022 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: CV)) {
3023 return MCSymbolRefExpr::create(Symbol: MCInstLowering.GetGlobalValueSymbol(GV, TargetFlags: 0),
3024 Ctx&: OutContext);
3025 }
3026
3027 return AsmPrinter::lowerConstant(CV);
3028}
3029
3030// Force static initialization.
3031extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() {
3032 RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
3033 RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
3034 RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
3035 RegisterAsmPrinter<AArch64AsmPrinter> W(getTheARM64_32Target());
3036 RegisterAsmPrinter<AArch64AsmPrinter> V(getTheAArch64_32Target());
3037}
3038