1//===- AArch64RegisterInfo.cpp - AArch64 Register Information -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the AArch64 implementation of the TargetRegisterInfo
10// class.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AArch64RegisterInfo.h"
15#include "AArch64FrameLowering.h"
16#include "AArch64InstrInfo.h"
17#include "AArch64MachineFunctionInfo.h"
18#include "AArch64SMEAttributes.h"
19#include "AArch64Subtarget.h"
20#include "MCTargetDesc/AArch64AddressingModes.h"
21#include "MCTargetDesc/AArch64InstPrinter.h"
22#include "llvm/ADT/BitVector.h"
23#include "llvm/BinaryFormat/Dwarf.h"
24#include "llvm/CodeGen/LiveRegMatrix.h"
25#include "llvm/CodeGen/MachineFrameInfo.h"
26#include "llvm/CodeGen/MachineInstrBuilder.h"
27#include "llvm/CodeGen/MachineRegisterInfo.h"
28#include "llvm/CodeGen/RegisterScavenging.h"
29#include "llvm/CodeGen/TargetFrameLowering.h"
30#include "llvm/IR/DebugInfoMetadata.h"
31#include "llvm/IR/DiagnosticInfo.h"
32#include "llvm/IR/Function.h"
33#include "llvm/TargetParser/Triple.h"
34
35using namespace llvm;
36
37#define GET_CC_REGISTER_LISTS
38#include "AArch64GenCallingConv.inc"
39#define GET_REGINFO_TARGET_DESC
40#include "AArch64GenRegisterInfo.inc"
41
42AArch64RegisterInfo::AArch64RegisterInfo(const Triple &TT, unsigned HwMode)
43 : AArch64GenRegisterInfo(AArch64::LR, 0, 0, 0, HwMode), TT(TT) {
44 AArch64_MC::initLLVMToCVRegMapping(MRI: this);
45}
46
47/// Return whether the register needs a CFI entry. Not all unwinders may know
48/// about SVE registers, so we assume the lowest common denominator, i.e. the
49/// callee-saves required by the base ABI. For the SVE registers z8-z15 only the
50/// lower 64-bits (d8-d15) need to be saved. The lower 64-bits subreg is
51/// returned in \p RegToUseForCFI.
52bool AArch64RegisterInfo::regNeedsCFI(MCRegister Reg,
53 MCRegister &RegToUseForCFI) const {
54 if (AArch64::PPRRegClass.contains(Reg))
55 return false;
56
57 if (AArch64::ZPRRegClass.contains(Reg)) {
58 RegToUseForCFI = getSubReg(Reg, Idx: AArch64::dsub);
59 for (int I = 0; CSR_AArch64_AAPCS_SaveList[I]; ++I) {
60 if (CSR_AArch64_AAPCS_SaveList[I] == RegToUseForCFI)
61 return true;
62 }
63 return false;
64 }
65
66 RegToUseForCFI = Reg;
67 return true;
68}
69
70const MCPhysReg *
71AArch64RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
72 assert(MF && "Invalid MachineFunction pointer.");
73
74 auto &AFI = *MF->getInfo<AArch64FunctionInfo>();
75 const auto &F = MF->getFunction();
76 const auto *TLI = MF->getSubtarget<AArch64Subtarget>().getTargetLowering();
77 const bool Darwin = MF->getSubtarget<AArch64Subtarget>().isTargetDarwin();
78 const bool Windows = MF->getSubtarget<AArch64Subtarget>().isTargetWindows();
79
80 if (TLI->supportSwiftError() &&
81 F.getAttributes().hasAttrSomewhere(Kind: Attribute::SwiftError)) {
82 if (Darwin)
83 return CSR_Darwin_AArch64_AAPCS_SwiftError_SaveList;
84 if (Windows)
85 return CSR_Win_AArch64_AAPCS_SwiftError_SaveList;
86 return CSR_AArch64_AAPCS_SwiftError_SaveList;
87 }
88
89 switch (F.getCallingConv()) {
90 case CallingConv::GHC:
91 // GHC set of callee saved regs is empty as all those regs are
92 // used for passing STG regs around
93 return CSR_AArch64_NoRegs_SaveList;
94
95 case CallingConv::PreserveNone:
96 // FIXME: Windows likely need this to be altered for properly unwinding.
97 return CSR_AArch64_NoneRegs_SaveList;
98
99 case CallingConv::AnyReg:
100 return CSR_AArch64_AllRegs_SaveList;
101
102 case CallingConv::ARM64EC_Thunk_X64:
103 return CSR_Win_AArch64_Arm64EC_Thunk_SaveList;
104
105 case CallingConv::PreserveMost:
106 if (Darwin)
107 return CSR_Darwin_AArch64_RT_MostRegs_SaveList;
108 if (Windows)
109 return CSR_Win_AArch64_RT_MostRegs_SaveList;
110 return CSR_AArch64_RT_MostRegs_SaveList;
111
112 case CallingConv::PreserveAll:
113 if (Darwin)
114 return CSR_Darwin_AArch64_RT_AllRegs_SaveList;
115 if (Windows)
116 return CSR_Win_AArch64_RT_AllRegs_SaveList;
117 return CSR_AArch64_RT_AllRegs_SaveList;
118
119 case CallingConv::CFGuard_Check:
120 if (Darwin)
121 report_fatal_error(
122 reason: "Calling convention CFGuard_Check is unsupported on Darwin.");
123 return CSR_Win_AArch64_CFGuard_Check_SaveList;
124
125 case CallingConv::SwiftTail:
126 if (Darwin)
127 return CSR_Darwin_AArch64_AAPCS_SwiftTail_SaveList;
128 if (Windows)
129 return CSR_Win_AArch64_AAPCS_SwiftTail_SaveList;
130 return CSR_AArch64_AAPCS_SwiftTail_SaveList;
131
132 case CallingConv::AArch64_VectorCall:
133 if (Darwin)
134 return CSR_Darwin_AArch64_AAVPCS_SaveList;
135 if (Windows)
136 return CSR_Win_AArch64_AAVPCS_SaveList;
137 return CSR_AArch64_AAVPCS_SaveList;
138
139 case CallingConv::AArch64_SVE_VectorCall:
140 if (Darwin)
141 report_fatal_error(
142 reason: "Calling convention SVE_VectorCall is unsupported on Darwin.");
143 if (Windows)
144 return CSR_Win_AArch64_SVE_AAPCS_SaveList;
145 return CSR_AArch64_SVE_AAPCS_SaveList;
146
147 case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0:
148 report_fatal_error(
149 reason: "Calling convention "
150 "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 is only "
151 "supported to improve calls to SME ACLE save/restore/disable-za "
152 "functions, and is not intended to be used beyond that scope.");
153
154 case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1:
155 report_fatal_error(
156 reason: "Calling convention "
157 "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1 is "
158 "only supported to improve calls to SME ACLE __arm_get_current_vg "
159 "function, and is not intended to be used beyond that scope.");
160
161 case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2:
162 report_fatal_error(
163 reason: "Calling convention "
164 "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 is "
165 "only supported to improve calls to SME ACLE __arm_sme_state "
166 "and is not intended to be used beyond that scope.");
167
168 case CallingConv::Win64:
169 if (Darwin)
170 return CSR_Darwin_AArch64_AAPCS_Win64_SaveList;
171 if (Windows)
172 return CSR_Win_AArch64_AAPCS_SaveList;
173 return CSR_AArch64_AAPCS_X18_SaveList;
174
175 case CallingConv::CXX_FAST_TLS:
176 if (Darwin)
177 return AFI.isSplitCSR() ? CSR_Darwin_AArch64_CXX_TLS_PE_SaveList
178 : CSR_Darwin_AArch64_CXX_TLS_SaveList;
179 // FIXME: this likely should be a `report_fatal_error` condition, however,
180 // that would be a departure from the previously implemented behaviour.
181 LLVM_FALLTHROUGH;
182
183 default:
184 if (Darwin)
185 return AFI.hasSVE_AAPCS(MF: *MF) ? CSR_Darwin_AArch64_SVE_AAPCS_SaveList
186 : CSR_Darwin_AArch64_AAPCS_SaveList;
187 if (Windows)
188 return AFI.hasSVE_AAPCS(MF: *MF) ? CSR_Win_AArch64_SVE_AAPCS_SaveList
189 : CSR_Win_AArch64_AAPCS_SaveList;
190 return AFI.hasSVE_AAPCS(MF: *MF) ? CSR_AArch64_SVE_AAPCS_SaveList
191 : CSR_AArch64_AAPCS_SaveList;
192 }
193}
194
195const MCPhysReg *AArch64RegisterInfo::getCalleeSavedRegsViaCopy(
196 const MachineFunction *MF) const {
197 assert(MF && "Invalid MachineFunction pointer.");
198 if (MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS &&
199 MF->getInfo<AArch64FunctionInfo>()->isSplitCSR())
200 return CSR_Darwin_AArch64_CXX_TLS_ViaCopy_SaveList;
201 return nullptr;
202}
203
204void AArch64RegisterInfo::UpdateCustomCalleeSavedRegs(
205 MachineFunction &MF) const {
206 const MCPhysReg *CSRs = getCalleeSavedRegs(MF: &MF);
207 SmallVector<MCPhysReg, 32> UpdatedCSRs;
208 for (const MCPhysReg *I = CSRs; *I; ++I)
209 UpdatedCSRs.push_back(Elt: *I);
210
211 for (size_t i = 0; i < AArch64::GPR64commonRegClass.getNumRegs(); ++i) {
212 if (MF.getSubtarget<AArch64Subtarget>().isXRegCustomCalleeSaved(i)) {
213 UpdatedCSRs.push_back(Elt: AArch64::GPR64commonRegClass.getRegister(i));
214 }
215 }
216 // Register lists are zero-terminated.
217 UpdatedCSRs.push_back(Elt: 0);
218 MF.getRegInfo().setCalleeSavedRegs(UpdatedCSRs);
219}
220
221const TargetRegisterClass *
222AArch64RegisterInfo::getSubClassWithSubReg(const TargetRegisterClass *RC,
223 unsigned Idx) const {
224 // edge case for GPR/FPR register classes
225 if (RC == &AArch64::GPR32allRegClass && Idx == AArch64::hsub)
226 return &AArch64::FPR32RegClass;
227 else if (RC == &AArch64::GPR64allRegClass && Idx == AArch64::hsub)
228 return &AArch64::FPR64RegClass;
229
230 // Forward to TableGen's default version.
231 return AArch64GenRegisterInfo::getSubClassWithSubReg(RC, Idx);
232}
233
234const uint32_t *
235AArch64RegisterInfo::getDarwinCallPreservedMask(const MachineFunction &MF,
236 CallingConv::ID CC) const {
237 assert(MF.getSubtarget<AArch64Subtarget>().isTargetDarwin() &&
238 "Invalid subtarget for getDarwinCallPreservedMask");
239
240 if (CC == CallingConv::CXX_FAST_TLS)
241 return CSR_Darwin_AArch64_CXX_TLS_RegMask;
242 if (CC == CallingConv::AArch64_VectorCall)
243 return CSR_Darwin_AArch64_AAVPCS_RegMask;
244 if (CC == CallingConv::AArch64_SVE_VectorCall)
245 return CSR_Darwin_AArch64_SVE_AAPCS_RegMask;
246 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0)
247 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask;
248 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1)
249 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1_RegMask;
250 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2)
251 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2_RegMask;
252 if (CC == CallingConv::CFGuard_Check)
253 report_fatal_error(
254 reason: "Calling convention CFGuard_Check is unsupported on Darwin.");
255 if (MF.getSubtarget<AArch64Subtarget>()
256 .getTargetLowering()
257 ->supportSwiftError() &&
258 MF.getFunction().getAttributes().hasAttrSomewhere(Kind: Attribute::SwiftError))
259 return CSR_Darwin_AArch64_AAPCS_SwiftError_RegMask;
260 if (CC == CallingConv::SwiftTail)
261 return CSR_Darwin_AArch64_AAPCS_SwiftTail_RegMask;
262 if (CC == CallingConv::PreserveMost)
263 return CSR_Darwin_AArch64_RT_MostRegs_RegMask;
264 if (CC == CallingConv::PreserveAll)
265 return CSR_Darwin_AArch64_RT_AllRegs_RegMask;
266 return CSR_Darwin_AArch64_AAPCS_RegMask;
267}
268
269const uint32_t *
270AArch64RegisterInfo::getCallPreservedMask(const MachineFunction &MF,
271 CallingConv::ID CC) const {
272 bool SCS = MF.getFunction().hasFnAttribute(Kind: Attribute::ShadowCallStack);
273 if (CC == CallingConv::GHC)
274 // This is academic because all GHC calls are (supposed to be) tail calls
275 return SCS ? CSR_AArch64_NoRegs_SCS_RegMask : CSR_AArch64_NoRegs_RegMask;
276 if (CC == CallingConv::PreserveNone)
277 return SCS ? CSR_AArch64_NoneRegs_SCS_RegMask
278 : CSR_AArch64_NoneRegs_RegMask;
279 if (CC == CallingConv::AnyReg)
280 return SCS ? CSR_AArch64_AllRegs_SCS_RegMask : CSR_AArch64_AllRegs_RegMask;
281
282 // All the following calling conventions are handled differently on Darwin.
283 if (MF.getSubtarget<AArch64Subtarget>().isTargetDarwin()) {
284 if (SCS)
285 report_fatal_error(reason: "ShadowCallStack attribute not supported on Darwin.");
286 return getDarwinCallPreservedMask(MF, CC);
287 }
288
289 if (CC == CallingConv::AArch64_VectorCall)
290 return SCS ? CSR_AArch64_AAVPCS_SCS_RegMask : CSR_AArch64_AAVPCS_RegMask;
291 if (CC == CallingConv::AArch64_SVE_VectorCall)
292 return SCS ? CSR_AArch64_SVE_AAPCS_SCS_RegMask
293 : CSR_AArch64_SVE_AAPCS_RegMask;
294 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0)
295 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask;
296 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1)
297 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1_RegMask;
298 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2)
299 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2_RegMask;
300 if (CC == CallingConv::CFGuard_Check)
301 return CSR_Win_AArch64_CFGuard_Check_RegMask;
302 if (MF.getSubtarget<AArch64Subtarget>().getTargetLowering()
303 ->supportSwiftError() &&
304 MF.getFunction().getAttributes().hasAttrSomewhere(Kind: Attribute::SwiftError))
305 return SCS ? CSR_AArch64_AAPCS_SwiftError_SCS_RegMask
306 : CSR_AArch64_AAPCS_SwiftError_RegMask;
307 if (CC == CallingConv::SwiftTail) {
308 if (SCS)
309 report_fatal_error(reason: "ShadowCallStack attribute not supported with swifttail");
310 return CSR_AArch64_AAPCS_SwiftTail_RegMask;
311 }
312 if (CC == CallingConv::PreserveMost)
313 return SCS ? CSR_AArch64_RT_MostRegs_SCS_RegMask
314 : CSR_AArch64_RT_MostRegs_RegMask;
315 if (CC == CallingConv::PreserveAll)
316 return SCS ? CSR_AArch64_RT_AllRegs_SCS_RegMask
317 : CSR_AArch64_RT_AllRegs_RegMask;
318
319 return SCS ? CSR_AArch64_AAPCS_SCS_RegMask : CSR_AArch64_AAPCS_RegMask;
320}
321
322const uint32_t *AArch64RegisterInfo::getCustomEHPadPreservedMask(
323 const MachineFunction &MF) const {
324 if (MF.getSubtarget<AArch64Subtarget>().isTargetLinux())
325 return CSR_AArch64_AAPCS_RegMask;
326
327 return nullptr;
328}
329
330const uint32_t *AArch64RegisterInfo::getTLSCallPreservedMask() const {
331 if (TT.isOSDarwin())
332 return CSR_Darwin_AArch64_TLS_RegMask;
333
334 assert(TT.isOSBinFormatELF() && "Invalid target");
335 return CSR_AArch64_TLS_ELF_RegMask;
336}
337
338void AArch64RegisterInfo::UpdateCustomCallPreservedMask(MachineFunction &MF,
339 const uint32_t **Mask) const {
340 uint32_t *UpdatedMask = MF.allocateRegMask();
341 unsigned RegMaskSize = MachineOperand::getRegMaskSize(NumRegs: getNumRegs());
342 memcpy(dest: UpdatedMask, src: *Mask, n: sizeof(UpdatedMask[0]) * RegMaskSize);
343
344 for (size_t i = 0; i < AArch64::GPR64commonRegClass.getNumRegs(); ++i) {
345 if (MF.getSubtarget<AArch64Subtarget>().isXRegCustomCalleeSaved(i)) {
346 for (MCPhysReg SubReg :
347 subregs_inclusive(Reg: AArch64::GPR64commonRegClass.getRegister(i))) {
348 // See TargetRegisterInfo::getCallPreservedMask for how to interpret the
349 // register mask.
350 UpdatedMask[SubReg / 32] |= 1u << (SubReg % 32);
351 }
352 }
353 }
354 *Mask = UpdatedMask;
355}
356
357const uint32_t *AArch64RegisterInfo::getSMStartStopCallPreservedMask() const {
358 return CSR_AArch64_SMStartStop_RegMask;
359}
360
361const uint32_t *
362AArch64RegisterInfo::SMEABISupportRoutinesCallPreservedMaskFromX0() const {
363 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask;
364}
365
366const uint32_t *AArch64RegisterInfo::getNoPreservedMask() const {
367 return CSR_AArch64_NoRegs_RegMask;
368}
369
370const uint32_t *
371AArch64RegisterInfo::getThisReturnPreservedMask(const MachineFunction &MF,
372 CallingConv::ID CC) const {
373 // This should return a register mask that is the same as that returned by
374 // getCallPreservedMask but that additionally preserves the register used for
375 // the first i64 argument (which must also be the register used to return a
376 // single i64 return value)
377 //
378 // In case that the calling convention does not use the same register for
379 // both, the function should return NULL (does not currently apply)
380 assert(CC != CallingConv::GHC && "should not be GHC calling convention.");
381 if (MF.getSubtarget<AArch64Subtarget>().isTargetDarwin())
382 return CSR_Darwin_AArch64_AAPCS_ThisReturn_RegMask;
383 return CSR_AArch64_AAPCS_ThisReturn_RegMask;
384}
385
386const uint32_t *AArch64RegisterInfo::getWindowsStackProbePreservedMask() const {
387 return CSR_AArch64_StackProbe_Windows_RegMask;
388}
389
390std::optional<std::string>
391AArch64RegisterInfo::explainReservedReg(const MachineFunction &MF,
392 MCRegister PhysReg) const {
393 if (hasBasePointer(MF) && MCRegisterInfo::regsOverlap(RegA: PhysReg, RegB: AArch64::X19))
394 return std::string("X19 is used as the frame base pointer register.");
395
396 if (MF.getSubtarget<AArch64Subtarget>().isWindowsArm64EC()) {
397 bool warn = false;
398 if (MCRegisterInfo::regsOverlap(RegA: PhysReg, RegB: AArch64::X13) ||
399 MCRegisterInfo::regsOverlap(RegA: PhysReg, RegB: AArch64::X14) ||
400 MCRegisterInfo::regsOverlap(RegA: PhysReg, RegB: AArch64::X23) ||
401 MCRegisterInfo::regsOverlap(RegA: PhysReg, RegB: AArch64::X24) ||
402 MCRegisterInfo::regsOverlap(RegA: PhysReg, RegB: AArch64::X28))
403 warn = true;
404
405 for (unsigned i = AArch64::B16; i <= AArch64::B31; ++i)
406 if (MCRegisterInfo::regsOverlap(RegA: PhysReg, RegB: i))
407 warn = true;
408
409 if (warn)
410 return std::string(AArch64InstPrinter::getRegisterName(Reg: PhysReg)) +
411 " is clobbered by asynchronous signals when using Arm64EC.";
412 }
413
414 return {};
415}
416
417BitVector
418AArch64RegisterInfo::getStrictlyReservedRegs(const MachineFunction &MF) const {
419 const AArch64FrameLowering *TFI = getFrameLowering(MF);
420
421 BitVector Reserved(getNumRegs());
422 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::WSP);
423 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::WZR);
424
425 if (TFI->isFPReserved(MF))
426 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W29);
427
428 if (MF.getSubtarget<AArch64Subtarget>().isWindowsArm64EC()) {
429 // x13, x14, x23, x24, x28, and v16-v31 are clobbered by asynchronous
430 // signals, so we can't ever use them.
431 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W13);
432 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W14);
433 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W23);
434 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W24);
435 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W28);
436 for (unsigned i = AArch64::B16; i <= AArch64::B31; ++i)
437 markSuperRegs(RegisterSet&: Reserved, Reg: i);
438 }
439
440 if (MF.getSubtarget<AArch64Subtarget>().isLFI()) {
441 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W28);
442 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W27);
443 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W26);
444 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W25);
445 if (!MF.getProperties().hasNoVRegs()) {
446 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::LR);
447 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W30);
448 }
449 }
450
451 for (size_t i = 0; i < AArch64::GPR32commonRegClass.getNumRegs(); ++i) {
452 if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(i))
453 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::GPR32commonRegClass.getRegister(i));
454 }
455
456 if (hasBasePointer(MF))
457 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W19);
458
459 // SLH uses register W16/X16 as the taint register.
460 if (MF.getFunction().hasFnAttribute(Kind: Attribute::SpeculativeLoadHardening))
461 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W16);
462
463 // FFR is modelled as global state that cannot be allocated.
464 if (MF.getSubtarget<AArch64Subtarget>().hasSVE())
465 Reserved.set(AArch64::FFR);
466
467 // SME tiles are not allocatable.
468 if (MF.getSubtarget<AArch64Subtarget>().hasSME()) {
469 for (MCPhysReg SubReg : subregs_inclusive(Reg: AArch64::ZA))
470 Reserved.set(SubReg);
471 }
472
473 // VG cannot be allocated
474 Reserved.set(AArch64::VG);
475
476 if (MF.getSubtarget<AArch64Subtarget>().hasSME2()) {
477 for (MCSubRegIterator SubReg(AArch64::ZT0, this, /*self=*/true);
478 SubReg.isValid(); ++SubReg)
479 Reserved.set(*SubReg);
480 }
481
482 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::FPCR);
483 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::FPMR);
484 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::FPSR);
485
486 if (MF.getFunction().getCallingConv() == CallingConv::GRAAL) {
487 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::X27);
488 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::X28);
489 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W27);
490 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W28);
491 }
492
493 assert(checkAllSuperRegsMarked(Reserved));
494
495 // Add _HI registers after checkAllSuperRegsMarked as this check otherwise
496 // becomes considerably more expensive.
497 Reserved.set(AArch64::WSP_HI);
498 Reserved.set(AArch64::WZR_HI);
499 static_assert(AArch64::W30_HI - AArch64::W0_HI == 30,
500 "Unexpected order of registers");
501 Reserved.set(I: AArch64::W0_HI, E: AArch64::W30_HI + 1);
502 static_assert(AArch64::B31_HI - AArch64::B0_HI == 31,
503 "Unexpected order of registers");
504 Reserved.set(I: AArch64::B0_HI, E: AArch64::B31_HI + 1);
505 static_assert(AArch64::H31_HI - AArch64::H0_HI == 31,
506 "Unexpected order of registers");
507 Reserved.set(I: AArch64::H0_HI, E: AArch64::H31_HI + 1);
508 static_assert(AArch64::S31_HI - AArch64::S0_HI == 31,
509 "Unexpected order of registers");
510 Reserved.set(I: AArch64::S0_HI, E: AArch64::S31_HI + 1);
511 static_assert(AArch64::D31_HI - AArch64::D0_HI == 31,
512 "Unexpected order of registers");
513 Reserved.set(I: AArch64::D0_HI, E: AArch64::D31_HI + 1);
514 static_assert(AArch64::Q31_HI - AArch64::Q0_HI == 31,
515 "Unexpected order of registers");
516 Reserved.set(I: AArch64::Q0_HI, E: AArch64::Q31_HI + 1);
517
518 return Reserved;
519}
520
521BitVector
522AArch64RegisterInfo::getUserReservedRegs(const MachineFunction &MF) const {
523 BitVector Reserved(getNumRegs());
524 for (size_t i = 0; i < AArch64::GPR32commonRegClass.getNumRegs(); ++i) {
525 // ReserveXRegister is set for registers manually reserved
526 // through +reserve-x#i.
527 if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(i))
528 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::GPR32commonRegClass.getRegister(i));
529 }
530 return Reserved;
531}
532
533BitVector
534AArch64RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
535 BitVector Reserved(getNumRegs());
536 for (size_t i = 0; i < AArch64::GPR32commonRegClass.getNumRegs(); ++i) {
537 if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReservedForRA(i))
538 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::GPR32commonRegClass.getRegister(i));
539 }
540
541 if (MF.getSubtarget<AArch64Subtarget>().isLRReservedForRA()) {
542 // In order to prevent the register allocator from using LR, we need to
543 // mark it as reserved. However we don't want to keep it reserved throughout
544 // the pipeline since it prevents other infrastructure from reasoning about
545 // it's liveness. We use the NoVRegs property instead of IsSSA because
546 // IsSSA is removed before VirtRegRewriter runs.
547 if (!MF.getProperties().hasNoVRegs())
548 // Reserve LR (X30) by marking from its subregister W30 because otherwise
549 // the register allocator could clobber the subregister.
550 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W30);
551 }
552
553 assert(checkAllSuperRegsMarked(Reserved));
554
555 // Handle strictlyReservedRegs separately to avoid re-evaluating the assert,
556 // which becomes considerably expensive when considering the _HI registers.
557 Reserved |= getStrictlyReservedRegs(MF);
558
559 return Reserved;
560}
561
562bool AArch64RegisterInfo::isReservedReg(const MachineFunction &MF,
563 MCRegister Reg) const {
564 return getReservedRegs(MF)[Reg];
565}
566
567bool AArch64RegisterInfo::isUserReservedReg(const MachineFunction &MF,
568 MCRegister Reg) const {
569 return getUserReservedRegs(MF)[Reg];
570}
571
572bool AArch64RegisterInfo::isStrictlyReservedReg(const MachineFunction &MF,
573 MCRegister Reg) const {
574 return getStrictlyReservedRegs(MF)[Reg];
575}
576
577bool AArch64RegisterInfo::isAnyArgRegReserved(const MachineFunction &MF) const {
578 for (size_t i = 0; i < 8; ++i) {
579 if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(i))
580 return true;
581 }
582 return false;
583}
584
585void AArch64RegisterInfo::emitReservedArgRegCallError(
586 const MachineFunction &MF) const {
587 const Function &F = MF.getFunction();
588 F.getContext().diagnose(DI: DiagnosticInfoUnsupported{F, ("AArch64 doesn't support"
589 " function calls if any of the argument registers is reserved.")});
590}
591
592bool AArch64RegisterInfo::isAsmClobberable(const MachineFunction &MF,
593 MCRegister PhysReg) const {
594 // SLH uses register X16 as the taint register but it will fallback to a different
595 // method if the user clobbers it. So X16 is not reserved for inline asm but is
596 // for normal codegen.
597 if (MF.getFunction().hasFnAttribute(Kind: Attribute::SpeculativeLoadHardening) &&
598 MCRegisterInfo::regsOverlap(RegA: PhysReg, RegB: AArch64::X16))
599 return true;
600
601 // ZA/ZT0 registers are reserved but may be permitted in the clobber list.
602 if (PhysReg == AArch64::ZA || PhysReg == AArch64::ZT0)
603 return true;
604
605 return !isReservedReg(MF, Reg: PhysReg);
606}
607
608const TargetRegisterClass *
609AArch64RegisterInfo::getCrossCopyRegClass(const TargetRegisterClass *RC) const {
610 if (RC == &AArch64::CCRRegClass)
611 return &AArch64::GPR64RegClass; // Only MSR & MRS copy NZCV.
612 return RC;
613}
614
615MCRegister AArch64RegisterInfo::getBaseRegister() const { return AArch64::X19; }
616
617bool AArch64RegisterInfo::hasBasePointer(const MachineFunction &MF) const {
618 const MachineFrameInfo &MFI = MF.getFrameInfo();
619
620 // In the presence of variable sized objects or funclets, if the fixed stack
621 // size is large enough that referencing from the FP won't result in things
622 // being in range relatively often, we can use a base pointer to allow access
623 // from the other direction like the SP normally works.
624 //
625 // Furthermore, if both variable sized objects are present, and the
626 // stack needs to be dynamically re-aligned, the base pointer is the only
627 // reliable way to reference the locals.
628 if (MFI.hasVarSizedObjects() || MF.hasEHFunclets()) {
629 if (hasStackRealignment(MF))
630 return true;
631
632 auto &ST = MF.getSubtarget<AArch64Subtarget>();
633 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
634 if (ST.hasSVE() || ST.isStreaming()) {
635 // Frames that have variable sized objects and scalable SVE objects,
636 // should always use a basepointer.
637 if (!AFI->hasCalculatedStackSizeSVE() || AFI->hasSVEStackSize())
638 return true;
639 }
640
641 // Frames with hazard padding can have a large offset between the frame
642 // pointer and GPR locals, which includes the emergency spill slot. If the
643 // emergency spill slot is not within range of the load/store instructions
644 // (which have a signed 9-bit range), we will fail to compile if it is used.
645 // Since hasBasePointer() is called before we know if we have hazard padding
646 // or an emergency spill slot we need to enable the basepointer
647 // conservatively.
648 if (ST.getStreamingHazardSize() &&
649 !AFI->getSMEFnAttrs().hasNonStreamingInterfaceAndBody()) {
650 return true;
651 }
652
653 // Conservatively estimate whether the negative offset from the frame
654 // pointer will be sufficient to reach. If a function has a smallish
655 // frame, it's less likely to have lots of spills and callee saved
656 // space, so it's all more likely to be within range of the frame pointer.
657 // If it's wrong, we'll materialize the constant and still get to the
658 // object; it's just suboptimal. Negative offsets use the unscaled
659 // load/store instructions, which have a 9-bit signed immediate.
660 return MFI.getLocalFrameSize() >= 256;
661 }
662
663 return false;
664}
665
666bool AArch64RegisterInfo::isArgumentRegister(const MachineFunction &MF,
667 MCRegister Reg) const {
668 CallingConv::ID CC = MF.getFunction().getCallingConv();
669 const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
670 bool IsVarArg = STI.isCallingConvWin64(CC: MF.getFunction().getCallingConv(),
671 IsVarArg: MF.getFunction().isVarArg());
672
673 auto HasReg = [](ArrayRef<MCRegister> RegList, MCRegister Reg) {
674 return llvm::is_contained(Range&: RegList, Element: Reg);
675 };
676
677 switch (CC) {
678 default:
679 report_fatal_error(reason: "Unsupported calling convention.");
680 case CallingConv::GHC:
681 return HasReg(CC_AArch64_GHC_ArgRegs, Reg);
682 case CallingConv::PreserveNone:
683 if (!MF.getFunction().isVarArg())
684 return HasReg(CC_AArch64_Preserve_None_ArgRegs, Reg);
685 [[fallthrough]];
686 case CallingConv::C:
687 case CallingConv::Fast:
688 case CallingConv::PreserveMost:
689 case CallingConv::PreserveAll:
690 case CallingConv::CXX_FAST_TLS:
691 case CallingConv::Swift:
692 case CallingConv::SwiftTail:
693 case CallingConv::Tail:
694 if (STI.isTargetWindows()) {
695 if (IsVarArg)
696 return HasReg(CC_AArch64_Win64_VarArg_ArgRegs, Reg);
697 switch (CC) {
698 default:
699 return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg);
700 case CallingConv::Swift:
701 case CallingConv::SwiftTail:
702 return HasReg(CC_AArch64_Win64PCS_Swift_ArgRegs, Reg) ||
703 HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg);
704 }
705 }
706 if (!STI.isTargetDarwin()) {
707 switch (CC) {
708 default:
709 return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg);
710 case CallingConv::Swift:
711 case CallingConv::SwiftTail:
712 return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg) ||
713 HasReg(CC_AArch64_AAPCS_Swift_ArgRegs, Reg);
714 }
715 }
716 if (!IsVarArg) {
717 switch (CC) {
718 default:
719 return HasReg(CC_AArch64_DarwinPCS_ArgRegs, Reg);
720 case CallingConv::Swift:
721 case CallingConv::SwiftTail:
722 return HasReg(CC_AArch64_DarwinPCS_ArgRegs, Reg) ||
723 HasReg(CC_AArch64_DarwinPCS_Swift_ArgRegs, Reg);
724 }
725 }
726 if (STI.isTargetILP32())
727 return HasReg(CC_AArch64_DarwinPCS_ILP32_VarArg_ArgRegs, Reg);
728 return HasReg(CC_AArch64_DarwinPCS_VarArg_ArgRegs, Reg);
729 case CallingConv::Win64:
730 if (IsVarArg)
731 HasReg(CC_AArch64_Win64_VarArg_ArgRegs, Reg);
732 return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg);
733 case CallingConv::CFGuard_Check:
734 return HasReg(CC_AArch64_Win64_CFGuard_Check_ArgRegs, Reg);
735 case CallingConv::AArch64_VectorCall:
736 case CallingConv::AArch64_SVE_VectorCall:
737 case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0:
738 case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1:
739 case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2:
740 if (STI.isTargetWindows())
741 return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg);
742 return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg);
743 }
744}
745
746Register
747AArch64RegisterInfo::getFrameRegister(const MachineFunction &MF) const {
748 const AArch64FrameLowering *TFI = getFrameLowering(MF);
749 return TFI->hasFP(MF) ? AArch64::FP : AArch64::SP;
750}
751
752bool AArch64RegisterInfo::requiresRegisterScavenging(
753 const MachineFunction &MF) const {
754 return true;
755}
756
757bool AArch64RegisterInfo::requiresVirtualBaseRegisters(
758 const MachineFunction &MF) const {
759 return true;
760}
761
762bool
763AArch64RegisterInfo::useFPForScavengingIndex(const MachineFunction &MF) const {
764 // This function indicates whether the emergency spillslot should be placed
765 // close to the beginning of the stackframe (closer to FP) or the end
766 // (closer to SP).
767 //
768 // The beginning works most reliably if we have a frame pointer.
769 // In the presence of any non-constant space between FP and locals,
770 // (e.g. in case of stack realignment or a scalable SVE area), it is
771 // better to use SP or BP.
772 const AArch64FrameLowering &TFI = *getFrameLowering(MF);
773 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
774 assert((!MF.getSubtarget<AArch64Subtarget>().hasSVE() ||
775 AFI->hasCalculatedStackSizeSVE()) &&
776 "Expected SVE area to be calculated by this point");
777 return TFI.hasFP(MF) && !hasStackRealignment(MF) && !AFI->hasSVEStackSize() &&
778 !AFI->hasStackHazardSlotIndex();
779}
780
781bool AArch64RegisterInfo::requiresFrameIndexScavenging(
782 const MachineFunction &MF) const {
783 return true;
784}
785
786bool
787AArch64RegisterInfo::cannotEliminateFrame(const MachineFunction &MF) const {
788 const MachineFrameInfo &MFI = MF.getFrameInfo();
789 if (MF.disableFramePointerElim() && MFI.adjustsStack())
790 return true;
791 return MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken();
792}
793
794/// needsFrameBaseReg - Returns true if the instruction's frame index
795/// reference would be better served by a base register other than FP
796/// or SP. Used by LocalStackFrameAllocation to determine which frame index
797/// references it should create new base registers for.
798bool AArch64RegisterInfo::needsFrameBaseReg(MachineInstr *MI,
799 int64_t Offset) const {
800 for (unsigned i = 0; !MI->getOperand(i).isFI(); ++i)
801 assert(i < MI->getNumOperands() &&
802 "Instr doesn't have FrameIndex operand!");
803
804 // It's the load/store FI references that cause issues, as it can be difficult
805 // to materialize the offset if it won't fit in the literal field. Estimate
806 // based on the size of the local frame and some conservative assumptions
807 // about the rest of the stack frame (note, this is pre-regalloc, so
808 // we don't know everything for certain yet) whether this offset is likely
809 // to be out of range of the immediate. Return true if so.
810
811 // We only generate virtual base registers for loads and stores, so
812 // return false for everything else.
813 if (!MI->mayLoad() && !MI->mayStore())
814 return false;
815
816 // Without a virtual base register, if the function has variable sized
817 // objects, all fixed-size local references will be via the frame pointer,
818 // Approximate the offset and see if it's legal for the instruction.
819 // Note that the incoming offset is based on the SP value at function entry,
820 // so it'll be negative.
821 MachineFunction &MF = *MI->getParent()->getParent();
822 const AArch64FrameLowering *TFI = getFrameLowering(MF);
823 MachineFrameInfo &MFI = MF.getFrameInfo();
824
825 // Estimate an offset from the frame pointer.
826 // Conservatively assume all GPR callee-saved registers get pushed.
827 // FP, LR, X19-X28, D8-D15. 64-bits each.
828 int64_t FPOffset = Offset - 16 * 20;
829 // Estimate an offset from the stack pointer.
830 // The incoming offset is relating to the SP at the start of the function,
831 // but when we access the local it'll be relative to the SP after local
832 // allocation, so adjust our SP-relative offset by that allocation size.
833 Offset += MFI.getLocalFrameSize();
834 // Assume that we'll have at least some spill slots allocated.
835 // FIXME: This is a total SWAG number. We should run some statistics
836 // and pick a real one.
837 Offset += 128; // 128 bytes of spill slots
838
839 // If there is a frame pointer, try using it.
840 // The FP is only available if there is no dynamic realignment. We
841 // don't know for sure yet whether we'll need that, so we guess based
842 // on whether there are any local variables that would trigger it.
843 if (TFI->hasFP(MF) && isFrameOffsetLegal(MI, BaseReg: AArch64::FP, Offset: FPOffset))
844 return false;
845
846 // If we can reference via the stack pointer or base pointer, try that.
847 // FIXME: This (and the code that resolves the references) can be improved
848 // to only disallow SP relative references in the live range of
849 // the VLA(s). In practice, it's unclear how much difference that
850 // would make, but it may be worth doing.
851 if (isFrameOffsetLegal(MI, BaseReg: AArch64::SP, Offset))
852 return false;
853
854 // If even offset 0 is illegal, we don't want a virtual base register.
855 if (!isFrameOffsetLegal(MI, BaseReg: AArch64::SP, Offset: 0))
856 return false;
857
858 // The offset likely isn't legal; we want to allocate a virtual base register.
859 return true;
860}
861
862bool AArch64RegisterInfo::isFrameOffsetLegal(const MachineInstr *MI,
863 Register BaseReg,
864 int64_t Offset) const {
865 assert(MI && "Unable to get the legal offset for nil instruction.");
866 StackOffset SaveOffset = StackOffset::getFixed(Fixed: Offset);
867 return isAArch64FrameOffsetLegal(MI: *MI, Offset&: SaveOffset) & AArch64FrameOffsetIsLegal;
868}
869
870/// Insert defining instruction(s) for BaseReg to be a pointer to FrameIdx
871/// at the beginning of the basic block.
872Register
873AArch64RegisterInfo::materializeFrameBaseRegister(MachineBasicBlock *MBB,
874 int FrameIdx,
875 int64_t Offset) const {
876 MachineBasicBlock::iterator Ins = MBB->begin();
877 DebugLoc DL; // Defaults to "unknown"
878 if (Ins != MBB->end())
879 DL = Ins->getDebugLoc();
880 const MachineFunction &MF = *MBB->getParent();
881 const AArch64InstrInfo *TII =
882 MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
883 const MCInstrDesc &MCID = TII->get(Opcode: AArch64::ADDXri);
884 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
885 Register BaseReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR64spRegClass);
886 MRI.constrainRegClass(Reg: BaseReg, RC: TII->getRegClass(MCID, OpNum: 0));
887 unsigned Shifter = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0);
888
889 BuildMI(BB&: *MBB, I: Ins, MIMD: DL, MCID, DestReg: BaseReg)
890 .addFrameIndex(Idx: FrameIdx)
891 .addImm(Val: Offset)
892 .addImm(Val: Shifter);
893
894 return BaseReg;
895}
896
897void AArch64RegisterInfo::resolveFrameIndex(MachineInstr &MI, Register BaseReg,
898 int64_t Offset) const {
899 // ARM doesn't need the general 64-bit offsets
900 StackOffset Off = StackOffset::getFixed(Fixed: Offset);
901
902 unsigned i = 0;
903 while (!MI.getOperand(i).isFI()) {
904 ++i;
905 assert(i < MI.getNumOperands() && "Instr doesn't have FrameIndex operand!");
906 }
907
908 const MachineFunction *MF = MI.getParent()->getParent();
909 const AArch64InstrInfo *TII =
910 MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
911 bool Done = rewriteAArch64FrameIndex(MI, FrameRegIdx: i, FrameReg: BaseReg, Offset&: Off, TII);
912 assert(Done && "Unable to resolve frame index!");
913 (void)Done;
914}
915
916// Create a scratch register for the frame index elimination in an instruction.
917// This function has special handling of stack tagging loop pseudos, in which
918// case it can also change the instruction opcode.
919static Register
920createScratchRegisterForInstruction(MachineInstr &MI, unsigned FIOperandNum,
921 const AArch64InstrInfo *TII) {
922 // ST*Gloop have a reserved scratch register in operand 1. Use it, and also
923 // replace the instruction with the writeback variant because it will now
924 // satisfy the operand constraints for it.
925 Register ScratchReg;
926 if (MI.getOpcode() == AArch64::STGloop ||
927 MI.getOpcode() == AArch64::STZGloop) {
928 assert(FIOperandNum == 3 &&
929 "Wrong frame index operand for STGloop/STZGloop");
930 unsigned Op = MI.getOpcode() == AArch64::STGloop ? AArch64::STGloop_wback
931 : AArch64::STZGloop_wback;
932 ScratchReg = MI.getOperand(i: 1).getReg();
933 MI.getOperand(i: 3).ChangeToRegister(Reg: ScratchReg, isDef: false, isImp: false, isKill: true);
934 MI.setDesc(TII->get(Opcode: Op));
935 MI.tieOperands(DefIdx: 1, UseIdx: 3);
936 } else {
937 ScratchReg =
938 MI.getMF()->getRegInfo().createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
939 MI.getOperand(i: FIOperandNum)
940 .ChangeToRegister(Reg: ScratchReg, isDef: false, isImp: false, isKill: true);
941 }
942 return ScratchReg;
943}
944
945void AArch64RegisterInfo::getOffsetOpcodes(
946 const StackOffset &Offset, SmallVectorImpl<uint64_t> &Ops) const {
947 // The smallest scalable element supported by scaled SVE addressing
948 // modes are predicates, which are 2 scalable bytes in size. So the scalable
949 // byte offset must always be a multiple of 2.
950 assert(Offset.getScalable() % 2 == 0 && "Invalid frame offset");
951
952 // Add fixed-sized offset using existing DIExpression interface.
953 DIExpression::appendOffset(Ops, Offset: Offset.getFixed());
954
955 unsigned VG = getDwarfRegNum(Reg: AArch64::VG, isEH: true);
956 int64_t VGSized = Offset.getScalable() / 2;
957 if (VGSized > 0) {
958 Ops.push_back(Elt: dwarf::DW_OP_constu);
959 Ops.push_back(Elt: VGSized);
960 Ops.append(IL: {dwarf::DW_OP_bregx, VG, 0ULL});
961 Ops.push_back(Elt: dwarf::DW_OP_mul);
962 Ops.push_back(Elt: dwarf::DW_OP_plus);
963 } else if (VGSized < 0) {
964 Ops.push_back(Elt: dwarf::DW_OP_constu);
965 Ops.push_back(Elt: -VGSized);
966 Ops.append(IL: {dwarf::DW_OP_bregx, VG, 0ULL});
967 Ops.push_back(Elt: dwarf::DW_OP_mul);
968 Ops.push_back(Elt: dwarf::DW_OP_minus);
969 }
970}
971
972bool AArch64RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
973 int SPAdj, unsigned FIOperandNum,
974 RegScavenger *RS) const {
975 assert(SPAdj == 0 && "Unexpected");
976
977 MachineInstr &MI = *II;
978 MachineBasicBlock &MBB = *MI.getParent();
979 MachineFunction &MF = *MBB.getParent();
980 const MachineFrameInfo &MFI = MF.getFrameInfo();
981 const AArch64InstrInfo *TII =
982 MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
983 const AArch64FrameLowering *TFI = getFrameLowering(MF);
984 int FrameIndex = MI.getOperand(i: FIOperandNum).getIndex();
985 bool Tagged =
986 MI.getOperand(i: FIOperandNum).getTargetFlags() & AArch64II::MO_TAGGED;
987 Register FrameReg;
988
989 // Special handling of dbg_value, stackmap patchpoint statepoint instructions.
990 if (MI.getOpcode() == TargetOpcode::STACKMAP ||
991 MI.getOpcode() == TargetOpcode::PATCHPOINT ||
992 MI.getOpcode() == TargetOpcode::STATEPOINT) {
993 StackOffset Offset =
994 TFI->resolveFrameIndexReference(MF, FI: FrameIndex, FrameReg,
995 /*PreferFP=*/true,
996 /*ForSimm=*/false);
997 Offset += StackOffset::getFixed(Fixed: MI.getOperand(i: FIOperandNum + 1).getImm());
998 MI.getOperand(i: FIOperandNum).ChangeToRegister(Reg: FrameReg, isDef: false /*isDef*/);
999 MI.getOperand(i: FIOperandNum + 1).ChangeToImmediate(ImmVal: Offset.getFixed());
1000 return false;
1001 }
1002
1003 if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE) {
1004 MachineOperand &FI = MI.getOperand(i: FIOperandNum);
1005 StackOffset Offset = TFI->getNonLocalFrameIndexReference(MF, FI: FrameIndex);
1006 assert(!Offset.getScalable() &&
1007 "Frame offsets with a scalable component are not supported");
1008 FI.ChangeToImmediate(ImmVal: Offset.getFixed());
1009 return false;
1010 }
1011
1012 StackOffset Offset;
1013 if (MI.getOpcode() == AArch64::TAGPstack) {
1014 // TAGPstack must use the virtual frame register in its 3rd operand.
1015 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1016 FrameReg = MI.getOperand(i: 3).getReg();
1017 Offset = StackOffset::getFixed(Fixed: MFI.getObjectOffset(ObjectIdx: FrameIndex) +
1018 AFI->getTaggedBasePointerOffset());
1019 } else if (Tagged) {
1020 StackOffset SPOffset = StackOffset::getFixed(
1021 Fixed: MFI.getObjectOffset(ObjectIdx: FrameIndex) + (int64_t)MFI.getStackSize());
1022 if (MFI.hasVarSizedObjects() ||
1023 isAArch64FrameOffsetLegal(MI, Offset&: SPOffset, OutUseUnscaledOp: nullptr, OutUnscaledOp: nullptr, EmittableOffset: nullptr) !=
1024 (AArch64FrameOffsetCanUpdate | AArch64FrameOffsetIsLegal)) {
1025 // Can't update to SP + offset in place. Precalculate the tagged pointer
1026 // in a scratch register.
1027 Offset = TFI->resolveFrameIndexReference(
1028 MF, FI: FrameIndex, FrameReg, /*PreferFP=*/false, /*ForSimm=*/true);
1029 Register ScratchReg =
1030 MF.getRegInfo().createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
1031 emitFrameOffset(MBB, MBBI: II, DL: MI.getDebugLoc(), DestReg: ScratchReg, SrcReg: FrameReg, Offset,
1032 TII);
1033 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::LDG), DestReg: ScratchReg)
1034 .addReg(RegNo: ScratchReg)
1035 .addReg(RegNo: ScratchReg)
1036 .addImm(Val: 0);
1037 MI.getOperand(i: FIOperandNum)
1038 .ChangeToRegister(Reg: ScratchReg, isDef: false, isImp: false, isKill: true);
1039 return false;
1040 }
1041 FrameReg = AArch64::SP;
1042 Offset = StackOffset::getFixed(Fixed: MFI.getObjectOffset(ObjectIdx: FrameIndex) +
1043 (int64_t)MFI.getStackSize());
1044 } else {
1045 Offset = TFI->resolveFrameIndexReference(
1046 MF, FI: FrameIndex, FrameReg, /*PreferFP=*/false, /*ForSimm=*/true);
1047 }
1048
1049 // Modify MI as necessary to handle as much of 'Offset' as possible
1050 if (rewriteAArch64FrameIndex(MI, FrameRegIdx: FIOperandNum, FrameReg, Offset, TII))
1051 return true;
1052
1053 assert((!RS || !RS->isScavengingFrameIndex(FrameIndex)) &&
1054 "Emergency spill slot is out of reach");
1055
1056 // If we get here, the immediate doesn't fit into the instruction. We folded
1057 // as much as possible above. Handle the rest, providing a register that is
1058 // SP+LargeImm.
1059 Register ScratchReg =
1060 createScratchRegisterForInstruction(MI, FIOperandNum, TII);
1061 emitFrameOffset(MBB, MBBI: II, DL: MI.getDebugLoc(), DestReg: ScratchReg, SrcReg: FrameReg, Offset, TII);
1062 return false;
1063}
1064
1065unsigned AArch64RegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
1066 MachineFunction &MF) const {
1067 const AArch64FrameLowering *TFI = getFrameLowering(MF);
1068
1069 switch (RC->getID()) {
1070 default:
1071 return 0;
1072 case AArch64::GPR32RegClassID:
1073 case AArch64::GPR32spRegClassID:
1074 case AArch64::GPR32allRegClassID:
1075 case AArch64::GPR64spRegClassID:
1076 case AArch64::GPR64allRegClassID:
1077 case AArch64::GPR64RegClassID:
1078 case AArch64::GPR32commonRegClassID:
1079 case AArch64::GPR64commonRegClassID:
1080 return 32 - 1 // XZR/SP
1081 - (TFI->hasFP(MF) || TT.isOSDarwin()) // FP
1082 - MF.getSubtarget<AArch64Subtarget>().getNumXRegisterReserved()
1083 - hasBasePointer(MF); // X19
1084 case AArch64::FPR8RegClassID:
1085 case AArch64::FPR16RegClassID:
1086 case AArch64::FPR32RegClassID:
1087 case AArch64::FPR64RegClassID:
1088 case AArch64::FPR128RegClassID:
1089 return 32;
1090
1091 case AArch64::MatrixIndexGPR32_8_11RegClassID:
1092 case AArch64::MatrixIndexGPR32_12_15RegClassID:
1093 return 4;
1094
1095 case AArch64::DDRegClassID:
1096 case AArch64::DDDRegClassID:
1097 case AArch64::DDDDRegClassID:
1098 case AArch64::QQRegClassID:
1099 case AArch64::QQQRegClassID:
1100 case AArch64::QQQQRegClassID:
1101 return 32;
1102
1103 case AArch64::FPR128_loRegClassID:
1104 case AArch64::FPR64_loRegClassID:
1105 case AArch64::FPR16_loRegClassID:
1106 return 16;
1107 case AArch64::FPR128_0to7RegClassID:
1108 return 8;
1109 }
1110}
1111
1112static bool HandleDestructivePredicateHint(
1113 Register VirtReg, ArrayRef<MCPhysReg> Order,
1114 SmallVectorImpl<MCPhysReg> &Hints, const VirtRegMap *VRM,
1115 const MachineRegisterInfo &MRI, const TargetInstrInfo &TII,
1116 const AArch64Subtarget &ST, const LiveRegMatrix *Matrix) {
1117 const TargetRegisterClass *RegRC = MRI.getRegClass(Reg: VirtReg);
1118 if (!ST.useDistinctPredicateDstReg() ||
1119 !AArch64::PPRRegClass.hasSubClassEq(RC: RegRC) || !MRI.hasOneDef(RegNo: VirtReg) ||
1120 Order.size() < 2)
1121 return false;
1122
1123 const MachineInstr *DefInst = MRI.getOneDef(Reg: VirtReg)->getParent();
1124 if ((TII.get(Opcode: DefInst->getOpcode()).TSFlags &
1125 AArch64::DestructiveInstTypeMask) != AArch64::DestructivePredicate)
1126 return false;
1127
1128 Register Op1Reg = DefInst->getOperand(i: 1).getReg();
1129 if (Op1Reg.isVirtual())
1130 Op1Reg = VRM->getPhys(virtReg: Op1Reg);
1131
1132 // If no register is allocated for the general-predicate, it's not yet
1133 // possible to choose a distinct register.
1134 if (!Op1Reg.isValid())
1135 return false;
1136
1137 // Move Op1Reg as the least preferred register.
1138 //
1139 // This might result in callee-save spills when the function takes/returns
1140 // arguments in SVE registers (i.e. needs to preserve p4-p15) and can't reuse
1141 // p0-p3. That's why we limit it to non-callee saved registers or to
1142 // callee-saved registers that have already been allocated for other uses in
1143 // the function.
1144 DenseSet<unsigned> CSRs;
1145 for (unsigned I = 0;; ++I) {
1146 Register R = MRI.getCalleeSavedRegs()[I];
1147 if (!R.isValid())
1148 break;
1149 if (AArch64::PPRRegClass.contains(Reg: R))
1150 CSRs.insert(V: R);
1151 }
1152
1153 Hints.append(in_start: Order.begin(), in_end: Order.end());
1154 auto CanUseReg = [&](Register R) {
1155 return !CSRs.contains(V: R) || !MRI.def_empty(RegNo: R) || Matrix->isPhysRegUsed(PhysReg: R);
1156 };
1157 llvm::stable_sort(Range&: Hints, C: [&](Register A, Register B) {
1158 bool PrefA = (A != Op1Reg) && CanUseReg(A);
1159 bool PrefB = (B != Op1Reg) && CanUseReg(B);
1160 return PrefA && !PrefB;
1161 });
1162 return true;
1163}
1164
1165// We add regalloc hints for different cases:
1166// * Choosing a better destination operand for predicated SVE instructions
1167// where the inactive lanes are undef, by choosing a register that is not
1168// unique to the other operands of the instruction.
1169//
1170// * Improve register allocation for SME multi-vector instructions where we can
1171// benefit from the strided- and contiguous register multi-vector tuples.
1172//
1173// Here COPY_INTO_TRANSPOSED_TUPLE nodes are created to improve register
1174// allocation where a consecutive multi-vector tuple is constructed from the
1175// same indices of multiple strided loads. This may still result in
1176// unnecessary copies between the loads and the tuple. Here we try to return a
1177// hint to assign the contiguous ZPRMulReg starting at the same register as
1178// the first operand of the pseudo, which should be a subregister of the first
1179// strided load.
1180//
1181// For example, if the first strided load has been assigned $z16_z20_z24_z28
1182// and the operands of the pseudo are each accessing subregister zsub2, we
1183// should look through through Order to find a contiguous register which
1184// begins with $z24 (i.e. $z24_z25_z26_z27).
1185bool AArch64RegisterInfo::getRegAllocationHints(
1186 Register VirtReg, ArrayRef<MCPhysReg> Order,
1187 SmallVectorImpl<MCPhysReg> &Hints, const MachineFunction &MF,
1188 const VirtRegMap *VRM, const LiveRegMatrix *Matrix) const {
1189 auto &ST = MF.getSubtarget<AArch64Subtarget>();
1190 const AArch64InstrInfo *TII =
1191 MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
1192 const MachineRegisterInfo &MRI = MF.getRegInfo();
1193
1194 bool ConsiderOnlyHints =
1195 TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, MF, VRM);
1196
1197 // For predicated SVE instructions where the inactive lanes are undef,
1198 // pick a destination register that is not unique to avoid introducing
1199 // a movprfx.
1200 const TargetRegisterClass *RegRC = MRI.getRegClass(Reg: VirtReg);
1201 if (AArch64::ZPRRegClass.hasSubClassEq(RC: RegRC)) {
1202 for (const MachineOperand &DefOp : MRI.def_operands(Reg: VirtReg)) {
1203 const MachineInstr &Def = *DefOp.getParent();
1204 if (DefOp.isImplicit() ||
1205 (TII->get(Opcode: Def.getOpcode()).TSFlags & AArch64::FalseLanesMask) !=
1206 AArch64::FalseLanesUndef)
1207 continue;
1208
1209 unsigned InstFlags =
1210 TII->get(Opcode: AArch64::getSVEPseudoMap(Opcode: Def.getOpcode())).TSFlags;
1211
1212 for (MCPhysReg R : Order) {
1213 auto AddHintIfSuitable = [&](MCPhysReg R,
1214 const MachineOperand &MO) -> bool {
1215 // R is a suitable register hint if R can reuse one of the other
1216 // source operands.
1217 MCPhysReg PhysReg = VRM->getPhys(virtReg: MO.getReg());
1218 if (PhysReg && MO.getSubReg())
1219 PhysReg = getSubReg(Reg: PhysReg, Idx: MO.getSubReg());
1220 if (PhysReg != R)
1221 return false;
1222 Hints.push_back(Elt: R);
1223 return true;
1224 };
1225
1226 switch (InstFlags & AArch64::DestructiveInstTypeMask) {
1227 default:
1228 break;
1229 case AArch64::DestructiveTernaryCommWithRev:
1230 AddHintIfSuitable(R, Def.getOperand(i: 2)) ||
1231 AddHintIfSuitable(R, Def.getOperand(i: 3)) ||
1232 AddHintIfSuitable(R, Def.getOperand(i: 4));
1233 break;
1234 case AArch64::DestructiveBinaryComm:
1235 case AArch64::DestructiveBinaryCommWithRev:
1236 AddHintIfSuitable(R, Def.getOperand(i: 2)) ||
1237 AddHintIfSuitable(R, Def.getOperand(i: 3));
1238 break;
1239 case AArch64::DestructiveBinary:
1240 case AArch64::DestructiveBinaryImm:
1241 AddHintIfSuitable(R, Def.getOperand(i: 2));
1242 break;
1243 case AArch64::DestructiveUnaryPassthru:
1244 AddHintIfSuitable(R, Def.getOperand(i: 3));
1245 break;
1246 case AArch64::DestructiveBinaryImmUnpred:
1247 case AArch64::DestructiveBinaryShImmUnpred:
1248 AddHintIfSuitable(R, Def.getOperand(i: 1));
1249 break;
1250 }
1251 }
1252 }
1253
1254 if (Hints.size())
1255 return ConsiderOnlyHints;
1256 }
1257
1258 if (HandleDestructivePredicateHint(VirtReg, Order, Hints, VRM, MRI, TII: *TII, ST,
1259 Matrix))
1260 return ConsiderOnlyHints;
1261
1262 if (!ST.hasSME() || !ST.isStreaming())
1263 return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, MF,
1264 VRM);
1265
1266 // The SVE calling convention preserves registers Z8-Z23. As a result, there
1267 // are no ZPR2Strided or ZPR4Strided registers that do not overlap with the
1268 // callee-saved registers and so by default these will be pushed to the back
1269 // of the allocation order for the ZPRStridedOrContiguous classes.
1270 // If any of the instructions which define VirtReg are used by the
1271 // COPY_INTO_TRANSPOSED_TUPLE pseudos, we want to favour reducing copy
1272 // instructions over reducing the number of clobbered callee-save registers,
1273 // so we add the strided registers as a hint.
1274 unsigned RegID = RegRC->getID();
1275 if (RegID == AArch64::ZPR2StridedOrContiguousRegClassID ||
1276 RegID == AArch64::ZPR4StridedOrContiguousRegClassID) {
1277
1278 // Look through uses of the register for COPY_INTO_TRANSPOSED_TUPLE.
1279 for (const MachineInstr &Use : MRI.use_nodbg_instructions(Reg: VirtReg)) {
1280 if (Use.getOpcode() != AArch64::COPY_INTO_TRANSPOSED_TUPLE)
1281 continue;
1282
1283 const MachineOperand &Src = Use.getOperand(i: 1);
1284 const MachineOperand &Dst = Use.getOperand(i: 0);
1285
1286 if (!Src.getSubReg() || !Dst.getSubReg())
1287 continue;
1288
1289 const TargetRegisterClass *StridedRC;
1290 switch (RegID) {
1291 case AArch64::ZPR2StridedOrContiguousRegClassID:
1292 StridedRC = &AArch64::ZPR2StridedRegClass;
1293 break;
1294 case AArch64::ZPR4StridedOrContiguousRegClassID:
1295 StridedRC = &AArch64::ZPR4StridedRegClass;
1296 break;
1297 default:
1298 llvm_unreachable("Unexpected RegID");
1299 }
1300
1301 SmallVector<MCPhysReg, 4> StridedOrder;
1302 for (MCPhysReg Reg : Order)
1303 if (StridedRC->contains(Reg))
1304 StridedOrder.push_back(Elt: Reg);
1305
1306 unsigned TupleSize = Use.getOperand(i: 2).getImm();
1307 unsigned TupIdx = Dst.getSubReg() - AArch64::zsub0;
1308
1309 unsigned TupleID = MRI.getRegClass(Reg: Dst.getReg())->getID();
1310 bool IsMulZPR = TupleID == AArch64::ZPR2Mul2RegClassID ||
1311 TupleID == AArch64::ZPR4Mul4RegClassID;
1312
1313 iterator_range Copies = MRI.def_instructions(Reg: Dst.getReg());
1314 MachineRegisterInfo::def_instr_iterator CopyWithAssignedSrc =
1315 llvm::find_if(Range&: Copies, P: [&](const MachineInstr &Def) {
1316 auto &Src = Def.getOperand(i: 1);
1317 return Def.getOpcode() == Use.getOpcode() &&
1318 VRM->hasPhys(virtReg: Src.getReg());
1319 });
1320
1321 // Example:
1322 //
1323 // When trying to find a suitable register allocation for VirtReg %v2 in:
1324 //
1325 // %v0:zpr2stridedorcontiguous = ld1 p0/z, [...]
1326 // %v1:zpr2stridedorcontiguous = ld1 p0/z, [...]
1327 // %v2:zpr2stridedorcontiguous = ld1 p0/z, [...]
1328 // %v3:zpr2stridedorcontiguous = ld1 p0/z, [...]
1329 // %v4.zsub0:zpr4mul4 = COPY_INTO_TRANSPOSED_TUPLE %v0:0
1330 // %v4.zsub1:zpr4mul4 = COPY_INTO_TRANSPOSED_TUPLE %v1:0
1331 // %v4.zsub2:zpr4mul4 = COPY_INTO_TRANSPOSED_TUPLE %v2:0
1332 // %v4.zsub3:zpr4mul4 = COPY_INTO_TRANSPOSED_TUPLE %v3:0
1333 //
1334 // One such suitable allocation would be:
1335 //
1336 // { z0, z8 } = ld1 p0/z, [...]
1337 // { z1, z9 } = ld1 p0/z, [...]
1338 // { z2, z10 } = ld1 p0/z, [...]
1339 // { z3, z11 } = ld1 p0/z, [...]
1340 // z0 = COPY_INTO_TRANSPOSED_TUPLE {z0, z8}:0
1341 // z1 = COPY_INTO_TRANSPOSED_TUPLE {z1, z9}:0
1342 // z2 = COPY_INTO_TRANSPOSED_TUPLE {z2, z10}:0
1343 // z3 = COPY_INTO_TRANSPOSED_TUPLE {z3, z11}:0
1344 //
1345 // Below we distinguish two cases when trying to find a register:
1346 // * None of the sources of the copies have been assigned a register yet.
1347 // In this case the code must ensure that there are at least TupleSize
1348 // free consecutive registers. If IsMulZPR is true, then the first of
1349 // registers must also be a multiple of TupleSize, e.g.
1350 // { z0, z1, z2, z3 } is valid but { z1, z2, z3, z5 } is not.
1351 // * One or more copies already have registers assigned to their sources,
1352 // which means only checking that a consecutive range of free tuple
1353 // registers exists which includes the assigned register.
1354 // e.g. in the example above, if { z0, z8 } is already allocated for
1355 // %v0, we just need to ensure that { z1, z9 }, { z2, z10 } and
1356 // { z3, z11 } are also free. If so, we add { z2, z10 }.
1357
1358 if (CopyWithAssignedSrc == Copies.end()) {
1359 // There are no registers already assigned to any of the pseudo
1360 // operands. Look for a valid starting register for the group.
1361 for (unsigned I = 0; I < StridedOrder.size(); ++I) {
1362 MCPhysReg Reg = StridedOrder[I];
1363
1364 // If the COPY_INTO_TRANSPOSED_TUPLE nodes use the ZPRMul classes, the
1365 // starting register of the first load should be a multiple of 2 or 4.
1366 unsigned SubRegIdx = Src.getSubReg();
1367 if (IsMulZPR &&
1368 (getSubReg(Reg, Idx: SubRegIdx) - AArch64::Z0) % TupleSize != TupIdx)
1369 continue;
1370
1371 // In the example above, if VirtReg is the third operand of the
1372 // tuple (%v2) and Reg == Z2_Z10, then we need to make sure that
1373 // Z0_Z8, Z1_Z9 and Z3_Z11 are also available.
1374 auto IsFreeConsecutiveReg = [&](unsigned I) {
1375 unsigned R = Reg - TupIdx + I;
1376 return StridedRC->contains(Reg: R) &&
1377 (I == 0 ||
1378 ((getSubReg(Reg: R, Idx: AArch64::zsub0) - AArch64::Z0) ==
1379 (getSubReg(Reg: R - 1, Idx: AArch64::zsub0) - AArch64::Z0) + 1)) &&
1380 !Matrix->isPhysRegUsed(PhysReg: R);
1381 };
1382 if (all_of(Range: seq(Begin: 0U, End: TupleSize), P: IsFreeConsecutiveReg))
1383 Hints.push_back(Elt: Reg);
1384 }
1385 } else {
1386 // At least copy already has a physical register assigned to its source.
1387 // Find the starting sub-register of this and use it to work out the
1388 // correct strided register to suggest based on the current op index.
1389 MachineOperand &AssignedSrc = CopyWithAssignedSrc->getOperand(i: 1);
1390 MachineOperand &AssignedDst = CopyWithAssignedSrc->getOperand(i: 0);
1391
1392 if (!AssignedSrc.getSubReg() || !AssignedDst.getSubReg())
1393 continue;
1394
1395 unsigned AssignedTupIdx = AssignedDst.getSubReg() - AArch64::zsub0;
1396 MCPhysReg TargetStartReg =
1397 getSubReg(Reg: VRM->getPhys(virtReg: AssignedSrc.getReg()), Idx: AArch64::zsub0) +
1398 (TupIdx - AssignedTupIdx);
1399
1400 for (unsigned I = 0; I < StridedOrder.size(); ++I)
1401 if (getSubReg(Reg: StridedOrder[I], Idx: AArch64::zsub0) == TargetStartReg)
1402 Hints.push_back(Elt: StridedOrder[I]);
1403 }
1404
1405 if (!Hints.empty())
1406 return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints,
1407 MF, VRM);
1408 }
1409 }
1410
1411 for (auto &Def : MRI.def_instructions(Reg: VirtReg)) {
1412 if (Def.getOpcode() != AArch64::COPY_INTO_TRANSPOSED_TUPLE)
1413 continue;
1414
1415 MachineOperand &Src = Def.getOperand(i: 1);
1416 MachineOperand &Dst = Def.getOperand(i: 0);
1417
1418 if (!Src.getSubReg() || !Dst.getSubReg())
1419 continue;
1420
1421 // FIXME: This is fragile. If we allocate a register to the Dst before Src,
1422 // our hints are won't have any effect... This is currently mitigated by
1423 // by trying to schedule copies immediately before their uses. This gives
1424 // them a short live range (so they're low priority to allocate).
1425 if (!VRM->hasPhys(virtReg: Src.getReg()))
1426 continue;
1427
1428 // Find the ZPR register mapped to the source of the copy.
1429 MCPhysReg SrcZPR = getSubReg(Reg: VRM->getPhys(virtReg: Src.getReg()), Idx: Src.getSubReg());
1430
1431 // Try to pick a tuple register for Dst with Src as a member.
1432 for (MCPhysReg R : Order) {
1433 if (getSubReg(Reg: R, Idx: Dst.getSubReg()) == SrcZPR)
1434 Hints.push_back(Elt: R);
1435 }
1436 }
1437
1438 return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, MF,
1439 VRM);
1440}
1441
1442unsigned AArch64RegisterInfo::getLocalAddressRegister(
1443 const MachineFunction &MF) const {
1444 const auto &MFI = MF.getFrameInfo();
1445 if (!MF.hasEHFunclets() && !MFI.hasVarSizedObjects())
1446 return AArch64::SP;
1447 else if (hasStackRealignment(MF))
1448 return getBaseRegister();
1449 return getFrameRegister(MF);
1450}
1451
1452/// SrcRC and DstRC will be morphed into NewRC if this returns true
1453bool AArch64RegisterInfo::shouldCoalesce(
1454 MachineInstr *MI, const TargetRegisterClass *SrcRC, unsigned SubReg,
1455 const TargetRegisterClass *DstRC, unsigned DstSubReg,
1456 const TargetRegisterClass *NewRC, LiveIntervals &LIS) const {
1457 MachineFunction &MF = *MI->getMF();
1458 MachineRegisterInfo &MRI = MF.getRegInfo();
1459
1460 if (MI->isSubregToReg() && MRI.subRegLivenessEnabled() &&
1461 !MF.getSubtarget<AArch64Subtarget>().enableSRLTSubregToRegMitigation())
1462 return false;
1463
1464 if (MI->isCopy() &&
1465 ((DstRC->getID() == AArch64::GPR64RegClassID) ||
1466 (DstRC->getID() == AArch64::GPR64commonRegClassID)) &&
1467 MI->getOperand(i: 0).getSubReg() && MI->getOperand(i: 1).getSubReg())
1468 // Do not coalesce in the case of a 32-bit subregister copy
1469 // which implements a 32 to 64 bit zero extension
1470 // which relies on the upper 32 bits being zeroed.
1471 return false;
1472
1473 auto IsCoalescerBarrier = [](const MachineInstr &MI) {
1474 switch (MI.getOpcode()) {
1475 case AArch64::COALESCER_BARRIER_FPR16:
1476 case AArch64::COALESCER_BARRIER_FPR32:
1477 case AArch64::COALESCER_BARRIER_FPR64:
1478 case AArch64::COALESCER_BARRIER_FPR128:
1479 return true;
1480 default:
1481 return false;
1482 }
1483 };
1484
1485 // For calls that temporarily have to toggle streaming mode as part of the
1486 // call-sequence, we need to be more careful when coalescing copy instructions
1487 // so that we don't end up coalescing the NEON/FP result or argument register
1488 // with a whole Z-register, such that after coalescing the register allocator
1489 // will try to spill/reload the entire Z register.
1490 //
1491 // We do this by checking if the node has any defs/uses that are
1492 // COALESCER_BARRIER pseudos. These are 'nops' in practice, but they exist to
1493 // instruct the coalescer to avoid coalescing the copy.
1494 if (MI->isCopy() && SubReg != DstSubReg &&
1495 (AArch64::ZPRRegClass.hasSubClassEq(RC: DstRC) ||
1496 AArch64::ZPRRegClass.hasSubClassEq(RC: SrcRC))) {
1497 unsigned SrcReg = MI->getOperand(i: 1).getReg();
1498 if (any_of(Range: MRI.def_instructions(Reg: SrcReg), P: IsCoalescerBarrier))
1499 return false;
1500 unsigned DstReg = MI->getOperand(i: 0).getReg();
1501 if (any_of(Range: MRI.use_nodbg_instructions(Reg: DstReg), P: IsCoalescerBarrier))
1502 return false;
1503 }
1504
1505 return true;
1506}
1507
1508bool AArch64RegisterInfo::shouldAnalyzePhysregInMachineLoopInfo(
1509 MCRegister R) const {
1510 return R == AArch64::VG;
1511}
1512
1513bool AArch64RegisterInfo::isIgnoredCVReg(MCRegister LLVMReg) const {
1514 return (LLVMReg >= AArch64::Z0 && LLVMReg <= AArch64::Z31) ||
1515 (LLVMReg >= AArch64::P0 && LLVMReg <= AArch64::P15);
1516}
1517