1//===- AArch64RegisterInfo.cpp - AArch64 Register Information -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the AArch64 implementation of the TargetRegisterInfo
10// class.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AArch64RegisterInfo.h"
15#include "AArch64FrameLowering.h"
16#include "AArch64InstrInfo.h"
17#include "AArch64MachineFunctionInfo.h"
18#include "AArch64SMEAttributes.h"
19#include "AArch64Subtarget.h"
20#include "MCTargetDesc/AArch64AddressingModes.h"
21#include "MCTargetDesc/AArch64InstPrinter.h"
22#include "llvm/ADT/BitVector.h"
23#include "llvm/BinaryFormat/Dwarf.h"
24#include "llvm/CodeGen/LiveRegMatrix.h"
25#include "llvm/CodeGen/MachineFrameInfo.h"
26#include "llvm/CodeGen/MachineInstrBuilder.h"
27#include "llvm/CodeGen/MachineRegisterInfo.h"
28#include "llvm/CodeGen/RegisterScavenging.h"
29#include "llvm/CodeGen/TargetFrameLowering.h"
30#include "llvm/IR/DebugInfoMetadata.h"
31#include "llvm/IR/DiagnosticInfo.h"
32#include "llvm/IR/Function.h"
33#include "llvm/TargetParser/Triple.h"
34
35using namespace llvm;
36
37#define GET_CC_REGISTER_LISTS
38#include "AArch64GenCallingConv.inc"
39#define GET_REGINFO_TARGET_DESC
40#include "AArch64GenRegisterInfo.inc"
41
42AArch64RegisterInfo::AArch64RegisterInfo(const Triple &TT, unsigned HwMode)
43 : AArch64GenRegisterInfo(AArch64::LR, 0, 0, 0, HwMode), TT(TT) {
44 AArch64_MC::initLLVMToCVRegMapping(MRI: this);
45}
46
47/// Return whether the register needs a CFI entry. Not all unwinders may know
48/// about SVE registers, so we assume the lowest common denominator, i.e. the
49/// callee-saves required by the base ABI. For the SVE registers z8-z15 only the
50/// lower 64-bits (d8-d15) need to be saved. The lower 64-bits subreg is
51/// returned in \p RegToUseForCFI.
52bool AArch64RegisterInfo::regNeedsCFI(MCRegister Reg,
53 MCRegister &RegToUseForCFI) const {
54 if (AArch64::PPRRegClass.contains(Reg))
55 return false;
56
57 if (AArch64::ZPRRegClass.contains(Reg)) {
58 RegToUseForCFI = getSubReg(Reg, Idx: AArch64::dsub);
59 for (int I = 0; CSR_AArch64_AAPCS_SaveList[I]; ++I) {
60 if (CSR_AArch64_AAPCS_SaveList[I] == RegToUseForCFI)
61 return true;
62 }
63 return false;
64 }
65
66 RegToUseForCFI = Reg;
67 return true;
68}
69
70const MCPhysReg *
71AArch64RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
72 assert(MF && "Invalid MachineFunction pointer.");
73
74 auto &AFI = *MF->getInfo<AArch64FunctionInfo>();
75 const auto &F = MF->getFunction();
76 const auto *TLI = MF->getSubtarget<AArch64Subtarget>().getTargetLowering();
77 const bool Darwin = MF->getSubtarget<AArch64Subtarget>().isTargetDarwin();
78 const bool Windows =
79 MF->getSubtarget<AArch64Subtarget>().isTargetWindowsOrUEFI();
80
81 if (TLI->supportSwiftError() &&
82 F.getAttributes().hasAttrSomewhere(Kind: Attribute::SwiftError)) {
83 if (Darwin)
84 return CSR_Darwin_AArch64_AAPCS_SwiftError_SaveList;
85 if (Windows)
86 return CSR_Win_AArch64_AAPCS_SwiftError_SaveList;
87 return CSR_AArch64_AAPCS_SwiftError_SaveList;
88 }
89
90 switch (F.getCallingConv()) {
91 case CallingConv::GHC:
92 // GHC set of callee saved regs is empty as all those regs are
93 // used for passing STG regs around
94 return CSR_AArch64_NoRegs_SaveList;
95
96 case CallingConv::PreserveNone:
97 // FIXME: Windows likely need this to be altered for properly unwinding.
98 return CSR_AArch64_NoneRegs_SaveList;
99
100 case CallingConv::AnyReg:
101 return CSR_AArch64_AllRegs_SaveList;
102
103 case CallingConv::ARM64EC_Thunk_X64:
104 return CSR_Win_AArch64_Arm64EC_Thunk_SaveList;
105
106 case CallingConv::PreserveMost:
107 if (Darwin)
108 return CSR_Darwin_AArch64_RT_MostRegs_SaveList;
109 if (Windows)
110 return CSR_Win_AArch64_RT_MostRegs_SaveList;
111 return CSR_AArch64_RT_MostRegs_SaveList;
112
113 case CallingConv::PreserveAll:
114 if (Darwin)
115 return CSR_Darwin_AArch64_RT_AllRegs_SaveList;
116 if (Windows)
117 return CSR_Win_AArch64_RT_AllRegs_SaveList;
118 return CSR_AArch64_RT_AllRegs_SaveList;
119
120 case CallingConv::CFGuard_Check:
121 if (Darwin)
122 report_fatal_error(
123 reason: "Calling convention CFGuard_Check is unsupported on Darwin.");
124 return CSR_Win_AArch64_CFGuard_Check_SaveList;
125
126 case CallingConv::SwiftTail:
127 if (Darwin)
128 return CSR_Darwin_AArch64_AAPCS_SwiftTail_SaveList;
129 if (Windows)
130 return CSR_Win_AArch64_AAPCS_SwiftTail_SaveList;
131 return CSR_AArch64_AAPCS_SwiftTail_SaveList;
132
133 case CallingConv::AArch64_VectorCall:
134 if (Darwin)
135 return CSR_Darwin_AArch64_AAVPCS_SaveList;
136 if (Windows)
137 return CSR_Win_AArch64_AAVPCS_SaveList;
138 return CSR_AArch64_AAVPCS_SaveList;
139
140 case CallingConv::AArch64_SVE_VectorCall:
141 if (Darwin)
142 report_fatal_error(
143 reason: "Calling convention SVE_VectorCall is unsupported on Darwin.");
144 if (Windows)
145 return CSR_Win_AArch64_SVE_AAPCS_SaveList;
146 return CSR_AArch64_SVE_AAPCS_SaveList;
147
148 case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0:
149 report_fatal_error(
150 reason: "Calling convention "
151 "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 is only "
152 "supported to improve calls to SME ACLE save/restore/disable-za "
153 "functions, and is not intended to be used beyond that scope.");
154
155 case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1:
156 report_fatal_error(
157 reason: "Calling convention "
158 "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1 is "
159 "only supported to improve calls to SME ACLE __arm_get_current_vg "
160 "function, and is not intended to be used beyond that scope.");
161
162 case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2:
163 report_fatal_error(
164 reason: "Calling convention "
165 "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 is "
166 "only supported to improve calls to SME ACLE __arm_sme_state "
167 "and is not intended to be used beyond that scope.");
168
169 case CallingConv::Win64:
170 if (Darwin)
171 return CSR_Darwin_AArch64_AAPCS_Win64_SaveList;
172 if (Windows)
173 return CSR_Win_AArch64_AAPCS_SaveList;
174 return CSR_AArch64_AAPCS_X18_SaveList;
175
176 case CallingConv::CXX_FAST_TLS:
177 if (Darwin)
178 return AFI.isSplitCSR() ? CSR_Darwin_AArch64_CXX_TLS_PE_SaveList
179 : CSR_Darwin_AArch64_CXX_TLS_SaveList;
180 // FIXME: this likely should be a `report_fatal_error` condition, however,
181 // that would be a departure from the previously implemented behaviour.
182 LLVM_FALLTHROUGH;
183
184 default:
185 if (Darwin)
186 return AFI.hasSVE_AAPCS(MF: *MF) ? CSR_Darwin_AArch64_SVE_AAPCS_SaveList
187 : CSR_Darwin_AArch64_AAPCS_SaveList;
188 if (Windows)
189 return AFI.hasSVE_AAPCS(MF: *MF) ? CSR_Win_AArch64_SVE_AAPCS_SaveList
190 : CSR_Win_AArch64_AAPCS_SaveList;
191 return AFI.hasSVE_AAPCS(MF: *MF) ? CSR_AArch64_SVE_AAPCS_SaveList
192 : CSR_AArch64_AAPCS_SaveList;
193 }
194}
195
196const MCPhysReg *AArch64RegisterInfo::getCalleeSavedRegsViaCopy(
197 const MachineFunction *MF) const {
198 assert(MF && "Invalid MachineFunction pointer.");
199 if (MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS &&
200 MF->getInfo<AArch64FunctionInfo>()->isSplitCSR())
201 return CSR_Darwin_AArch64_CXX_TLS_ViaCopy_SaveList;
202 return nullptr;
203}
204
205void AArch64RegisterInfo::UpdateCustomCalleeSavedRegs(
206 MachineFunction &MF) const {
207 const MCPhysReg *CSRs = getCalleeSavedRegs(MF: &MF);
208 SmallVector<MCPhysReg, 32> UpdatedCSRs;
209 for (const MCPhysReg *I = CSRs; *I; ++I)
210 UpdatedCSRs.push_back(Elt: *I);
211
212 for (size_t i = 0; i < AArch64::GPR64commonRegClass.getNumRegs(); ++i) {
213 if (MF.getSubtarget<AArch64Subtarget>().isXRegCustomCalleeSaved(i)) {
214 UpdatedCSRs.push_back(Elt: AArch64::GPR64commonRegClass.getRegister(i));
215 }
216 }
217 // Register lists are zero-terminated.
218 UpdatedCSRs.push_back(Elt: 0);
219 MF.getRegInfo().setCalleeSavedRegs(UpdatedCSRs);
220}
221
222const TargetRegisterClass *
223AArch64RegisterInfo::getSubClassWithSubReg(const TargetRegisterClass *RC,
224 unsigned Idx) const {
225 // edge case for GPR/FPR register classes
226 if (RC == &AArch64::GPR32allRegClass && Idx == AArch64::hsub)
227 return &AArch64::FPR32RegClass;
228 else if (RC == &AArch64::GPR64allRegClass && Idx == AArch64::hsub)
229 return &AArch64::FPR64RegClass;
230
231 // Forward to TableGen's default version.
232 return AArch64GenRegisterInfo::getSubClassWithSubReg(RC, Idx);
233}
234
235const uint32_t *
236AArch64RegisterInfo::getDarwinCallPreservedMask(const MachineFunction &MF,
237 CallingConv::ID CC) const {
238 assert(MF.getSubtarget<AArch64Subtarget>().isTargetDarwin() &&
239 "Invalid subtarget for getDarwinCallPreservedMask");
240
241 if (CC == CallingConv::CXX_FAST_TLS)
242 return CSR_Darwin_AArch64_CXX_TLS_RegMask;
243 if (CC == CallingConv::AArch64_VectorCall)
244 return CSR_Darwin_AArch64_AAVPCS_RegMask;
245 if (CC == CallingConv::AArch64_SVE_VectorCall)
246 return CSR_Darwin_AArch64_SVE_AAPCS_RegMask;
247 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0)
248 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask;
249 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1)
250 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1_RegMask;
251 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2)
252 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2_RegMask;
253 if (CC == CallingConv::CFGuard_Check)
254 report_fatal_error(
255 reason: "Calling convention CFGuard_Check is unsupported on Darwin.");
256 if (MF.getSubtarget<AArch64Subtarget>()
257 .getTargetLowering()
258 ->supportSwiftError() &&
259 MF.getFunction().getAttributes().hasAttrSomewhere(Kind: Attribute::SwiftError))
260 return CSR_Darwin_AArch64_AAPCS_SwiftError_RegMask;
261 if (CC == CallingConv::SwiftTail)
262 return CSR_Darwin_AArch64_AAPCS_SwiftTail_RegMask;
263 if (CC == CallingConv::PreserveMost)
264 return CSR_Darwin_AArch64_RT_MostRegs_RegMask;
265 if (CC == CallingConv::PreserveAll)
266 return CSR_Darwin_AArch64_RT_AllRegs_RegMask;
267 return CSR_Darwin_AArch64_AAPCS_RegMask;
268}
269
270const uint32_t *
271AArch64RegisterInfo::getCallPreservedMask(const MachineFunction &MF,
272 CallingConv::ID CC) const {
273 bool SCS = MF.getFunction().hasFnAttribute(Kind: Attribute::ShadowCallStack);
274 if (CC == CallingConv::GHC)
275 // This is academic because all GHC calls are (supposed to be) tail calls
276 return SCS ? CSR_AArch64_NoRegs_SCS_RegMask : CSR_AArch64_NoRegs_RegMask;
277 if (CC == CallingConv::PreserveNone)
278 return SCS ? CSR_AArch64_NoneRegs_SCS_RegMask
279 : CSR_AArch64_NoneRegs_RegMask;
280 if (CC == CallingConv::AnyReg)
281 return SCS ? CSR_AArch64_AllRegs_SCS_RegMask : CSR_AArch64_AllRegs_RegMask;
282
283 // All the following calling conventions are handled differently on Darwin.
284 if (MF.getSubtarget<AArch64Subtarget>().isTargetDarwin()) {
285 if (SCS)
286 report_fatal_error(reason: "ShadowCallStack attribute not supported on Darwin.");
287 return getDarwinCallPreservedMask(MF, CC);
288 }
289
290 if (CC == CallingConv::AArch64_VectorCall)
291 return SCS ? CSR_AArch64_AAVPCS_SCS_RegMask : CSR_AArch64_AAVPCS_RegMask;
292 if (CC == CallingConv::AArch64_SVE_VectorCall)
293 return SCS ? CSR_AArch64_SVE_AAPCS_SCS_RegMask
294 : CSR_AArch64_SVE_AAPCS_RegMask;
295 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0)
296 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask;
297 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1)
298 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1_RegMask;
299 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2)
300 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2_RegMask;
301 if (CC == CallingConv::CFGuard_Check)
302 return CSR_Win_AArch64_CFGuard_Check_RegMask;
303 if (MF.getSubtarget<AArch64Subtarget>().getTargetLowering()
304 ->supportSwiftError() &&
305 MF.getFunction().getAttributes().hasAttrSomewhere(Kind: Attribute::SwiftError))
306 return SCS ? CSR_AArch64_AAPCS_SwiftError_SCS_RegMask
307 : CSR_AArch64_AAPCS_SwiftError_RegMask;
308 if (CC == CallingConv::SwiftTail) {
309 if (SCS)
310 report_fatal_error(reason: "ShadowCallStack attribute not supported with swifttail");
311 return CSR_AArch64_AAPCS_SwiftTail_RegMask;
312 }
313 if (CC == CallingConv::PreserveMost)
314 return SCS ? CSR_AArch64_RT_MostRegs_SCS_RegMask
315 : CSR_AArch64_RT_MostRegs_RegMask;
316 if (CC == CallingConv::PreserveAll)
317 return SCS ? CSR_AArch64_RT_AllRegs_SCS_RegMask
318 : CSR_AArch64_RT_AllRegs_RegMask;
319
320 return SCS ? CSR_AArch64_AAPCS_SCS_RegMask : CSR_AArch64_AAPCS_RegMask;
321}
322
323const uint32_t *AArch64RegisterInfo::getCustomEHPadPreservedMask(
324 const MachineFunction &MF) const {
325 if (MF.getSubtarget<AArch64Subtarget>().isTargetLinux())
326 return CSR_AArch64_AAPCS_RegMask;
327
328 return nullptr;
329}
330
331const uint32_t *AArch64RegisterInfo::getTLSCallPreservedMask() const {
332 if (TT.isOSDarwin())
333 return CSR_Darwin_AArch64_TLS_RegMask;
334
335 assert(TT.isOSBinFormatELF() && "Invalid target");
336 return CSR_AArch64_TLS_ELF_RegMask;
337}
338
339void AArch64RegisterInfo::UpdateCustomCallPreservedMask(MachineFunction &MF,
340 const uint32_t **Mask) const {
341 uint32_t *UpdatedMask = MF.allocateRegMask();
342 unsigned RegMaskSize = MachineOperand::getRegMaskSize(NumRegs: getNumRegs());
343 memcpy(dest: UpdatedMask, src: *Mask, n: sizeof(UpdatedMask[0]) * RegMaskSize);
344
345 for (size_t i = 0; i < AArch64::GPR64commonRegClass.getNumRegs(); ++i) {
346 if (MF.getSubtarget<AArch64Subtarget>().isXRegCustomCalleeSaved(i)) {
347 for (MCPhysReg SubReg :
348 subregs_inclusive(Reg: AArch64::GPR64commonRegClass.getRegister(i))) {
349 // See TargetRegisterInfo::getCallPreservedMask for how to interpret the
350 // register mask.
351 UpdatedMask[SubReg / 32] |= 1u << (SubReg % 32);
352 }
353 }
354 }
355 *Mask = UpdatedMask;
356}
357
358const uint32_t *AArch64RegisterInfo::getSMStartStopCallPreservedMask() const {
359 return CSR_AArch64_SMStartStop_RegMask;
360}
361
362const uint32_t *
363AArch64RegisterInfo::SMEABISupportRoutinesCallPreservedMaskFromX0() const {
364 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask;
365}
366
367const uint32_t *AArch64RegisterInfo::getNoPreservedMask() const {
368 return CSR_AArch64_NoRegs_RegMask;
369}
370
371const uint32_t *
372AArch64RegisterInfo::getThisReturnPreservedMask(const MachineFunction &MF,
373 CallingConv::ID CC) const {
374 // This should return a register mask that is the same as that returned by
375 // getCallPreservedMask but that additionally preserves the register used for
376 // the first i64 argument (which must also be the register used to return a
377 // single i64 return value)
378 //
379 // In case that the calling convention does not use the same register for
380 // both, the function should return NULL (does not currently apply)
381 assert(CC != CallingConv::GHC && "should not be GHC calling convention.");
382 if (MF.getSubtarget<AArch64Subtarget>().isTargetDarwin())
383 return CSR_Darwin_AArch64_AAPCS_ThisReturn_RegMask;
384 return CSR_AArch64_AAPCS_ThisReturn_RegMask;
385}
386
387const uint32_t *AArch64RegisterInfo::getWindowsStackProbePreservedMask() const {
388 return CSR_AArch64_StackProbe_Windows_RegMask;
389}
390
391std::optional<std::string>
392AArch64RegisterInfo::explainReservedReg(const MachineFunction &MF,
393 MCRegister PhysReg) const {
394 if (hasBasePointer(MF) && MCRegisterInfo::regsOverlap(RegA: PhysReg, RegB: AArch64::X19))
395 return std::string("X19 is used as the frame base pointer register.");
396
397 if (MF.getSubtarget<AArch64Subtarget>().isWindowsArm64EC()) {
398 bool warn = false;
399 if (MCRegisterInfo::regsOverlap(RegA: PhysReg, RegB: AArch64::X13) ||
400 MCRegisterInfo::regsOverlap(RegA: PhysReg, RegB: AArch64::X14) ||
401 MCRegisterInfo::regsOverlap(RegA: PhysReg, RegB: AArch64::X23) ||
402 MCRegisterInfo::regsOverlap(RegA: PhysReg, RegB: AArch64::X24) ||
403 MCRegisterInfo::regsOverlap(RegA: PhysReg, RegB: AArch64::X28))
404 warn = true;
405
406 for (unsigned i = AArch64::B16; i <= AArch64::B31; ++i)
407 if (MCRegisterInfo::regsOverlap(RegA: PhysReg, RegB: i))
408 warn = true;
409
410 if (warn)
411 return std::string(AArch64InstPrinter::getRegisterName(Reg: PhysReg)) +
412 " is clobbered by asynchronous signals when using Arm64EC.";
413 }
414
415 return {};
416}
417
418BitVector
419AArch64RegisterInfo::getStrictlyReservedRegs(const MachineFunction &MF) const {
420 const AArch64FrameLowering *TFI = getFrameLowering(MF);
421
422 BitVector Reserved(getNumRegs());
423 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::WSP);
424 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::WZR);
425
426 if (TFI->isFPReserved(MF))
427 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W29);
428
429 if (MF.getSubtarget<AArch64Subtarget>().isWindowsArm64EC()) {
430 // x13, x14, x23, x24, x28, and v16-v31 are clobbered by asynchronous
431 // signals, so we can't ever use them.
432 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W13);
433 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W14);
434 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W23);
435 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W24);
436 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W28);
437 for (unsigned i = AArch64::B16; i <= AArch64::B31; ++i)
438 markSuperRegs(RegisterSet&: Reserved, Reg: i);
439 }
440
441 for (size_t i = 0; i < AArch64::GPR32commonRegClass.getNumRegs(); ++i) {
442 if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(i))
443 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::GPR32commonRegClass.getRegister(i));
444 }
445
446 if (hasBasePointer(MF))
447 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W19);
448
449 // SLH uses register W16/X16 as the taint register.
450 if (MF.getFunction().hasFnAttribute(Kind: Attribute::SpeculativeLoadHardening))
451 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W16);
452
453 // FFR is modelled as global state that cannot be allocated.
454 if (MF.getSubtarget<AArch64Subtarget>().hasSVE())
455 Reserved.set(AArch64::FFR);
456
457 // SME tiles are not allocatable.
458 if (MF.getSubtarget<AArch64Subtarget>().hasSME()) {
459 for (MCPhysReg SubReg : subregs_inclusive(Reg: AArch64::ZA))
460 Reserved.set(SubReg);
461 }
462
463 // VG cannot be allocated
464 Reserved.set(AArch64::VG);
465
466 if (MF.getSubtarget<AArch64Subtarget>().hasSME2()) {
467 for (MCSubRegIterator SubReg(AArch64::ZT0, this, /*self=*/true);
468 SubReg.isValid(); ++SubReg)
469 Reserved.set(*SubReg);
470 }
471
472 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::FPCR);
473 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::FPMR);
474 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::FPSR);
475
476 if (MF.getFunction().getCallingConv() == CallingConv::GRAAL) {
477 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::X27);
478 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::X28);
479 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W27);
480 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W28);
481 }
482
483 assert(checkAllSuperRegsMarked(Reserved));
484
485 // Add _HI registers after checkAllSuperRegsMarked as this check otherwise
486 // becomes considerably more expensive.
487 Reserved.set(AArch64::WSP_HI);
488 Reserved.set(AArch64::WZR_HI);
489 static_assert(AArch64::W30_HI - AArch64::W0_HI == 30,
490 "Unexpected order of registers");
491 Reserved.set(I: AArch64::W0_HI, E: AArch64::W30_HI + 1);
492 static_assert(AArch64::B31_HI - AArch64::B0_HI == 31,
493 "Unexpected order of registers");
494 Reserved.set(I: AArch64::B0_HI, E: AArch64::B31_HI + 1);
495 static_assert(AArch64::H31_HI - AArch64::H0_HI == 31,
496 "Unexpected order of registers");
497 Reserved.set(I: AArch64::H0_HI, E: AArch64::H31_HI + 1);
498 static_assert(AArch64::S31_HI - AArch64::S0_HI == 31,
499 "Unexpected order of registers");
500 Reserved.set(I: AArch64::S0_HI, E: AArch64::S31_HI + 1);
501 static_assert(AArch64::D31_HI - AArch64::D0_HI == 31,
502 "Unexpected order of registers");
503 Reserved.set(I: AArch64::D0_HI, E: AArch64::D31_HI + 1);
504 static_assert(AArch64::Q31_HI - AArch64::Q0_HI == 31,
505 "Unexpected order of registers");
506 Reserved.set(I: AArch64::Q0_HI, E: AArch64::Q31_HI + 1);
507
508 return Reserved;
509}
510
511BitVector
512AArch64RegisterInfo::getUserReservedRegs(const MachineFunction &MF) const {
513 BitVector Reserved(getNumRegs());
514 for (size_t i = 0; i < AArch64::GPR32commonRegClass.getNumRegs(); ++i) {
515 // ReserveXRegister is set for registers manually reserved
516 // through +reserve-x#i.
517 if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(i))
518 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::GPR32commonRegClass.getRegister(i));
519 }
520 return Reserved;
521}
522
523BitVector
524AArch64RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
525 BitVector Reserved(getNumRegs());
526 for (size_t i = 0; i < AArch64::GPR32commonRegClass.getNumRegs(); ++i) {
527 if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReservedForRA(i))
528 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::GPR32commonRegClass.getRegister(i));
529 }
530
531 if (MF.getSubtarget<AArch64Subtarget>().isLRReservedForRA()) {
532 // In order to prevent the register allocator from using LR, we need to
533 // mark it as reserved. However we don't want to keep it reserved throughout
534 // the pipeline since it prevents other infrastructure from reasoning about
535 // it's liveness. We use the NoVRegs property instead of IsSSA because
536 // IsSSA is removed before VirtRegRewriter runs.
537 if (!MF.getProperties().hasNoVRegs())
538 // Reserve LR (X30) by marking from its subregister W30 because otherwise
539 // the register allocator could clobber the subregister.
540 markSuperRegs(RegisterSet&: Reserved, Reg: AArch64::W30);
541 }
542
543 assert(checkAllSuperRegsMarked(Reserved));
544
545 // Handle strictlyReservedRegs separately to avoid re-evaluating the assert,
546 // which becomes considerably expensive when considering the _HI registers.
547 Reserved |= getStrictlyReservedRegs(MF);
548
549 return Reserved;
550}
551
552bool AArch64RegisterInfo::isReservedReg(const MachineFunction &MF,
553 MCRegister Reg) const {
554 return getReservedRegs(MF)[Reg];
555}
556
557bool AArch64RegisterInfo::isUserReservedReg(const MachineFunction &MF,
558 MCRegister Reg) const {
559 return getUserReservedRegs(MF)[Reg];
560}
561
562bool AArch64RegisterInfo::isStrictlyReservedReg(const MachineFunction &MF,
563 MCRegister Reg) const {
564 return getStrictlyReservedRegs(MF)[Reg];
565}
566
567bool AArch64RegisterInfo::isAnyArgRegReserved(const MachineFunction &MF) const {
568 for (size_t i = 0; i < 8; ++i) {
569 if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(i))
570 return true;
571 }
572 return false;
573}
574
575void AArch64RegisterInfo::emitReservedArgRegCallError(
576 const MachineFunction &MF) const {
577 const Function &F = MF.getFunction();
578 F.getContext().diagnose(DI: DiagnosticInfoUnsupported{F, ("AArch64 doesn't support"
579 " function calls if any of the argument registers is reserved.")});
580}
581
582bool AArch64RegisterInfo::isAsmClobberable(const MachineFunction &MF,
583 MCRegister PhysReg) const {
584 // SLH uses register X16 as the taint register but it will fallback to a different
585 // method if the user clobbers it. So X16 is not reserved for inline asm but is
586 // for normal codegen.
587 if (MF.getFunction().hasFnAttribute(Kind: Attribute::SpeculativeLoadHardening) &&
588 MCRegisterInfo::regsOverlap(RegA: PhysReg, RegB: AArch64::X16))
589 return true;
590
591 // ZA/ZT0 registers are reserved but may be permitted in the clobber list.
592 if (PhysReg == AArch64::ZA || PhysReg == AArch64::ZT0)
593 return true;
594
595 return !isReservedReg(MF, Reg: PhysReg);
596}
597
598const TargetRegisterClass *
599AArch64RegisterInfo::getCrossCopyRegClass(const TargetRegisterClass *RC) const {
600 if (RC == &AArch64::CCRRegClass)
601 return &AArch64::GPR64RegClass; // Only MSR & MRS copy NZCV.
602 return RC;
603}
604
605MCRegister AArch64RegisterInfo::getBaseRegister() const { return AArch64::X19; }
606
607bool AArch64RegisterInfo::hasBasePointer(const MachineFunction &MF) const {
608 const MachineFrameInfo &MFI = MF.getFrameInfo();
609
610 // In the presence of variable sized objects or funclets, if the fixed stack
611 // size is large enough that referencing from the FP won't result in things
612 // being in range relatively often, we can use a base pointer to allow access
613 // from the other direction like the SP normally works.
614 //
615 // Furthermore, if both variable sized objects are present, and the
616 // stack needs to be dynamically re-aligned, the base pointer is the only
617 // reliable way to reference the locals.
618 if (MFI.hasVarSizedObjects() || MF.hasEHFunclets()) {
619 if (hasStackRealignment(MF))
620 return true;
621
622 auto &ST = MF.getSubtarget<AArch64Subtarget>();
623 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
624 if (ST.hasSVE() || ST.isStreaming()) {
625 // Frames that have variable sized objects and scalable SVE objects,
626 // should always use a basepointer.
627 if (!AFI->hasCalculatedStackSizeSVE() || AFI->hasSVEStackSize())
628 return true;
629 }
630
631 // Frames with hazard padding can have a large offset between the frame
632 // pointer and GPR locals, which includes the emergency spill slot. If the
633 // emergency spill slot is not within range of the load/store instructions
634 // (which have a signed 9-bit range), we will fail to compile if it is used.
635 // Since hasBasePointer() is called before we know if we have hazard padding
636 // or an emergency spill slot we need to enable the basepointer
637 // conservatively.
638 if (ST.getStreamingHazardSize() &&
639 !AFI->getSMEFnAttrs().hasNonStreamingInterfaceAndBody()) {
640 return true;
641 }
642
643 // Conservatively estimate whether the negative offset from the frame
644 // pointer will be sufficient to reach. If a function has a smallish
645 // frame, it's less likely to have lots of spills and callee saved
646 // space, so it's all more likely to be within range of the frame pointer.
647 // If it's wrong, we'll materialize the constant and still get to the
648 // object; it's just suboptimal. Negative offsets use the unscaled
649 // load/store instructions, which have a 9-bit signed immediate.
650 return MFI.getLocalFrameSize() >= 256;
651 }
652
653 return false;
654}
655
656bool AArch64RegisterInfo::isArgumentRegister(const MachineFunction &MF,
657 MCRegister Reg) const {
658 CallingConv::ID CC = MF.getFunction().getCallingConv();
659 const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
660 bool IsVarArg = STI.isCallingConvWin64(CC: MF.getFunction().getCallingConv(),
661 IsVarArg: MF.getFunction().isVarArg());
662
663 auto HasReg = [](ArrayRef<MCRegister> RegList, MCRegister Reg) {
664 return llvm::is_contained(Range&: RegList, Element: Reg);
665 };
666
667 switch (CC) {
668 default:
669 report_fatal_error(reason: "Unsupported calling convention.");
670 case CallingConv::GHC:
671 return HasReg(CC_AArch64_GHC_ArgRegs, Reg);
672 case CallingConv::PreserveNone:
673 if (!MF.getFunction().isVarArg())
674 return HasReg(CC_AArch64_Preserve_None_ArgRegs, Reg);
675 [[fallthrough]];
676 case CallingConv::C:
677 case CallingConv::Fast:
678 case CallingConv::PreserveMost:
679 case CallingConv::PreserveAll:
680 case CallingConv::CXX_FAST_TLS:
681 case CallingConv::Swift:
682 case CallingConv::SwiftTail:
683 case CallingConv::Tail:
684 if (STI.isTargetWindowsOrUEFI()) {
685 if (IsVarArg)
686 return HasReg(CC_AArch64_Win64_VarArg_ArgRegs, Reg);
687 switch (CC) {
688 default:
689 return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg);
690 case CallingConv::Swift:
691 case CallingConv::SwiftTail:
692 return HasReg(CC_AArch64_Win64PCS_Swift_ArgRegs, Reg) ||
693 HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg);
694 }
695 }
696 if (!STI.isTargetDarwin()) {
697 switch (CC) {
698 default:
699 return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg);
700 case CallingConv::Swift:
701 case CallingConv::SwiftTail:
702 return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg) ||
703 HasReg(CC_AArch64_AAPCS_Swift_ArgRegs, Reg);
704 }
705 }
706 if (!IsVarArg) {
707 switch (CC) {
708 default:
709 return HasReg(CC_AArch64_DarwinPCS_ArgRegs, Reg);
710 case CallingConv::Swift:
711 case CallingConv::SwiftTail:
712 return HasReg(CC_AArch64_DarwinPCS_ArgRegs, Reg) ||
713 HasReg(CC_AArch64_DarwinPCS_Swift_ArgRegs, Reg);
714 }
715 }
716 if (STI.isTargetILP32())
717 return HasReg(CC_AArch64_DarwinPCS_ILP32_VarArg_ArgRegs, Reg);
718 return HasReg(CC_AArch64_DarwinPCS_VarArg_ArgRegs, Reg);
719 case CallingConv::Win64:
720 if (IsVarArg)
721 HasReg(CC_AArch64_Win64_VarArg_ArgRegs, Reg);
722 return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg);
723 case CallingConv::CFGuard_Check:
724 return HasReg(CC_AArch64_Win64_CFGuard_Check_ArgRegs, Reg);
725 case CallingConv::AArch64_VectorCall:
726 case CallingConv::AArch64_SVE_VectorCall:
727 case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0:
728 case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1:
729 case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2:
730 if (STI.isTargetWindowsOrUEFI())
731 return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg);
732 return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg);
733 }
734}
735
736Register
737AArch64RegisterInfo::getFrameRegister(const MachineFunction &MF) const {
738 const AArch64FrameLowering *TFI = getFrameLowering(MF);
739 return TFI->hasFP(MF) ? AArch64::FP : AArch64::SP;
740}
741
742bool AArch64RegisterInfo::requiresRegisterScavenging(
743 const MachineFunction &MF) const {
744 return true;
745}
746
747bool AArch64RegisterInfo::requiresVirtualBaseRegisters(
748 const MachineFunction &MF) const {
749 return MF.getTarget().getOptLevel() != CodeGenOptLevel::None;
750}
751
752bool
753AArch64RegisterInfo::useFPForScavengingIndex(const MachineFunction &MF) const {
754 // This function indicates whether the emergency spillslot should be placed
755 // close to the beginning of the stackframe (closer to FP) or the end
756 // (closer to SP).
757 //
758 // The beginning works most reliably if we have a frame pointer.
759 // In the presence of any non-constant space between FP and locals,
760 // (e.g. in case of stack realignment or a scalable SVE area), it is
761 // better to use SP or BP.
762 const AArch64FrameLowering &TFI = *getFrameLowering(MF);
763 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
764 assert((!MF.getSubtarget<AArch64Subtarget>().hasSVE() ||
765 AFI->hasCalculatedStackSizeSVE()) &&
766 "Expected SVE area to be calculated by this point");
767 return TFI.hasFP(MF) && !hasStackRealignment(MF) && !AFI->hasSVEStackSize() &&
768 !AFI->hasStackHazardSlotIndex();
769}
770
771bool AArch64RegisterInfo::requiresFrameIndexScavenging(
772 const MachineFunction &MF) const {
773 return true;
774}
775
776bool
777AArch64RegisterInfo::cannotEliminateFrame(const MachineFunction &MF) const {
778 const MachineFrameInfo &MFI = MF.getFrameInfo();
779 if (MF.disableFramePointerElim() && MFI.adjustsStack())
780 return true;
781 return MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken();
782}
783
784/// needsFrameBaseReg - Returns true if the instruction's frame index
785/// reference would be better served by a base register other than FP
786/// or SP. Used by LocalStackFrameAllocation to determine which frame index
787/// references it should create new base registers for.
788bool AArch64RegisterInfo::needsFrameBaseReg(MachineInstr *MI,
789 int64_t Offset) const {
790 for (unsigned i = 0; !MI->getOperand(i).isFI(); ++i)
791 assert(i < MI->getNumOperands() &&
792 "Instr doesn't have FrameIndex operand!");
793
794 // It's the load/store FI references that cause issues, as it can be difficult
795 // to materialize the offset if it won't fit in the literal field. Estimate
796 // based on the size of the local frame and some conservative assumptions
797 // about the rest of the stack frame (note, this is pre-regalloc, so
798 // we don't know everything for certain yet) whether this offset is likely
799 // to be out of range of the immediate. Return true if so.
800
801 // We only generate virtual base registers for loads and stores, so
802 // return false for everything else.
803 if (!MI->mayLoad() && !MI->mayStore())
804 return false;
805
806 // Without a virtual base register, if the function has variable sized
807 // objects, all fixed-size local references will be via the frame pointer,
808 // Approximate the offset and see if it's legal for the instruction.
809 // Note that the incoming offset is based on the SP value at function entry,
810 // so it'll be negative.
811 MachineFunction &MF = *MI->getParent()->getParent();
812 const AArch64FrameLowering *TFI = getFrameLowering(MF);
813 MachineFrameInfo &MFI = MF.getFrameInfo();
814
815 // Estimate an offset from the frame pointer.
816 // Conservatively assume all GPR callee-saved registers get pushed.
817 // FP, LR, X19-X28, D8-D15. 64-bits each.
818 int64_t FPOffset = Offset - 16 * 20;
819 // Estimate an offset from the stack pointer.
820 // The incoming offset is relating to the SP at the start of the function,
821 // but when we access the local it'll be relative to the SP after local
822 // allocation, so adjust our SP-relative offset by that allocation size.
823 Offset += MFI.getLocalFrameSize();
824 // Assume that we'll have at least some spill slots allocated.
825 // FIXME: This is a total SWAG number. We should run some statistics
826 // and pick a real one.
827 Offset += 128; // 128 bytes of spill slots
828
829 // If there is a frame pointer, try using it.
830 // The FP is only available if there is no dynamic realignment. We
831 // don't know for sure yet whether we'll need that, so we guess based
832 // on whether there are any local variables that would trigger it.
833 if (TFI->hasFP(MF) && isFrameOffsetLegal(MI, BaseReg: AArch64::FP, Offset: FPOffset))
834 return false;
835
836 // If we can reference via the stack pointer or base pointer, try that.
837 // FIXME: This (and the code that resolves the references) can be improved
838 // to only disallow SP relative references in the live range of
839 // the VLA(s). In practice, it's unclear how much difference that
840 // would make, but it may be worth doing.
841 if (isFrameOffsetLegal(MI, BaseReg: AArch64::SP, Offset))
842 return false;
843
844 // If even offset 0 is illegal, we don't want a virtual base register.
845 if (!isFrameOffsetLegal(MI, BaseReg: AArch64::SP, Offset: 0))
846 return false;
847
848 // The offset likely isn't legal; we want to allocate a virtual base register.
849 return true;
850}
851
852bool AArch64RegisterInfo::isFrameOffsetLegal(const MachineInstr *MI,
853 Register BaseReg,
854 int64_t Offset) const {
855 assert(MI && "Unable to get the legal offset for nil instruction.");
856 StackOffset SaveOffset = StackOffset::getFixed(Fixed: Offset);
857 return isAArch64FrameOffsetLegal(MI: *MI, Offset&: SaveOffset) & AArch64FrameOffsetIsLegal;
858}
859
860/// Insert defining instruction(s) for BaseReg to be a pointer to FrameIdx
861/// at the beginning of the basic block.
862Register
863AArch64RegisterInfo::materializeFrameBaseRegister(MachineBasicBlock *MBB,
864 int FrameIdx,
865 int64_t Offset) const {
866 MachineBasicBlock::iterator Ins = MBB->begin();
867 DebugLoc DL; // Defaults to "unknown"
868 if (Ins != MBB->end())
869 DL = Ins->getDebugLoc();
870 const MachineFunction &MF = *MBB->getParent();
871 const AArch64InstrInfo *TII =
872 MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
873 const MCInstrDesc &MCID = TII->get(Opcode: AArch64::ADDXri);
874 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
875 Register BaseReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR64spRegClass);
876 MRI.constrainRegClass(Reg: BaseReg, RC: TII->getRegClass(MCID, OpNum: 0));
877 unsigned Shifter = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0);
878
879 BuildMI(BB&: *MBB, I: Ins, MIMD: DL, MCID, DestReg: BaseReg)
880 .addFrameIndex(Idx: FrameIdx)
881 .addImm(Val: Offset)
882 .addImm(Val: Shifter);
883
884 return BaseReg;
885}
886
887void AArch64RegisterInfo::resolveFrameIndex(MachineInstr &MI, Register BaseReg,
888 int64_t Offset) const {
889 // ARM doesn't need the general 64-bit offsets
890 StackOffset Off = StackOffset::getFixed(Fixed: Offset);
891
892 unsigned i = 0;
893 while (!MI.getOperand(i).isFI()) {
894 ++i;
895 assert(i < MI.getNumOperands() && "Instr doesn't have FrameIndex operand!");
896 }
897
898 const MachineFunction *MF = MI.getParent()->getParent();
899 const AArch64InstrInfo *TII =
900 MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
901 bool Done = rewriteAArch64FrameIndex(MI, FrameRegIdx: i, FrameReg: BaseReg, Offset&: Off, TII);
902 assert(Done && "Unable to resolve frame index!");
903 (void)Done;
904}
905
906// Create a scratch register for the frame index elimination in an instruction.
907// This function has special handling of stack tagging loop pseudos, in which
908// case it can also change the instruction opcode.
909static Register
910createScratchRegisterForInstruction(MachineInstr &MI, unsigned FIOperandNum,
911 const AArch64InstrInfo *TII) {
912 // ST*Gloop have a reserved scratch register in operand 1. Use it, and also
913 // replace the instruction with the writeback variant because it will now
914 // satisfy the operand constraints for it.
915 Register ScratchReg;
916 if (MI.getOpcode() == AArch64::STGloop ||
917 MI.getOpcode() == AArch64::STZGloop) {
918 assert(FIOperandNum == 3 &&
919 "Wrong frame index operand for STGloop/STZGloop");
920 unsigned Op = MI.getOpcode() == AArch64::STGloop ? AArch64::STGloop_wback
921 : AArch64::STZGloop_wback;
922 ScratchReg = MI.getOperand(i: 1).getReg();
923 MI.getOperand(i: 3).ChangeToRegister(Reg: ScratchReg, isDef: false, isImp: false, isKill: true);
924 MI.setDesc(TII->get(Opcode: Op));
925 MI.tieOperands(DefIdx: 1, UseIdx: 3);
926 } else {
927 ScratchReg =
928 MI.getMF()->getRegInfo().createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
929 MI.getOperand(i: FIOperandNum)
930 .ChangeToRegister(Reg: ScratchReg, isDef: false, isImp: false, isKill: true);
931 }
932 return ScratchReg;
933}
934
935void AArch64RegisterInfo::getOffsetOpcodes(
936 const StackOffset &Offset, SmallVectorImpl<uint64_t> &Ops) const {
937 // The smallest scalable element supported by scaled SVE addressing
938 // modes are predicates, which are 2 scalable bytes in size. So the scalable
939 // byte offset must always be a multiple of 2.
940 assert(Offset.getScalable() % 2 == 0 && "Invalid frame offset");
941
942 // Add fixed-sized offset using existing DIExpression interface.
943 DIExpression::appendOffset(Ops, Offset: Offset.getFixed());
944
945 unsigned VG = getDwarfRegNum(Reg: AArch64::VG, isEH: true);
946 int64_t VGSized = Offset.getScalable() / 2;
947 if (VGSized > 0) {
948 Ops.push_back(Elt: dwarf::DW_OP_constu);
949 Ops.push_back(Elt: VGSized);
950 Ops.append(IL: {dwarf::DW_OP_bregx, VG, 0ULL});
951 Ops.push_back(Elt: dwarf::DW_OP_mul);
952 Ops.push_back(Elt: dwarf::DW_OP_plus);
953 } else if (VGSized < 0) {
954 Ops.push_back(Elt: dwarf::DW_OP_constu);
955 Ops.push_back(Elt: -VGSized);
956 Ops.append(IL: {dwarf::DW_OP_bregx, VG, 0ULL});
957 Ops.push_back(Elt: dwarf::DW_OP_mul);
958 Ops.push_back(Elt: dwarf::DW_OP_minus);
959 }
960}
961
962bool AArch64RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
963 int SPAdj, unsigned FIOperandNum,
964 RegScavenger *RS) const {
965 assert(SPAdj == 0 && "Unexpected");
966
967 MachineInstr &MI = *II;
968 MachineBasicBlock &MBB = *MI.getParent();
969 MachineFunction &MF = *MBB.getParent();
970 const MachineFrameInfo &MFI = MF.getFrameInfo();
971 const AArch64InstrInfo *TII =
972 MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
973 const AArch64FrameLowering *TFI = getFrameLowering(MF);
974 int FrameIndex = MI.getOperand(i: FIOperandNum).getIndex();
975 bool Tagged =
976 MI.getOperand(i: FIOperandNum).getTargetFlags() & AArch64II::MO_TAGGED;
977 Register FrameReg;
978
979 // Special handling of dbg_value, stackmap patchpoint statepoint instructions.
980 if (MI.getOpcode() == TargetOpcode::STACKMAP ||
981 MI.getOpcode() == TargetOpcode::PATCHPOINT ||
982 MI.getOpcode() == TargetOpcode::STATEPOINT) {
983 StackOffset Offset =
984 TFI->resolveFrameIndexReference(MF, FI: FrameIndex, FrameReg,
985 /*PreferFP=*/true,
986 /*ForSimm=*/false);
987 Offset += StackOffset::getFixed(Fixed: MI.getOperand(i: FIOperandNum + 1).getImm());
988 MI.getOperand(i: FIOperandNum).ChangeToRegister(Reg: FrameReg, isDef: false /*isDef*/);
989 MI.getOperand(i: FIOperandNum + 1).ChangeToImmediate(ImmVal: Offset.getFixed());
990 return false;
991 }
992
993 if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE) {
994 MachineOperand &FI = MI.getOperand(i: FIOperandNum);
995 StackOffset Offset = TFI->getNonLocalFrameIndexReference(MF, FI: FrameIndex);
996 assert(!Offset.getScalable() &&
997 "Frame offsets with a scalable component are not supported");
998 FI.ChangeToImmediate(ImmVal: Offset.getFixed());
999 return false;
1000 }
1001
1002 StackOffset Offset;
1003 if (MI.getOpcode() == AArch64::TAGPstack) {
1004 // TAGPstack must use the virtual frame register in its 3rd operand.
1005 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1006 FrameReg = MI.getOperand(i: 3).getReg();
1007 Offset = StackOffset::getFixed(Fixed: MFI.getObjectOffset(ObjectIdx: FrameIndex) +
1008 AFI->getTaggedBasePointerOffset());
1009 } else if (Tagged) {
1010 StackOffset SPOffset = StackOffset::getFixed(
1011 Fixed: MFI.getObjectOffset(ObjectIdx: FrameIndex) + (int64_t)MFI.getStackSize());
1012 if (MFI.hasVarSizedObjects() ||
1013 isAArch64FrameOffsetLegal(MI, Offset&: SPOffset, OutUseUnscaledOp: nullptr, OutUnscaledOp: nullptr, EmittableOffset: nullptr) !=
1014 (AArch64FrameOffsetCanUpdate | AArch64FrameOffsetIsLegal)) {
1015 // Can't update to SP + offset in place. Precalculate the tagged pointer
1016 // in a scratch register.
1017 Offset = TFI->resolveFrameIndexReference(
1018 MF, FI: FrameIndex, FrameReg, /*PreferFP=*/false, /*ForSimm=*/true);
1019 Register ScratchReg =
1020 MF.getRegInfo().createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
1021 emitFrameOffset(MBB, MBBI: II, DL: MI.getDebugLoc(), DestReg: ScratchReg, SrcReg: FrameReg, Offset,
1022 TII);
1023 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::LDG), DestReg: ScratchReg)
1024 .addReg(RegNo: ScratchReg)
1025 .addReg(RegNo: ScratchReg)
1026 .addImm(Val: 0);
1027 MI.getOperand(i: FIOperandNum)
1028 .ChangeToRegister(Reg: ScratchReg, isDef: false, isImp: false, isKill: true);
1029 return false;
1030 }
1031 FrameReg = AArch64::SP;
1032 Offset = StackOffset::getFixed(Fixed: MFI.getObjectOffset(ObjectIdx: FrameIndex) +
1033 (int64_t)MFI.getStackSize());
1034 } else {
1035 Offset = TFI->resolveFrameIndexReference(
1036 MF, FI: FrameIndex, FrameReg, /*PreferFP=*/false, /*ForSimm=*/true);
1037 }
1038
1039 // Modify MI as necessary to handle as much of 'Offset' as possible
1040 if (rewriteAArch64FrameIndex(MI, FrameRegIdx: FIOperandNum, FrameReg, Offset, TII))
1041 return true;
1042
1043 assert((!RS || !RS->isScavengingFrameIndex(FrameIndex)) &&
1044 "Emergency spill slot is out of reach");
1045
1046 // If we get here, the immediate doesn't fit into the instruction. We folded
1047 // as much as possible above. Handle the rest, providing a register that is
1048 // SP+LargeImm.
1049 Register ScratchReg =
1050 createScratchRegisterForInstruction(MI, FIOperandNum, TII);
1051 emitFrameOffset(MBB, MBBI: II, DL: MI.getDebugLoc(), DestReg: ScratchReg, SrcReg: FrameReg, Offset, TII);
1052 return false;
1053}
1054
1055unsigned AArch64RegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
1056 MachineFunction &MF) const {
1057 const AArch64FrameLowering *TFI = getFrameLowering(MF);
1058
1059 switch (RC->getID()) {
1060 default:
1061 return 0;
1062 case AArch64::GPR32RegClassID:
1063 case AArch64::GPR32spRegClassID:
1064 case AArch64::GPR32allRegClassID:
1065 case AArch64::GPR64spRegClassID:
1066 case AArch64::GPR64allRegClassID:
1067 case AArch64::GPR64RegClassID:
1068 case AArch64::GPR32commonRegClassID:
1069 case AArch64::GPR64commonRegClassID:
1070 return 32 - 1 // XZR/SP
1071 - (TFI->hasFP(MF) || TT.isOSDarwin()) // FP
1072 - MF.getSubtarget<AArch64Subtarget>().getNumXRegisterReserved()
1073 - hasBasePointer(MF); // X19
1074 case AArch64::FPR8RegClassID:
1075 case AArch64::FPR16RegClassID:
1076 case AArch64::FPR32RegClassID:
1077 case AArch64::FPR64RegClassID:
1078 case AArch64::FPR128RegClassID:
1079 return 32;
1080
1081 case AArch64::MatrixIndexGPR32_8_11RegClassID:
1082 case AArch64::MatrixIndexGPR32_12_15RegClassID:
1083 return 4;
1084
1085 case AArch64::DDRegClassID:
1086 case AArch64::DDDRegClassID:
1087 case AArch64::DDDDRegClassID:
1088 case AArch64::QQRegClassID:
1089 case AArch64::QQQRegClassID:
1090 case AArch64::QQQQRegClassID:
1091 return 32;
1092
1093 case AArch64::FPR128_loRegClassID:
1094 case AArch64::FPR64_loRegClassID:
1095 case AArch64::FPR16_loRegClassID:
1096 return 16;
1097 case AArch64::FPR128_0to7RegClassID:
1098 return 8;
1099 }
1100}
1101
1102static bool HandleDestructivePredicateHint(
1103 Register VirtReg, ArrayRef<MCPhysReg> Order,
1104 SmallSetVector<MCPhysReg, 16> &Hints, const VirtRegMap *VRM,
1105 const MachineRegisterInfo &MRI, const TargetInstrInfo &TII,
1106 const AArch64Subtarget &ST, const LiveRegMatrix *Matrix) {
1107 const TargetRegisterClass *RegRC = MRI.getRegClass(Reg: VirtReg);
1108 if (!ST.useDistinctPredicateDstReg() ||
1109 !AArch64::PPRRegClass.hasSubClassEq(RC: RegRC) || !MRI.hasOneDef(RegNo: VirtReg) ||
1110 Order.size() < 2)
1111 return false;
1112
1113 const MachineInstr *DefInst = MRI.getOneDef(Reg: VirtReg)->getParent();
1114 if ((TII.get(Opcode: DefInst->getOpcode()).TSFlags &
1115 AArch64::DestructiveInstTypeMask) != AArch64::DestructivePredicate)
1116 return false;
1117
1118 Register Op1Reg = DefInst->getOperand(i: 1).getReg();
1119 if (Op1Reg.isVirtual())
1120 Op1Reg = VRM->getPhys(virtReg: Op1Reg);
1121
1122 // If no register is allocated for the general-predicate, it's not yet
1123 // possible to choose a distinct register.
1124 if (!Op1Reg.isValid())
1125 return false;
1126
1127 // Move Op1Reg as the least preferred register.
1128 //
1129 // This might result in callee-save spills when the function takes/returns
1130 // arguments in SVE registers (i.e. needs to preserve p4-p15) and can't reuse
1131 // p0-p3. That's why we limit it to non-callee saved registers or to
1132 // callee-saved registers that have already been allocated for other uses in
1133 // the function.
1134 DenseSet<unsigned> CSRs;
1135 for (unsigned I = 0;; ++I) {
1136 Register R = MRI.getCalleeSavedRegs()[I];
1137 if (!R.isValid())
1138 break;
1139 if (AArch64::PPRRegClass.contains(Reg: R))
1140 CSRs.insert(V: R);
1141 }
1142
1143 Hints.insert_range(R&: Order);
1144 auto CanUseReg = [&](Register R) {
1145 return !CSRs.contains(V: R) || !MRI.def_empty(RegNo: R) || Matrix->isPhysRegUsed(PhysReg: R);
1146 };
1147 SmallVector<MCPhysReg, 16> SortedHints = Hints.takeVector();
1148 llvm::stable_sort(Range&: SortedHints, C: [&](Register A, Register B) {
1149 bool PrefA = (A != Op1Reg) && CanUseReg(A);
1150 bool PrefB = (B != Op1Reg) && CanUseReg(B);
1151 return PrefA && !PrefB;
1152 });
1153 Hints.insert_range(R&: SortedHints);
1154 return true;
1155}
1156
1157// We add regalloc hints for different cases:
1158// * Choosing a better destination operand for predicated SVE instructions
1159// where the inactive lanes are undef, by choosing a register that is not
1160// unique to the other operands of the instruction.
1161//
1162// * Improve register allocation for SME multi-vector instructions where we can
1163// benefit from the strided- and contiguous register multi-vector tuples.
1164//
1165// Here COPY_INTO_TRANSPOSED_TUPLE nodes are created to improve register
1166// allocation where a consecutive multi-vector tuple is constructed from the
1167// same indices of multiple strided loads. This may still result in
1168// unnecessary copies between the loads and the tuple. Here we try to return a
1169// hint to assign the contiguous ZPRMulReg starting at the same register as
1170// the first operand of the pseudo, which should be a subregister of the first
1171// strided load.
1172//
1173// For example, if the first strided load has been assigned $z16_z20_z24_z28
1174// and the operands of the pseudo are each accessing subregister zsub2, we
1175// should look through through Order to find a contiguous register which
1176// begins with $z24 (i.e. $z24_z25_z26_z27).
1177bool AArch64RegisterInfo::getRegAllocationHints(
1178 Register VirtReg, ArrayRef<MCPhysReg> Order,
1179 SmallSetVector<MCPhysReg, 16> &Hints, const MachineFunction &MF,
1180 const VirtRegMap *VRM, const LiveRegMatrix *Matrix) const {
1181 auto &ST = MF.getSubtarget<AArch64Subtarget>();
1182 const AArch64InstrInfo *TII =
1183 MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
1184 const MachineRegisterInfo &MRI = MF.getRegInfo();
1185
1186 bool ConsiderOnlyHints =
1187 TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, MF, VRM);
1188
1189 // For predicated SVE instructions where the inactive lanes are undef,
1190 // pick a destination register that is not unique to avoid introducing
1191 // a movprfx.
1192 const TargetRegisterClass *RegRC = MRI.getRegClass(Reg: VirtReg);
1193 if (AArch64::ZPRRegClass.hasSubClassEq(RC: RegRC)) {
1194 for (const MachineOperand &DefOp : MRI.def_operands(Reg: VirtReg)) {
1195 const MachineInstr &Def = *DefOp.getParent();
1196 if (DefOp.isImplicit() ||
1197 (TII->get(Opcode: Def.getOpcode()).TSFlags & AArch64::FalseLanesMask) !=
1198 AArch64::FalseLanesUndef)
1199 continue;
1200
1201 unsigned InstFlags =
1202 TII->get(Opcode: AArch64::getSVEPseudoMap(Opcode: Def.getOpcode())).TSFlags;
1203
1204 for (MCPhysReg R : Order) {
1205 auto AddHintIfSuitable = [&](MCPhysReg R,
1206 const MachineOperand &MO) -> bool {
1207 // R is a suitable register hint if R can reuse one of the other
1208 // source operands.
1209 MCPhysReg PhysReg = VRM->getPhys(virtReg: MO.getReg());
1210 if (PhysReg && MO.getSubReg())
1211 PhysReg = getSubReg(Reg: PhysReg, Idx: MO.getSubReg());
1212 if (PhysReg != R)
1213 return false;
1214 Hints.insert(X: R);
1215 return true;
1216 };
1217
1218 switch (InstFlags & AArch64::DestructiveInstTypeMask) {
1219 default:
1220 break;
1221 case AArch64::DestructiveTernaryCommWithRev:
1222 AddHintIfSuitable(R, Def.getOperand(i: 2)) ||
1223 AddHintIfSuitable(R, Def.getOperand(i: 3)) ||
1224 AddHintIfSuitable(R, Def.getOperand(i: 4));
1225 break;
1226 case AArch64::DestructiveBinaryComm:
1227 case AArch64::DestructiveBinaryCommWithRev:
1228 AddHintIfSuitable(R, Def.getOperand(i: 2)) ||
1229 AddHintIfSuitable(R, Def.getOperand(i: 3));
1230 break;
1231 case AArch64::DestructiveBinary:
1232 case AArch64::DestructiveBinaryImm:
1233 AddHintIfSuitable(R, Def.getOperand(i: 2));
1234 break;
1235 case AArch64::DestructiveUnaryPassthru:
1236 AddHintIfSuitable(R, Def.getOperand(i: 3));
1237 break;
1238 case AArch64::DestructiveBinaryImmUnpred:
1239 case AArch64::DestructiveBinaryShImmUnpred:
1240 AddHintIfSuitable(R, Def.getOperand(i: 1));
1241 break;
1242 }
1243 }
1244 }
1245
1246 if (Hints.size())
1247 return ConsiderOnlyHints;
1248 }
1249
1250 if (HandleDestructivePredicateHint(VirtReg, Order, Hints, VRM, MRI, TII: *TII, ST,
1251 Matrix))
1252 return ConsiderOnlyHints;
1253
1254 if (!ST.hasSME() || !ST.isStreaming())
1255 return ConsiderOnlyHints;
1256
1257 // The SVE calling convention preserves registers Z8-Z23. As a result, there
1258 // are no ZPR2Strided or ZPR4Strided registers that do not overlap with the
1259 // callee-saved registers and so by default these will be pushed to the back
1260 // of the allocation order for the ZPRStridedOrContiguous classes.
1261 // If any of the instructions which define VirtReg are used by the
1262 // COPY_INTO_TRANSPOSED_TUPLE pseudos, we want to favour reducing copy
1263 // instructions over reducing the number of clobbered callee-save registers,
1264 // so we add the strided registers as a hint.
1265 unsigned RegID = RegRC->getID();
1266 if (RegID == AArch64::ZPR2StridedOrContiguousRegClassID ||
1267 RegID == AArch64::ZPR4StridedOrContiguousRegClassID) {
1268
1269 // Look through uses of the register for COPY_INTO_TRANSPOSED_TUPLE.
1270 for (const MachineInstr &Use : MRI.use_nodbg_instructions(Reg: VirtReg)) {
1271 if (Use.getOpcode() != AArch64::COPY_INTO_TRANSPOSED_TUPLE)
1272 continue;
1273
1274 const MachineOperand &Src = Use.getOperand(i: 1);
1275 const MachineOperand &Dst = Use.getOperand(i: 0);
1276
1277 if (!Src.getSubReg() || !Dst.getSubReg())
1278 continue;
1279
1280 const TargetRegisterClass *StridedRC;
1281 switch (RegID) {
1282 case AArch64::ZPR2StridedOrContiguousRegClassID:
1283 StridedRC = &AArch64::ZPR2StridedRegClass;
1284 break;
1285 case AArch64::ZPR4StridedOrContiguousRegClassID:
1286 StridedRC = &AArch64::ZPR4StridedRegClass;
1287 break;
1288 default:
1289 llvm_unreachable("Unexpected RegID");
1290 }
1291
1292 SmallVector<MCPhysReg, 4> StridedOrder;
1293 for (MCPhysReg Reg : Order)
1294 if (StridedRC->contains(Reg))
1295 StridedOrder.push_back(Elt: Reg);
1296
1297 unsigned TupleSize = Use.getOperand(i: 2).getImm();
1298 unsigned TupIdx = Dst.getSubReg() - AArch64::zsub0;
1299
1300 unsigned TupleID = MRI.getRegClass(Reg: Dst.getReg())->getID();
1301 bool IsMulZPR = TupleID == AArch64::ZPR2Mul2RegClassID ||
1302 TupleID == AArch64::ZPR4Mul4RegClassID;
1303
1304 iterator_range Copies = MRI.def_instructions(Reg: Dst.getReg());
1305 MachineRegisterInfo::def_instr_iterator CopyWithAssignedSrc =
1306 llvm::find_if(Range&: Copies, P: [&](const MachineInstr &Def) {
1307 auto &Src = Def.getOperand(i: 1);
1308 return Def.getOpcode() == Use.getOpcode() &&
1309 VRM->hasPhys(virtReg: Src.getReg());
1310 });
1311
1312 // Example:
1313 //
1314 // When trying to find a suitable register allocation for VirtReg %v2 in:
1315 //
1316 // %v0:zpr2stridedorcontiguous = ld1 p0/z, [...]
1317 // %v1:zpr2stridedorcontiguous = ld1 p0/z, [...]
1318 // %v2:zpr2stridedorcontiguous = ld1 p0/z, [...]
1319 // %v3:zpr2stridedorcontiguous = ld1 p0/z, [...]
1320 // %v4.zsub0:zpr4mul4 = COPY_INTO_TRANSPOSED_TUPLE %v0:0
1321 // %v4.zsub1:zpr4mul4 = COPY_INTO_TRANSPOSED_TUPLE %v1:0
1322 // %v4.zsub2:zpr4mul4 = COPY_INTO_TRANSPOSED_TUPLE %v2:0
1323 // %v4.zsub3:zpr4mul4 = COPY_INTO_TRANSPOSED_TUPLE %v3:0
1324 //
1325 // One such suitable allocation would be:
1326 //
1327 // { z0, z8 } = ld1 p0/z, [...]
1328 // { z1, z9 } = ld1 p0/z, [...]
1329 // { z2, z10 } = ld1 p0/z, [...]
1330 // { z3, z11 } = ld1 p0/z, [...]
1331 // z0 = COPY_INTO_TRANSPOSED_TUPLE {z0, z8}:0
1332 // z1 = COPY_INTO_TRANSPOSED_TUPLE {z1, z9}:0
1333 // z2 = COPY_INTO_TRANSPOSED_TUPLE {z2, z10}:0
1334 // z3 = COPY_INTO_TRANSPOSED_TUPLE {z3, z11}:0
1335 //
1336 // Below we distinguish two cases when trying to find a register:
1337 // * None of the sources of the copies have been assigned a register yet.
1338 // In this case the code must ensure that there are at least TupleSize
1339 // free consecutive registers. If IsMulZPR is true, then the first of
1340 // registers must also be a multiple of TupleSize, e.g.
1341 // { z0, z1, z2, z3 } is valid but { z1, z2, z3, z5 } is not.
1342 // * One or more copies already have registers assigned to their sources,
1343 // which means only checking that a consecutive range of free tuple
1344 // registers exists which includes the assigned register.
1345 // e.g. in the example above, if { z0, z8 } is already allocated for
1346 // %v0, we just need to ensure that { z1, z9 }, { z2, z10 } and
1347 // { z3, z11 } are also free. If so, we add { z2, z10 }.
1348
1349 if (CopyWithAssignedSrc == Copies.end()) {
1350 // There are no registers already assigned to any of the pseudo
1351 // operands. Look for a valid starting register for the group.
1352 for (unsigned I = 0; I < StridedOrder.size(); ++I) {
1353 MCPhysReg Reg = StridedOrder[I];
1354
1355 // If the COPY_INTO_TRANSPOSED_TUPLE nodes use the ZPRMul classes, the
1356 // starting register of the first load should be a multiple of 2 or 4.
1357 unsigned SubRegIdx = Src.getSubReg();
1358 if (IsMulZPR &&
1359 (getSubReg(Reg, Idx: SubRegIdx) - AArch64::Z0) % TupleSize != TupIdx)
1360 continue;
1361
1362 // In the example above, if VirtReg is the third operand of the
1363 // tuple (%v2) and Reg == Z2_Z10, then we need to make sure that
1364 // Z0_Z8, Z1_Z9 and Z3_Z11 are also available.
1365 auto IsFreeConsecutiveReg = [&](unsigned I) {
1366 unsigned R = Reg - TupIdx + I;
1367 return StridedRC->contains(Reg: R) &&
1368 (I == 0 ||
1369 ((getSubReg(Reg: R, Idx: AArch64::zsub0) - AArch64::Z0) ==
1370 (getSubReg(Reg: R - 1, Idx: AArch64::zsub0) - AArch64::Z0) + 1)) &&
1371 !Matrix->isPhysRegUsed(PhysReg: R);
1372 };
1373 if (all_of(Range: seq(Begin: 0U, End: TupleSize), P: IsFreeConsecutiveReg))
1374 Hints.insert(X: Reg);
1375 }
1376 } else {
1377 // At least copy already has a physical register assigned to its source.
1378 // Find the starting sub-register of this and use it to work out the
1379 // correct strided register to suggest based on the current op index.
1380 MachineOperand &AssignedSrc = CopyWithAssignedSrc->getOperand(i: 1);
1381 MachineOperand &AssignedDst = CopyWithAssignedSrc->getOperand(i: 0);
1382
1383 if (!AssignedSrc.getSubReg() || !AssignedDst.getSubReg())
1384 continue;
1385
1386 unsigned AssignedTupIdx = AssignedDst.getSubReg() - AArch64::zsub0;
1387 MCPhysReg TargetStartReg =
1388 getSubReg(Reg: VRM->getPhys(virtReg: AssignedSrc.getReg()), Idx: AArch64::zsub0) +
1389 (TupIdx - AssignedTupIdx);
1390
1391 for (unsigned I = 0; I < StridedOrder.size(); ++I)
1392 if (getSubReg(Reg: StridedOrder[I], Idx: AArch64::zsub0) == TargetStartReg)
1393 Hints.insert(X: StridedOrder[I]);
1394 }
1395
1396 if (!Hints.empty())
1397 return ConsiderOnlyHints;
1398 }
1399 }
1400
1401 for (auto &Def : MRI.def_instructions(Reg: VirtReg)) {
1402 if (Def.getOpcode() != AArch64::COPY_INTO_TRANSPOSED_TUPLE)
1403 continue;
1404
1405 MachineOperand &Src = Def.getOperand(i: 1);
1406 MachineOperand &Dst = Def.getOperand(i: 0);
1407
1408 if (!Src.getSubReg() || !Dst.getSubReg())
1409 continue;
1410
1411 // FIXME: This is fragile. If we allocate a register to the Dst before Src,
1412 // our hints are won't have any effect... This is currently mitigated by
1413 // by trying to schedule copies immediately before their uses. This gives
1414 // them a short live range (so they're low priority to allocate).
1415 if (!VRM->hasPhys(virtReg: Src.getReg()))
1416 continue;
1417
1418 // Find the ZPR register mapped to the source of the copy.
1419 MCPhysReg SrcZPR = getSubReg(Reg: VRM->getPhys(virtReg: Src.getReg()), Idx: Src.getSubReg());
1420
1421 // Try to pick a tuple register for Dst with Src as a member.
1422 for (MCPhysReg R : Order) {
1423 if (getSubReg(Reg: R, Idx: Dst.getSubReg()) == SrcZPR)
1424 Hints.insert(X: R);
1425 }
1426 }
1427
1428 return ConsiderOnlyHints;
1429}
1430
1431unsigned AArch64RegisterInfo::getLocalAddressRegister(
1432 const MachineFunction &MF) const {
1433 const auto &MFI = MF.getFrameInfo();
1434 if (!MF.hasEHFunclets() && !MFI.hasVarSizedObjects())
1435 return AArch64::SP;
1436 else if (hasStackRealignment(MF))
1437 return getBaseRegister();
1438 return getFrameRegister(MF);
1439}
1440
1441/// SrcRC and DstRC will be morphed into NewRC if this returns true
1442bool AArch64RegisterInfo::shouldCoalesce(
1443 MachineInstr *MI, const TargetRegisterClass *SrcRC, unsigned SubReg,
1444 const TargetRegisterClass *DstRC, unsigned DstSubReg,
1445 const TargetRegisterClass *NewRC, LiveIntervals &LIS) const {
1446 MachineFunction &MF = *MI->getMF();
1447 MachineRegisterInfo &MRI = MF.getRegInfo();
1448
1449 if (MI->isSubregToReg() && MRI.subRegLivenessEnabled() &&
1450 !MF.getSubtarget<AArch64Subtarget>().enableSRLTSubregToRegMitigation())
1451 return false;
1452
1453 if (MI->isCopy() &&
1454 ((DstRC->getID() == AArch64::GPR64RegClassID) ||
1455 (DstRC->getID() == AArch64::GPR64commonRegClassID)) &&
1456 MI->getOperand(i: 0).getSubReg() && MI->getOperand(i: 1).getSubReg())
1457 // Do not coalesce in the case of a 32-bit subregister copy
1458 // which implements a 32 to 64 bit zero extension
1459 // which relies on the upper 32 bits being zeroed.
1460 return false;
1461
1462 auto IsCoalescerBarrier = [](const MachineInstr &MI) {
1463 switch (MI.getOpcode()) {
1464 case AArch64::COALESCER_BARRIER_FPR16:
1465 case AArch64::COALESCER_BARRIER_FPR32:
1466 case AArch64::COALESCER_BARRIER_FPR64:
1467 case AArch64::COALESCER_BARRIER_FPR128:
1468 return true;
1469 default:
1470 return false;
1471 }
1472 };
1473
1474 // For calls that temporarily have to toggle streaming mode as part of the
1475 // call-sequence, we need to be more careful when coalescing copy instructions
1476 // so that we don't end up coalescing the NEON/FP result or argument register
1477 // with a whole Z-register, such that after coalescing the register allocator
1478 // will try to spill/reload the entire Z register.
1479 //
1480 // We do this by checking if the node has any defs/uses that are
1481 // COALESCER_BARRIER pseudos. These are 'nops' in practice, but they exist to
1482 // instruct the coalescer to avoid coalescing the copy.
1483 if (MI->isCopy() && SubReg != DstSubReg &&
1484 (AArch64::ZPRRegClass.hasSubClassEq(RC: DstRC) ||
1485 AArch64::ZPRRegClass.hasSubClassEq(RC: SrcRC))) {
1486 unsigned SrcReg = MI->getOperand(i: 1).getReg();
1487 if (any_of(Range: MRI.def_instructions(Reg: SrcReg), P: IsCoalescerBarrier))
1488 return false;
1489 unsigned DstReg = MI->getOperand(i: 0).getReg();
1490 if (any_of(Range: MRI.use_nodbg_instructions(Reg: DstReg), P: IsCoalescerBarrier))
1491 return false;
1492 }
1493
1494 return true;
1495}
1496
1497bool AArch64RegisterInfo::shouldAnalyzePhysregInMachineLoopInfo(
1498 MCRegister R) const {
1499 return R == AArch64::VG;
1500}
1501
1502bool AArch64RegisterInfo::isIgnoredCVReg(MCRegister LLVMReg) const {
1503 return (LLVMReg >= AArch64::Z0 && LLVMReg <= AArch64::Z31) ||
1504 (LLVMReg >= AArch64::P0 && LLVMReg <= AArch64::P15);
1505}
1506