AArch64FrameLowering.cpp source code [llvm_projects/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp]

1	//===- AArch64FrameLowering.cpp - AArch64 Frame Lowering -------- C++ --====//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file contains the AArch64 implementation of TargetFrameLowering class.
10	//
11	// On AArch64, stack frames are structured as follows:
12	//
13	// The stack grows downward.
14	//
15	// All of the individual frame areas on the frame below are optional, i.e. it's
16	// possible to create a function so that the particular area isn't present
17	// in the frame.
18	//
19	// At function entry, the "frame" looks as follows:
20	//
21	// \| \| Higher address
22	// \|-----------------------------------\|
23	// \| \|
24	// \| arguments passed on the stack \|
25	// \| \|
26	// \|-----------------------------------\| <- sp
27	// \| \| Lower address
28	//
29	//
30	// After the prologue has run, the frame has the following general structure.
31	// Note that this doesn't depict the case where a red-zone is used. Also,
32	// technically the last frame area (VLAs) doesn't get created until in the
33	// main function body, after the prologue is run. However, it's depicted here
34	// for completeness.
35	//
36	// \| \| Higher address
37	// \|-----------------------------------\|
38	// \| \|
39	// \| arguments passed on the stack \|
40	// \| \|
41	// \|-----------------------------------\|
42	// \| \|
43	// \| (Win64 only) varargs from reg \|
44	// \| \|
45	// \|-----------------------------------\|
46	// \| \|
47	// \| (Win64 only) callee-saved SVE reg \|
48	// \| \|
49	// \|-----------------------------------\|
50	// \| \|
51	// \| callee-saved gpr registers \| <--.
52	// \| \| \| On Darwin platforms these
53	// \|- - - - - - - - - - - - - - - - - -\| \| callee saves are swapped,
54	// \| prev_lr \| \| (frame record first)
55	// \| prev_fp \| <--'
56	// \| async context if needed \|
57	// \| (a.k.a. "frame record") \|
58	// \|-----------------------------------\| <- fp(=x29)
59	// Default SVE stack layout Split SVE objects
60	// (aarch64-split-sve-objects=false) (aarch64-split-sve-objects=true)
61	// \|-----------------------------------\| \|-----------------------------------\|
62	// \| <hazard padding> \| \| callee-saved PPR registers \|
63	// \|-----------------------------------\| \|-----------------------------------\|
64	// \| \| \| PPR stack objects \|
65	// \| callee-saved fp/simd/SVE regs \| \|-----------------------------------\|
66	// \| \| \| <hazard padding> \|
67	// \|-----------------------------------\| \|-----------------------------------\|
68	// \| \| \| callee-saved ZPR/FPR registers \|
69	// \| SVE stack objects \| \|-----------------------------------\|
70	// \| \| \| ZPR stack objects \|
71	// \|-----------------------------------\| \|-----------------------------------\|
72	// ^ NB: FPR CSRs are promoted to ZPRs
73	// \|-----------------------------------\|
74	// \|.empty.space.to.make.part.below....\|
75	// \|.aligned.in.case.it.needs.more.than\| (size of this area is unknown at
76	// \|.the.standard.16-byte.alignment....\| compile time; if present)
77	// \|-----------------------------------\|
78	// \| local variables of fixed size \|
79	// \| including spill slots \|
80	// \| <FPR> \|
81	// \| <hazard padding> \|
82	// \| <GPR> \|
83	// \|-----------------------------------\| <- bp(not defined by ABI,
84	// \|.variable-sized.local.variables....\| LLVM chooses X19)
85	// \|.(VLAs)............................\| (size of this area is unknown at
86	// \|...................................\| compile time)
87	// \|-----------------------------------\| <- sp
88	// \| \| Lower address
89	//
90	//
91	// To access the data in a frame, at-compile time, a constant offset must be
92	// computable from one of the pointers (fp, bp, sp) to access it. The size
93	// of the areas with a dotted background cannot be computed at compile-time
94	// if they are present, making it required to have all three of fp, bp and
95	// sp to be set up to be able to access all contents in the frame areas,
96	// assuming all of the frame areas are non-empty.
97	//
98	// For most functions, some of the frame areas are empty. For those functions,
99	// it may not be necessary to set up fp or bp:
100	// A base pointer is definitely needed when there are both VLAs and local*
101	// variables with more-than-default alignment requirements.
102	// A frame pointer is definitely needed when there are local variables with*
103	// more-than-default alignment requirements.
104	//
105	// For Darwin platforms the frame-record (fp, lr) is stored at the top of the
106	// callee-saved area, since the unwind encoding does not allow for encoding
107	// this dynamically and existing tools depend on this layout. For other
108	// platforms, the frame-record is stored at the bottom of the (gpr) callee-saved
109	// area to allow SVE stack objects (allocated directly below the callee-saves,
110	// if available) to be accessed directly from the framepointer.
111	// The SVE spill/fill instructions have VL-scaled addressing modes such
112	// as:
113	// ldr z8, [fp, #-7 mul vl]
114	// For SVE the size of the vector length (VL) is not known at compile-time, so
115	// '#-7 mul vl' is an offset that can only be evaluated at runtime. With this
116	// layout, we don't need to add an unscaled offset to the framepointer before
117	// accessing the SVE object in the frame.
118	//
119	// In some cases when a base pointer is not strictly needed, it is generated
120	// anyway when offsets from the frame pointer to access local variables become
121	// so large that the offset can't be encoded in the immediate fields of loads
122	// or stores.
123	//
124	// Outgoing function arguments must be at the bottom of the stack frame when
125	// calling another function. If we do not have variable-sized stack objects, we
126	// can allocate a "reserved call frame" area at the bottom of the local
127	// variable area, large enough for all outgoing calls. If we do have VLAs, then
128	// the stack pointer must be decremented and incremented around each call to
129	// make space for the arguments below the VLAs.
130	//
131	// FIXME: also explain the redzone concept.
132	//
133	// About stack hazards: Under some SME contexts, a coprocessor with its own
134	// separate cache can used for FP operations. This can create hazards if the CPU
135	// and the SME unit try to access the same area of memory, including if the
136	// access is to an area of the stack. To try to alleviate this we attempt to
137	// introduce extra padding into the stack frame between FP and GPR accesses,
138	// controlled by the aarch64-stack-hazard-size option. Without changing the
139	// layout of the stack frame in the diagram above, a stack object of size
140	// aarch64-stack-hazard-size is added between GPR and FPR CSRs. Another is added
141	// to the stack objects section, and stack objects are sorted so that FPR >
142	// Hazard padding slot > GPRs (where possible). Unfortunately some things are
143	// not handled well (VLA area, arguments on the stack, objects with both GPR and
144	// FPR accesses), but if those are controlled by the user then the entire stack
145	// frame becomes GPR at the start/end with FPR in the middle, surrounded by
146	// Hazard padding.
147	//
148	// An example of the prologue:
149	//
150	// .globl __foo
151	// .align 2
152	// __foo:
153	// Ltmp0:
154	// .cfi_startproc
155	// .cfi_personality 155, ___gxx_personality_v0
156	// Leh_func_begin:
157	// .cfi_lsda 16, Lexception33
158	//
159	// stp xa,bx, [sp, -#offset]!
160	// ...
161	// stp x28, x27, [sp, #offset-32]
162	// stp fp, lr, [sp, #offset-16]
163	// add fp, sp, #offset - 16
164	// sub sp, sp, #1360
165	//
166	// The Stack:
167	// +-------------------------------------------+
168	// 10000 \| ........ \| ........ \| ........ \| ........ \|
169	// 10004 \| ........ \| ........ \| ........ \| ........ \|
170	// +-------------------------------------------+
171	// 10008 \| ........ \| ........ \| ........ \| ........ \|
172	// 1000c \| ........ \| ........ \| ........ \| ........ \|
173	// +===========================================+
174	// 10010 \| X28 Register \|
175	// 10014 \| X28 Register \|
176	// +-------------------------------------------+
177	// 10018 \| X27 Register \|
178	// 1001c \| X27 Register \|
179	// +===========================================+
180	// 10020 \| Frame Pointer \|
181	// 10024 \| Frame Pointer \|
182	// +-------------------------------------------+
183	// 10028 \| Link Register \|
184	// 1002c \| Link Register \|
185	// +===========================================+
186	// 10030 \| ........ \| ........ \| ........ \| ........ \|
187	// 10034 \| ........ \| ........ \| ........ \| ........ \|
188	// +-------------------------------------------+
189	// 10038 \| ........ \| ........ \| ........ \| ........ \|
190	// 1003c \| ........ \| ........ \| ........ \| ........ \|
191	// +-------------------------------------------+
192	//
193	// [sp] = 10030 :: >>initial value<<
194	// sp = 10020 :: stp fp, lr, [sp, #-16]!
195	// fp = sp == 10020 :: mov fp, sp
196	// [sp] == 10020 :: stp x28, x27, [sp, #-16]!
197	// sp == 10010 :: >>final value<<
198	//
199	// The frame pointer (w29) points to address 10020. If we use an offset of
200	// '16' from 'w29', we get the CFI offsets of -8 for w30, -16 for w29, -24
201	// for w27, and -32 for w28:
202	//
203	// Ltmp1:
204	// .cfi_def_cfa w29, 16
205	// Ltmp2:
206	// .cfi_offset w30, -8
207	// Ltmp3:
208	// .cfi_offset w29, -16
209	// Ltmp4:
210	// .cfi_offset w27, -24
211	// Ltmp5:
212	// .cfi_offset w28, -32
213	//
214	//===----------------------------------------------------------------------===//
215
216	#include "AArch64FrameLowering.h"
217	#include "AArch64InstrInfo.h"
218	#include "AArch64MachineFunctionInfo.h"
219	#include "AArch64PrologueEpilogue.h"
220	#include "AArch64RegisterInfo.h"
221	#include "AArch64SMEAttributes.h"
222	#include "AArch64Subtarget.h"
223	#include "MCTargetDesc/AArch64AddressingModes.h"
224	#include "MCTargetDesc/AArch64MCTargetDesc.h"
225	#include "llvm/ADT/ScopeExit.h"
226	#include "llvm/ADT/SmallVector.h"
227	#include "llvm/Analysis/ValueTracking.h"
228	#include "llvm/CodeGen/CFIInstBuilder.h"
229	#include "llvm/CodeGen/LivePhysRegs.h"
230	#include "llvm/CodeGen/MachineBasicBlock.h"
231	#include "llvm/CodeGen/MachineFrameInfo.h"
232	#include "llvm/CodeGen/MachineFunction.h"
233	#include "llvm/CodeGen/MachineInstr.h"
234	#include "llvm/CodeGen/MachineInstrBuilder.h"
235	#include "llvm/CodeGen/MachineMemOperand.h"
236	#include "llvm/CodeGen/MachineModuleInfo.h"
237	#include "llvm/CodeGen/MachineOperand.h"
238	#include "llvm/CodeGen/MachineRegisterInfo.h"
239	#include "llvm/CodeGen/RegisterScavenging.h"
240	#include "llvm/CodeGen/TargetInstrInfo.h"
241	#include "llvm/CodeGen/TargetRegisterInfo.h"
242	#include "llvm/CodeGen/TargetSubtargetInfo.h"
243	#include "llvm/CodeGen/WinEHFuncInfo.h"
244	#include "llvm/IR/Attributes.h"
245	#include "llvm/IR/CallingConv.h"
246	#include "llvm/IR/DataLayout.h"
247	#include "llvm/IR/DebugLoc.h"
248	#include "llvm/IR/Function.h"
249	#include "llvm/MC/MCAsmInfo.h"
250	#include "llvm/MC/MCDwarf.h"
251	#include "llvm/Support/CommandLine.h"
252	#include "llvm/Support/Debug.h"
253	#include "llvm/Support/ErrorHandling.h"
254	#include "llvm/Support/FormatVariadic.h"
255	#include "llvm/Support/MathExtras.h"
256	#include "llvm/Support/raw_ostream.h"
257	#include "llvm/Target/TargetMachine.h"
258	#include "llvm/Target/TargetOptions.h"
259	#include <cassert>
260	#include <cstdint>
261	#include <iterator>
262	#include <optional>
263	#include <vector>
264
265	using namespace llvm;
266
267	#define DEBUG_TYPE "frame-info"
268
269	static cl::opt<bool> EnableRedZone("aarch64-redzone",
270	cl::desc ("enable use of redzone on AArch64"),
271	cl::init(Val: false), cl::Hidden);
272
273	static cl::opt<bool> StackTaggingMergeSetTag(
274	"stack-tagging-merge-settag",
275	cl::desc ("merge settag instruction in function epilog"), cl::init(Val: true),
276	cl::Hidden);
277
278	static cl::opt<bool> OrderFrameObjects("aarch64-order-frame-objects",
279	cl::desc ("sort stack allocations"),
280	cl::init(Val: true), cl::Hidden);
281
282	static cl::opt<bool>
283	SplitSVEObjects("aarch64-split-sve-objects",
284	cl::desc ("Split allocation of ZPR & PPR objects"),
285	cl::init(Val: true), cl::Hidden);
286
287	cl::opt<bool> EnableHomogeneousPrologEpilog(
288	"homogeneous-prolog-epilog", cl::Hidden,
289	cl::desc ("Emit homogeneous prologue and epilogue for the size "
290	"optimization (default = off)"));
291
292	// Stack hazard size for analysis remarks. StackHazardSize takes precedence.
293	static cl::opt<unsigned>
294	StackHazardRemarkSize("aarch64-stack-hazard-remark-size", cl::init(Val: `0`),
295	cl::Hidden);
296	// Whether to insert padding into non-streaming functions (for testing).
297	static cl::opt<bool>
298	StackHazardInNonStreaming("aarch64-stack-hazard-in-non-streaming",
299	cl::init(Val: false), cl::Hidden);
300
301	static cl::opt<bool> DisableMultiVectorSpillFill(
302	"aarch64-disable-multivector-spill-fill",
303	cl::desc ("Disable use of LD/ST pairs for SME2 or SVE2p1"), cl::init(Val: false),
304	cl::Hidden);
305
306	int64_t
307	AArch64FrameLowering::getArgumentStackToRestore(MachineFunction &MF,
308	MachineBasicBlock &MBB) const {
309	MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
310	AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
311	bool IsTailCallReturn = (MBB.end() != MBBI)
312	? AArch64InstrInfo::isTailCallReturnInst(MI: *MBBI)
313	: false;
314
315	int64_t ArgumentPopSize = `0`;
316	if (IsTailCallReturn) {
317	MachineOperand &StackAdjust = MBBI ->getOperand(i: `1`);
318
319	// For a tail-call in a callee-pops-arguments environment, some or all of
320	// the stack may actually be in use for the call's arguments, this is
321	// calculated during LowerCall and consumed here...
322	ArgumentPopSize = StackAdjust.getImm();
323	} else {
324	// ... otherwise the amount to pop is all* of the argument space,*
325	// conveniently stored in the MachineFunctionInfo by
326	// LowerFormalArguments. This will, of course, be zero for the C calling
327	// convention.
328	ArgumentPopSize = AFI->getArgumentStackToRestore();
329	}
330
331	return ArgumentPopSize;
332	}
333
334	static bool produceCompactUnwindFrame(const AArch64FrameLowering &,
335	MachineFunction &MF);
336
337	enum class AssignObjectOffsets { No, Yes };
338	/// Process all the SVE stack objects and the SVE stack size and offsets for
339	/// each object. If AssignOffsets is "Yes", the offsets get assigned (and SVE
340	/// stack sizes set). Returns the size of the SVE stack.
341	static SVEStackSizes determineSVEStackSizes(MachineFunction &MF,
342	AssignObjectOffsets AssignOffsets);
343
344	static unsigned getStackHazardSize(const MachineFunction &MF) {
345	return MF.getSubtarget<AArch64Subtarget>().getStreamingHazardSize();
346	}
347
348	StackOffset
349	AArch64FrameLowering::getZPRStackSize(const MachineFunction &MF) const {
350	const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
351	return StackOffset::getScalable(Scalable: AFI->getStackSizeZPR());
352	}
353
354	StackOffset
355	AArch64FrameLowering::getPPRStackSize(const MachineFunction &MF) const {
356	// With split SVE objects, the hazard padding is added to the PPR region,
357	// which places it between the [GPR, PPR] area and the [ZPR, FPR] area. This
358	// avoids hazards between both GPRs and FPRs and ZPRs and PPRs.
359	const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
360	return StackOffset::get(Fixed: AFI->hasSplitSVEObjects() ? getStackHazardSize(MF)
361	: `0`,
362	Scalable: AFI->getStackSizePPR());
363	}
364
365	// Conservatively, returns true if the function is likely to have SVE vectors
366	// on the stack. This function is safe to be called before callee-saves or
367	// object offsets have been determined.
368	static bool isLikelyToHaveSVEStack(const AArch64FrameLowering &AFL,
369	const MachineFunction &MF) {
370	auto *AFI = MF.getInfo<AArch64FunctionInfo>();
371	if (AFI->isSVECC())
372	return true;
373
374	if (AFI->hasCalculatedStackSizeSVE())
375	return bool(AFL.getSVEStackSize(MF));
376
377	const MachineFrameInfo &MFI = MF.getFrameInfo();
378	for (int FI = MFI.getObjectIndexBegin(); FI < MFI.getObjectIndexEnd(); FI++) {
379	if (MFI.hasScalableStackID(ObjectIdx: FI))
380	return true;
381	}
382
383	return false;
384	}
385
386	static bool isTargetWindows(const MachineFunction &MF) {
387	return MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
388	}
389
390	bool AArch64FrameLowering::hasSVECalleeSavesAboveFrameRecord(
391	const MachineFunction &MF) const {
392	auto *AFI = MF.getInfo<AArch64FunctionInfo>();
393	return isTargetWindows(MF) && AFI->getSVECalleeSavedStackSize();
394	}
395
396	/// Returns true if a homogeneous prolog or epilog code can be emitted
397	/// for the size optimization. If possible, a frame helper call is injected.
398	/// When Exit block is given, this check is for epilog.
399	bool AArch64FrameLowering::homogeneousPrologEpilog(
400	MachineFunction &MF, MachineBasicBlock Exit) const* {
401	if (!MF.getFunction().hasMinSize())
402	return false;
403	if (!EnableHomogeneousPrologEpilog)
404	return false;
405	if (EnableRedZone)
406	return false;
407
408	// TODO: Window is supported yet.
409	if (isTargetWindows(MF))
410	return false;
411
412	// TODO: SVE is not supported yet.
413	if (isLikelyToHaveSVEStack(AFL: *this, MF))
414	return false;
415
416	// Bail on stack adjustment needed on return for simplicity.
417	const MachineFrameInfo &MFI = MF.getFrameInfo();
418	const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
419	if (MFI.hasVarSizedObjects() \|\| RegInfo->hasStackRealignment(MF))
420	return false;
421	if (Exit && getArgumentStackToRestore(MF, MBB&: *Exit))
422	return false;
423
424	auto *AFI = MF.getInfo<AArch64FunctionInfo>();
425	if (AFI->hasSwiftAsyncContext() \|\| AFI->hasStreamingModeChanges())
426	return false;
427
428	// If there are an odd number of GPRs before LR and FP in the CSRs list,
429	// they will not be paired into one RegPairInfo, which is incompatible with
430	// the assumption made by the homogeneous prolog epilog pass.
431	const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
432	unsigned NumGPRs = `0`;
433	for (unsigned I = `0`; CSRegs[I]; ++I) {
434	Register Reg = CSRegs[I];
435	if (Reg == AArch64::LR) {
436	assert(CSRegs[I + `1`] == AArch64::FP);
437	if (NumGPRs % `2` != `0`)
438	return false;
439	break;
440	}
441	if (AArch64::GPR64RegClass.contains(Reg))
442	++NumGPRs;
443	}
444
445	return true;
446	}
447
448	/// Returns true if CSRs should be paired.
449	bool AArch64FrameLowering::producePairRegisters(MachineFunction &MF) const {
450	return produceCompactUnwindFrame(*this, MF) \|\| homogeneousPrologEpilog(MF);
451	}
452
453	/// This is the biggest offset to the stack pointer we can encode in aarch64
454	/// instructions (without using a separate calculation and a temp register).
455	/// Note that the exception here are vector stores/loads which cannot encode any
456	/// displacements (see estimateRSStackSizeLimit(), isAArch64FrameOffsetLegal()).
457	static const unsigned DefaultSafeSPDisplacement = `255`;
458
459	/// Look at each instruction that references stack frames and return the stack
460	/// size limit beyond which some of these instructions will require a scratch
461	/// register during their expansion later.
462	static unsigned estimateRSStackSizeLimit(MachineFunction &MF) {
463	// FIXME: For now, just conservatively guesstimate based on unscaled indexing
464	// range. We'll end up allocating an unnecessary spill slot a lot, but
465	// realistically that's not a big deal at this stage of the game.
466	for (MachineBasicBlock &MBB : MF) {
467	for (MachineInstr &MI : MBB) {
468	if (MI.isDebugInstr() \|\| MI.isPseudo() \|\|
469	MI.getOpcode() == AArch64::ADDXri \|\|
470	MI.getOpcode() == AArch64::ADDSXri)
471	continue;
472
473	for (const MachineOperand &MO : MI.operands()) {
474	if (!MO.isFI())
475	continue;
476
477	StackOffset Offset;
478	if (isAArch64FrameOffsetLegal(MI, Offset, OutUseUnscaledOp: nullptr, OutUnscaledOp: nullptr, EmittableOffset: nullptr) ==
479	AArch64FrameOffsetCannotUpdate)
480	return `0`;
481	}
482	}
483	}
484	return DefaultSafeSPDisplacement;
485	}
486
487	TargetStackID::Value
488	AArch64FrameLowering::getStackIDForScalableVectors() const {
489	return TargetStackID::ScalableVector;
490	}
491
492	unsigned
493	AArch64FrameLowering::getFixedObjectSize(const MachineFunction &MF,
494	const AArch64FunctionInfo *AFI,
495	bool IsWin64, bool IsFunclet) const {
496	assert(AFI->getTailCallReservedStack() % `16` == `0` &&
497	"Tail call reserved stack must be aligned to 16 bytes");
498	if (!IsWin64 \|\| IsFunclet) {
499	return AFI->getTailCallReservedStack();
500	} else {
501	if (AFI->getTailCallReservedStack() != `0` &&
502	!MF.getFunction().getAttributes().hasAttrSomewhere(
503	Kind: Attribute::SwiftAsync))
504	report_fatal_error(reason: "cannot generate ABI-changing tail call for Win64");
505	unsigned FixedObjectSize = AFI->getTailCallReservedStack();
506
507	// Var args are stored here in the primary function.
508	FixedObjectSize += AFI->getVarArgsGPRSize();
509
510	if (MF.hasEHFunclets()) {
511	// Catch objects are stored here in the primary function.
512	const MachineFrameInfo &MFI = MF.getFrameInfo();
513	const WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo();
514	SmallSetVector<int, `8`> CatchObjFrameIndices;
515	for (const WinEHTryBlockMapEntry &TBME : EHInfo.TryBlockMap) {
516	for (const WinEHHandlerType &H : TBME.HandlerArray) {
517	int FrameIndex = H.CatchObj.FrameIndex;
518	if ((FrameIndex != INT_MAX) &&
519	CatchObjFrameIndices.insert(X: FrameIndex)) {
520	FixedObjectSize = alignTo(Value: FixedObjectSize,
521	Align: MFI.getObjectAlign(ObjectIdx: FrameIndex).value()) +
522	MFI.getObjectSize(ObjectIdx: FrameIndex);
523	}
524	}
525	}
526	// To support EH funclets we allocate an UnwindHelp object
527	FixedObjectSize += `8`;
528	}
529	return alignTo(Value: FixedObjectSize, Align: `16`);
530	}
531	}
532
533	bool AArch64FrameLowering::canUseRedZone(const MachineFunction &MF) const {
534	if (!EnableRedZone)
535	return false;
536
537	// Don't use the red zone if the function explicitly asks us not to.
538	// This is typically used for kernel code.
539	const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
540	const unsigned RedZoneSize =
541	Subtarget.getTargetLowering()->getRedZoneSize(F: MF.getFunction());
542	if (!RedZoneSize)
543	return false;
544
545	const MachineFrameInfo &MFI = MF.getFrameInfo();
546	const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
547	uint64_t NumBytes = AFI->getLocalStackSize();
548
549	// If neither NEON or SVE are available, a COPY from one Q-reg to
550	// another requires a spill -> reload sequence. We can do that
551	// using a pre-decrementing store/post-decrementing load, but
552	// if we do so, we can't use the Red Zone.
553	bool LowerQRegCopyThroughMem = Subtarget.hasFPARMv8() &&
554	!Subtarget.isNeonAvailable() &&
555	!Subtarget.hasSVE();
556
557	return !(MFI.hasCalls() \|\| hasFP(MF) \|\| NumBytes > RedZoneSize \|\|
558	AFI->hasSVEStackSize() \|\| LowerQRegCopyThroughMem);
559	}
560
561	/// hasFPImpl - Return true if the specified function should have a dedicated
562	/// frame pointer register.
563	bool AArch64FrameLowering::hasFPImpl(const MachineFunction &MF) const {
564	const MachineFrameInfo &MFI = MF.getFrameInfo();
565	const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
566	const AArch64FunctionInfo &AFI = *MF.getInfo<AArch64FunctionInfo>();
567
568	// Win64 EH requires a frame pointer if funclets are present, as the locals
569	// are accessed off the frame pointer in both the parent function and the
570	// funclets.
571	if (MF.hasEHFunclets())
572	return true;
573	// Retain behavior of always omitting the FP for leaf functions when possible.
574	if (MF.getTarget().Options.DisableFramePointerElim(MF))
575	return true;
576	if (MFI.hasVarSizedObjects() \|\| MFI.isFrameAddressTaken() \|\|
577	MFI.hasStackMap() \|\| MFI.hasPatchPoint() \|\|
578	RegInfo->hasStackRealignment(MF))
579	return true;
580
581	// If we:
582	//
583	// 1. Have streaming mode changes
584	// OR:
585	// 2. Have a streaming body with SVE stack objects
586	//
587	// Then the value of VG restored when unwinding to this function may not match
588	// the value of VG used to set up the stack.
589	//
590	// This is a problem as the CFA can be described with an expression of the
591	// form: CFA = SP + NumBytes + VG NumScalableBytes.*
592	//
593	// If the value of VG used in that expression does not match the value used to
594	// set up the stack, an incorrect address for the CFA will be computed, and
595	// unwinding will fail.
596	//
597	// We work around this issue by ensuring the frame-pointer can describe the
598	// CFA in either of these cases.
599	if (AFI.needsDwarfUnwindInfo(MF) &&
600	((requiresSaveVG(MF) \|\| AFI.getSMEFnAttrs().hasStreamingBody()) &&
601	(!AFI.hasCalculatedStackSizeSVE() \|\| AFI.hasSVEStackSize())))
602	return true;
603	// With large callframes around we may need to use FP to access the scavenging
604	// emergency spillslot.
605	//
606	// Unfortunately some calls to hasFP() like machine verifier ->
607	// getReservedReg() -> hasFP in the middle of global isel are too early
608	// to know the max call frame size. Hopefully conservatively returning "true"
609	// in those cases is fine.
610	// DefaultSafeSPDisplacement is fine as we only emergency spill GP regs.
611	if (!MFI.isMaxCallFrameSizeComputed() \|\|
612	MFI.getMaxCallFrameSize() > DefaultSafeSPDisplacement)
613	return true;
614
615	return false;
616	}
617
618	/// Should the Frame Pointer be reserved for the current function?
619	bool AArch64FrameLowering::isFPReserved(const MachineFunction &MF) const {
620	const TargetMachine &TM = MF.getTarget();
621	const Triple &TT = TM.getTargetTriple();
622
623	// These OSes require the frame chain is valid, even if the current frame does
624	// not use a frame pointer.
625	if (TT.isOSDarwin() \|\| TT.isOSWindows())
626	return true;
627
628	// If the function has a frame pointer, it is reserved.
629	if (hasFP(MF))
630	return true;
631
632	// Frontend has requested to preserve the frame pointer.
633	if (TM.Options.FramePointerIsReserved(MF))
634	return true;
635
636	return false;
637	}
638
639	/// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
640	/// not required, we reserve argument space for call sites in the function
641	/// immediately on entry to the current function. This eliminates the need for
642	/// add/sub sp brackets around call sites. Returns true if the call frame is
643	/// included as part of the stack frame.
644	bool AArch64FrameLowering::hasReservedCallFrame(
645	const MachineFunction &MF) const {
646	// The stack probing code for the dynamically allocated outgoing arguments
647	// area assumes that the stack is probed at the top - either by the prologue
648	// code, which issues a probe if `hasVarSizedObjects` return true, or by the
649	// most recent variable-sized object allocation. Changing the condition here
650	// may need to be followed up by changes to the probe issuing logic.
651	return !MF.getFrameInfo().hasVarSizedObjects();
652	}
653
654	MachineBasicBlock::iterator AArch64FrameLowering::eliminateCallFramePseudoInstr(
655	MachineFunction &MF, MachineBasicBlock &MBB,
656	MachineBasicBlock::iterator I) const {
657
658	const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
659	const AArch64InstrInfo *TII = Subtarget.getInstrInfo();
660	const AArch64TargetLowering *TLI = Subtarget.getTargetLowering();
661	[[maybe_unused]] MachineFrameInfo &MFI = MF.getFrameInfo();
662	DebugLoc DL = I ->getDebugLoc();
663	unsigned Opc = I ->getOpcode();
664	bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
665	uint64_t CalleePopAmount = IsDestroy ? I ->getOperand(i: `1`).getImm() : `0`;
666
667	if (!hasReservedCallFrame(MF)) {
668	int64_t Amount = I ->getOperand(i: `0`).getImm();
669	Amount = alignTo(Size: Amount, A: getStackAlign());
670	if (!IsDestroy)
671	Amount = -Amount;
672
673	// N.b. if CalleePopAmount is valid but zero (i.e. callee would pop, but it
674	// doesn't have to pop anything), then the first operand will be zero too so
675	// this adjustment is a no-op.
676	if (CalleePopAmount == `0`) {
677	// FIXME: in-function stack adjustment for calls is limited to 24-bits
678	// because there's no guaranteed temporary register available.
679	//
680	// ADD/SUB (immediate) has only LSL #0 and LSL #12 available.
681	// 1) For offset <= 12-bit, we use LSL #0
682	// 2) For 12-bit <= offset <= 24-bit, we use two instructions. One uses
683	// LSL #0, and the other uses LSL #12.
684	//
685	// Most call frames will be allocated at the start of a function so
686	// this is OK, but it is a limitation that needs dealing with.
687	assert(Amount > -`0xffffff` && Amount < `0xffffff` && "call frame too large");
688
689	if (TLI->hasInlineStackProbe(MF) &&
690	-Amount >= AArch64::StackProbeMaxUnprobedStack) {
691	// When stack probing is enabled, the decrement of SP may need to be
692	// probed. We only need to do this if the call site needs 1024 bytes of
693	// space or more, because a region smaller than that is allowed to be
694	// unprobed at an ABI boundary. We rely on the fact that SP has been
695	// probed exactly at this point, either by the prologue or most recent
696	// dynamic allocation.
697	assert(MFI.hasVarSizedObjects() &&
698	"non-reserved call frame without var sized objects?");
699	Register ScratchReg =
700	MF.getRegInfo().createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
701	inlineStackProbeFixed(MBBI: I, ScratchReg, FrameSize: -Amount, CFAOffset: StackOffset::get(Fixed: `0`, Scalable: `0`));
702	} else {
703	emitFrameOffset(MBB, MBBI: I, DL, DestReg: AArch64::SP, SrcReg: AArch64::SP,
704	Offset: StackOffset::getFixed(Fixed: Amount), TII);
705	}
706	}
707	} else if (CalleePopAmount != `0`) {
708	// If the calling convention demands that the callee pops arguments from the
709	// stack, we want to add it back if we have a reserved call frame.
710	assert(CalleePopAmount < `0xffffff` && "call frame too large");
711	emitFrameOffset(MBB, MBBI: I, DL, DestReg: AArch64::SP, SrcReg: AArch64::SP,
712	Offset: StackOffset::getFixed(Fixed: -(int64_t)CalleePopAmount), TII);
713	}
714	return MBB.erase(I);
715	}
716
717	void AArch64FrameLowering::resetCFIToInitialState(
718	MachineBasicBlock &MBB) const {
719
720	MachineFunction &MF = *MBB.getParent();
721	const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
722	const auto &TRI = *Subtarget.getRegisterInfo();
723	const auto &MFI = *MF.getInfo<AArch64FunctionInfo>();
724
725	CFIInstBuilder CFIBuilder(MBB, MBB.begin(), MachineInstr::NoFlags);
726
727	// Reset the CFA to `SP + 0`.
728	CFIBuilder.buildDefCFA(Reg: AArch64::SP, Offset: `0`);
729
730	// Flip the RA sign state.
731	if (MFI.shouldSignReturnAddress(MF))
732	MFI.branchProtectionPAuthLR() ? CFIBuilder.buildNegateRAStateWithPC()
733	: CFIBuilder.buildNegateRAState();
734
735	// Shadow call stack uses X18, reset it.
736	if (MFI.needsShadowCallStackPrologueEpilogue(MF))
737	CFIBuilder.buildSameValue(Reg: AArch64::X18);
738
739	// Emit .cfi_same_value for callee-saved registers.
740	const std::vector<CalleeSavedInfo> &CSI =
741	MF.getFrameInfo().getCalleeSavedInfo();
742	for (const auto &Info : CSI) {
743	MCRegister Reg = Info.getReg();
744	if (!TRI.regNeedsCFI(Reg, RegToUseForCFI&: Reg))
745	continue;
746	CFIBuilder.buildSameValue(Reg);
747	}
748	}
749
750	static MCRegister getRegisterOrZero(MCRegister Reg, bool HasSVE) {
751	switch (Reg.id()) {
752	default:
753	// The called routine is expected to preserve r19-r28
754	// r29 and r30 are used as frame pointer and link register resp.
755	return `0`;
756
757	// GPRs
758	#define CASE(n) \
759	case AArch64::W##n: \
760	case AArch64::X##n: \
761	return AArch64::X##n
762	CASE(`0`);
763	CASE(`1`);
764	CASE(`2`);
765	CASE(`3`);
766	CASE(`4`);
767	CASE(`5`);
768	CASE(`6`);
769	CASE(`7`);
770	CASE(`8`);
771	CASE(`9`);
772	CASE(`10`);
773	CASE(`11`);
774	CASE(`12`);
775	CASE(`13`);
776	CASE(`14`);
777	CASE(`15`);
778	CASE(`16`);
779	CASE(`17`);
780	CASE(`18`);
781	#undef CASE
782
783	// FPRs
784	#define CASE(n) \
785	case AArch64::B##n: \
786	case AArch64::H##n: \
787	case AArch64::S##n: \
788	case AArch64::D##n: \
789	case AArch64::Q##n: \
790	return HasSVE ? AArch64::Z##n : AArch64::Q##n
791	CASE(`0`);
792	CASE(`1`);
793	CASE(`2`);
794	CASE(`3`);
795	CASE(`4`);
796	CASE(`5`);
797	CASE(`6`);
798	CASE(`7`);
799	CASE(`8`);
800	CASE(`9`);
801	CASE(`10`);
802	CASE(`11`);
803	CASE(`12`);
804	CASE(`13`);
805	CASE(`14`);
806	CASE(`15`);
807	CASE(`16`);
808	CASE(`17`);
809	CASE(`18`);
810	CASE(`19`);
811	CASE(`20`);
812	CASE(`21`);
813	CASE(`22`);
814	CASE(`23`);
815	CASE(`24`);
816	CASE(`25`);
817	CASE(`26`);
818	CASE(`27`);
819	CASE(`28`);
820	CASE(`29`);
821	CASE(`30`);
822	CASE(`31`);
823	#undef CASE
824	}
825	}
826
827	void AArch64FrameLowering::emitZeroCallUsedRegs(BitVector RegsToZero,
828	MachineBasicBlock &MBB) const {
829	// Insertion point.
830	MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
831
832	// Fake a debug loc.
833	DebugLoc DL;
834	if (MBBI != MBB.end())
835	DL = MBBI ->getDebugLoc();
836
837	const MachineFunction &MF = *MBB.getParent();
838	const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
839	const AArch64RegisterInfo &TRI = *STI.getRegisterInfo();
840
841	BitVector GPRsToZero(TRI.getNumRegs());
842	BitVector FPRsToZero(TRI.getNumRegs());
843	bool HasSVE = STI.isSVEorStreamingSVEAvailable();
844	for (MCRegister Reg : RegsToZero.set_bits()) {
845	if (TRI.isGeneralPurposeRegister(MF, Reg)) {
846	// For GPRs, we only care to clear out the 64-bit register.
847	if (MCRegister XReg = getRegisterOrZero(Reg, HasSVE))
848	GPRsToZero.set(XReg);
849	} else if (AArch64InstrInfo::isFpOrNEON(Reg)) {
850	// For FPRs,
851	if (MCRegister XReg = getRegisterOrZero(Reg, HasSVE))
852	FPRsToZero.set(XReg);
853	}
854	}
855
856	const AArch64InstrInfo &TII = *STI.getInstrInfo();
857
858	// Zero out GPRs.
859	for (MCRegister Reg : GPRsToZero.set_bits())
860	TII.buildClearRegister(Reg, MBB, Iter: MBBI, DL);
861
862	// Zero out FP/vector registers.
863	for (MCRegister Reg : FPRsToZero.set_bits())
864	TII.buildClearRegister(Reg, MBB, Iter: MBBI, DL);
865
866	if (HasSVE) {
867	for (MCRegister PReg :
868	{AArch64::P0, AArch64::P1, AArch64::P2, AArch64::P3, AArch64::P4,
869	AArch64::P5, AArch64::P6, AArch64::P7, AArch64::P8, AArch64::P9,
870	AArch64::P10, AArch64::P11, AArch64::P12, AArch64::P13, AArch64::P14,
871	AArch64::P15}) {
872	if (RegsToZero [PReg])
873	BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AArch64::PFALSE), DestReg: PReg);
874	}
875	}
876	}
877
878	bool AArch64FrameLowering::windowsRequiresStackProbe(
879	const MachineFunction &MF, uint64_t StackSizeInBytes) const {
880	const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
881	const AArch64FunctionInfo &MFI = *MF.getInfo<AArch64FunctionInfo>();
882	// TODO: When implementing stack protectors, take that into account
883	// for the probe threshold.
884	return Subtarget.isTargetWindows() && MFI.hasStackProbing() &&
885	StackSizeInBytes >= uint64_t(MFI.getStackProbeSize());
886	}
887
888	static void getLiveRegsForEntryMBB(LivePhysRegs &LiveRegs,
889	const MachineBasicBlock &MBB) {
890	const MachineFunction *MF = MBB.getParent();
891	LiveRegs.addLiveIns(MBB);
892	// Mark callee saved registers as used so we will not choose them.
893	const MCPhysReg *CSRegs = MF->getRegInfo().getCalleeSavedRegs();
894	for (unsigned i = `0`; CSRegs[i]; ++i)
895	LiveRegs.addReg(Reg: CSRegs[i]);
896	}
897
898	Register
899	AArch64FrameLowering::findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB,
900	bool HasCall) const {
901	MachineFunction *MF = MBB->getParent();
902
903	// If MBB is an entry block, use X9 as the scratch register
904	// preserve_none functions may be using X9 to pass arguments,
905	// so prefer to pick an available register below.
906	if (&MF->front() == MBB &&
907	MF->getFunction().getCallingConv() != CallingConv::PreserveNone)
908	return AArch64::X9;
909
910	const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
911	const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo();
912	LivePhysRegs LiveRegs(TRI);
913	getLiveRegsForEntryMBB(LiveRegs, MBB: *MBB);
914	if (HasCall) {
915	LiveRegs.addReg(Reg: AArch64::X16);
916	LiveRegs.addReg(Reg: AArch64::X17);
917	LiveRegs.addReg(Reg: AArch64::X18);
918	}
919
920	// Prefer X9 since it was historically used for the prologue scratch reg.
921	const MachineRegisterInfo &MRI = MF->getRegInfo();
922	if (LiveRegs.available(MRI, Reg: AArch64::X9))
923	return AArch64::X9;
924
925	for (unsigned Reg : AArch64::GPR64RegClass) {
926	if (LiveRegs.available(MRI, Reg))
927	return Reg;
928	}
929	return AArch64::NoRegister;
930	}
931
932	bool AArch64FrameLowering::canUseAsPrologue(
933	const MachineBasicBlock &MBB) const {
934	const MachineFunction *MF = MBB.getParent();
935	MachineBasicBlock TmpMBB = const_cast<MachineBasicBlock >(&MBB);
936	const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
937	const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
938	const AArch64TargetLowering *TLI = Subtarget.getTargetLowering();
939	const AArch64FunctionInfo *AFI = MF->getInfo<AArch64FunctionInfo>();
940
941	if (AFI->hasSwiftAsyncContext()) {
942	const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo();
943	const MachineRegisterInfo &MRI = MF->getRegInfo();
944	LivePhysRegs LiveRegs(TRI);
945	getLiveRegsForEntryMBB(LiveRegs, MBB);
946	// The StoreSwiftAsyncContext clobbers X16 and X17. Make sure they are
947	// available.
948	if (!LiveRegs.available(MRI, Reg: AArch64::X16) \|\|
949	!LiveRegs.available(MRI, Reg: AArch64::X17))
950	return false;
951	}
952
953	// Certain stack probing sequences might clobber flags, then we can't use
954	// the block as a prologue if the flags register is a live-in.
955	if (MF->getInfo<AArch64FunctionInfo>()->hasStackProbing() &&
956	MBB.isLiveIn(Reg: AArch64::NZCV))
957	return false;
958
959	if (RegInfo->hasStackRealignment(MF: MF) \|\| TLI->hasInlineStackProbe(MF: MF))
960	if (findScratchNonCalleeSaveRegister(MBB: TmpMBB) == AArch64::NoRegister)
961	return false;
962
963	// May need a scratch register (for return value) if require making a special
964	// call
965	if (requiresSaveVG(MF: *MF) \|\|
966	windowsRequiresStackProbe(MF: *MF, StackSizeInBytes: std::numeric_limits<uint64_t>::max()))
967	if (findScratchNonCalleeSaveRegister(MBB: TmpMBB, HasCall: true) == AArch64::NoRegister)
968	return false;
969
970	return true;
971	}
972
973	bool AArch64FrameLowering::needsWinCFI(const MachineFunction &MF) const {
974	const Function &F = MF.getFunction();
975	return MF.getTarget().getMCAsmInfo()->usesWindowsCFI() &&
976	F.needsUnwindTableEntry();
977	}
978
979	bool AArch64FrameLowering::shouldSignReturnAddressEverywhere(
980	const MachineFunction &MF) const {
981	// FIXME: With WinCFI, extra care should be taken to place SEH_PACSignLR
982	// and SEH_EpilogEnd instructions in the correct order.
983	if (MF.getTarget().getMCAsmInfo()->usesWindowsCFI())
984	return false;
985	const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
986	return AFI->getSignReturnAddressCondition() == SignReturnAddress::All;
987	}
988
989	// Given a load or a store instruction, generate an appropriate unwinding SEH
990	// code on Windows.
991	MachineBasicBlock::iterator
992	AArch64FrameLowering::insertSEH(MachineBasicBlock::iterator MBBI,
993	const AArch64InstrInfo &TII,
994	MachineInstr::MIFlag Flag) const {
995	unsigned Opc = MBBI ->getOpcode();
996	MachineBasicBlock *MBB = MBBI ->getParent();
997	MachineFunction &MF = *MBB->getParent();
998	DebugLoc DL = MBBI ->getDebugLoc();
999	unsigned ImmIdx = MBBI ->getNumOperands() - `1`;
1000	int Imm = MBBI ->getOperand(i: ImmIdx).getImm();
1001	MachineInstrBuilder MIB;
1002	const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1003	const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
1004
1005	switch (Opc) {
1006	default:
1007	report_fatal_error(reason: "No SEH Opcode for this instruction");
1008	case AArch64::STR_ZXI:
1009	case AArch64::LDR_ZXI: {
1010	unsigned Reg0 = RegInfo->getSEHRegNum(i: MBBI ->getOperand(i: `0`).getReg());
1011	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveZReg))
1012	.addImm(Val: Reg0)
1013	.addImm(Val: Imm)
1014	.setMIFlag(Flag);
1015	break;
1016	}
1017	case AArch64::STR_PXI:
1018	case AArch64::LDR_PXI: {
1019	unsigned Reg0 = RegInfo->getSEHRegNum(i: MBBI ->getOperand(i: `0`).getReg());
1020	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SavePReg))
1021	.addImm(Val: Reg0)
1022	.addImm(Val: Imm)
1023	.setMIFlag(Flag);
1024	break;
1025	}
1026	case AArch64::LDPDpost:
1027	Imm = -Imm;
1028	[[fallthrough]];
1029	case AArch64::STPDpre: {
1030	unsigned Reg0 = RegInfo->getSEHRegNum(i: MBBI ->getOperand(i: `1`).getReg());
1031	unsigned Reg1 = RegInfo->getSEHRegNum(i: MBBI ->getOperand(i: `2`).getReg());
1032	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveFRegP_X))
1033	.addImm(Val: Reg0)
1034	.addImm(Val: Reg1)
1035	.addImm(Val: Imm * `8`)
1036	.setMIFlag(Flag);
1037	break;
1038	}
1039	case AArch64::LDPXpost:
1040	Imm = -Imm;
1041	[[fallthrough]];
1042	case AArch64::STPXpre: {
1043	Register Reg0 = MBBI ->getOperand(i: `1`).getReg();
1044	Register Reg1 = MBBI ->getOperand(i: `2`).getReg();
1045	if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
1046	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveFPLR_X))
1047	.addImm(Val: Imm * `8`)
1048	.setMIFlag(Flag);
1049	else
1050	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveRegP_X))
1051	.addImm(Val: RegInfo->getSEHRegNum(i: Reg0))
1052	.addImm(Val: RegInfo->getSEHRegNum(i: Reg1))
1053	.addImm(Val: Imm * `8`)
1054	.setMIFlag(Flag);
1055	break;
1056	}
1057	case AArch64::LDRDpost:
1058	Imm = -Imm;
1059	[[fallthrough]];
1060	case AArch64::STRDpre: {
1061	unsigned Reg = RegInfo->getSEHRegNum(i: MBBI ->getOperand(i: `1`).getReg());
1062	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveFReg_X))
1063	.addImm(Val: Reg)
1064	.addImm(Val: Imm)
1065	.setMIFlag(Flag);
1066	break;
1067	}
1068	case AArch64::LDRXpost:
1069	Imm = -Imm;
1070	[[fallthrough]];
1071	case AArch64::STRXpre: {
1072	unsigned Reg = RegInfo->getSEHRegNum(i: MBBI ->getOperand(i: `1`).getReg());
1073	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveReg_X))
1074	.addImm(Val: Reg)
1075	.addImm(Val: Imm)
1076	.setMIFlag(Flag);
1077	break;
1078	}
1079	case AArch64::STPDi:
1080	case AArch64::LDPDi: {
1081	unsigned Reg0 = RegInfo->getSEHRegNum(i: MBBI ->getOperand(i: `0`).getReg());
1082	unsigned Reg1 = RegInfo->getSEHRegNum(i: MBBI ->getOperand(i: `1`).getReg());
1083	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveFRegP))
1084	.addImm(Val: Reg0)
1085	.addImm(Val: Reg1)
1086	.addImm(Val: Imm * `8`)
1087	.setMIFlag(Flag);
1088	break;
1089	}
1090	case AArch64::STPXi:
1091	case AArch64::LDPXi: {
1092	Register Reg0 = MBBI ->getOperand(i: `0`).getReg();
1093	Register Reg1 = MBBI ->getOperand(i: `1`).getReg();
1094
1095	int SEHReg0 = RegInfo->getSEHRegNum(i: Reg0);
1096	int SEHReg1 = RegInfo->getSEHRegNum(i: Reg1);
1097
1098	if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
1099	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveFPLR))
1100	.addImm(Val: Imm * `8`)
1101	.setMIFlag(Flag);
1102	else if (SEHReg0 >= `19` && SEHReg1 >= `19`)
1103	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveRegP))
1104	.addImm(Val: SEHReg0)
1105	.addImm(Val: SEHReg1)
1106	.addImm(Val: Imm * `8`)
1107	.setMIFlag(Flag);
1108	else
1109	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveAnyRegIP))
1110	.addImm(Val: SEHReg0)
1111	.addImm(Val: SEHReg1)
1112	.addImm(Val: Imm * `8`)
1113	.setMIFlag(Flag);
1114	break;
1115	}
1116	case AArch64::STRXui:
1117	case AArch64::LDRXui: {
1118	int Reg = RegInfo->getSEHRegNum(i: MBBI ->getOperand(i: `0`).getReg());
1119	if (Reg >= `19`)
1120	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveReg))
1121	.addImm(Val: Reg)
1122	.addImm(Val: Imm * `8`)
1123	.setMIFlag(Flag);
1124	else
1125	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveAnyRegI))
1126	.addImm(Val: Reg)
1127	.addImm(Val: Imm * `8`)
1128	.setMIFlag(Flag);
1129	break;
1130	}
1131	case AArch64::STRDui:
1132	case AArch64::LDRDui: {
1133	unsigned Reg = RegInfo->getSEHRegNum(i: MBBI ->getOperand(i: `0`).getReg());
1134	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveFReg))
1135	.addImm(Val: Reg)
1136	.addImm(Val: Imm * `8`)
1137	.setMIFlag(Flag);
1138	break;
1139	}
1140	case AArch64::STPQi:
1141	case AArch64::LDPQi: {
1142	unsigned Reg0 = RegInfo->getSEHRegNum(i: MBBI ->getOperand(i: `0`).getReg());
1143	unsigned Reg1 = RegInfo->getSEHRegNum(i: MBBI ->getOperand(i: `1`).getReg());
1144	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveAnyRegQP))
1145	.addImm(Val: Reg0)
1146	.addImm(Val: Reg1)
1147	.addImm(Val: Imm * `16`)
1148	.setMIFlag(Flag);
1149	break;
1150	}
1151	case AArch64::LDPQpost:
1152	Imm = -Imm;
1153	[[fallthrough]];
1154	case AArch64::STPQpre: {
1155	unsigned Reg0 = RegInfo->getSEHRegNum(i: MBBI ->getOperand(i: `1`).getReg());
1156	unsigned Reg1 = RegInfo->getSEHRegNum(i: MBBI ->getOperand(i: `2`).getReg());
1157	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveAnyRegQPX))
1158	.addImm(Val: Reg0)
1159	.addImm(Val: Reg1)
1160	.addImm(Val: Imm * `16`)
1161	.setMIFlag(Flag);
1162	break;
1163	}
1164	}
1165	auto I = MBB->insertAfter(I: MBBI, MI: MIB);
1166	return I;
1167	}
1168
1169	bool AArch64FrameLowering::requiresSaveVG(const MachineFunction &MF) const {
1170	const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1171	if (!AFI->needsDwarfUnwindInfo(MF) \|\| !AFI->hasStreamingModeChanges())
1172	return false;
1173	// For Darwin platforms we don't save VG for non-SVE functions, even if SME
1174	// is enabled with streaming mode changes.
1175	auto &ST = MF.getSubtarget<AArch64Subtarget>();
1176	if (ST.isTargetDarwin())
1177	return ST.hasSVE();
1178	return true;
1179	}
1180
1181	void AArch64FrameLowering::emitPacRetPlusLeafHardening(
1182	MachineFunction &MF) const {
1183	const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1184	const AArch64InstrInfo *TII = Subtarget.getInstrInfo();
1185
1186	auto EmitSignRA = [&](MachineBasicBlock &MBB) {
1187	DebugLoc DL; // Set debug location to unknown.
1188	MachineBasicBlock::iterator MBBI = MBB.begin();
1189
1190	BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PAUTH_PROLOGUE))
1191	.setMIFlag(MachineInstr::FrameSetup);
1192	};
1193
1194	auto EmitAuthRA = [&](MachineBasicBlock &MBB) {
1195	DebugLoc DL;
1196	MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
1197	if (MBBI != MBB.end())
1198	DL = MBBI ->getDebugLoc();
1199
1200	BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PAUTH_EPILOGUE))
1201	.setMIFlag(MachineInstr::FrameDestroy);
1202	};
1203
1204	// This should be in sync with PEIImpl::calculateSaveRestoreBlocks.
1205	EmitSignRA (MF.front());
1206	for (MachineBasicBlock &MBB : MF) {
1207	if (MBB.isEHFuncletEntry())
1208	EmitSignRA (MBB);
1209	if (MBB.isReturnBlock())
1210	EmitAuthRA (MBB);
1211	}
1212	}
1213
1214	void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
1215	MachineBasicBlock &MBB) const {
1216	AArch64PrologueEmitter PrologueEmitter(MF, MBB, *this);
1217	PrologueEmitter.emitPrologue();
1218	}
1219
1220	void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
1221	MachineBasicBlock &MBB) const {
1222	AArch64EpilogueEmitter EpilogueEmitter(MF, MBB, *this);
1223	EpilogueEmitter.emitEpilogue();
1224	}
1225
1226	bool AArch64FrameLowering::enableCFIFixup(const MachineFunction &MF) const {
1227	return TargetFrameLowering::enableCFIFixup(MF) &&
1228	MF.getInfo<AArch64FunctionInfo>()->needsDwarfUnwindInfo(MF);
1229	}
1230
1231	bool AArch64FrameLowering::enableFullCFIFixup(const MachineFunction &MF) const {
1232	return enableCFIFixup(MF) &&
1233	MF.getInfo<AArch64FunctionInfo>()->needsAsyncDwarfUnwindInfo(MF);
1234	}
1235
1236	/// getFrameIndexReference - Provide a base+offset reference to an FI slot for
1237	/// debug info. It's the same as what we use for resolving the code-gen
1238	/// references for now. FIXME: This can go wrong when references are
1239	/// SP-relative and simple call frames aren't used.
1240	StackOffset
1241	AArch64FrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
1242	Register &FrameReg) const {
1243	return resolveFrameIndexReference(
1244	MF, FI, FrameReg,
1245	/PreferFP=/
1246	MF.getFunction().hasFnAttribute(Kind: Attribute::SanitizeHWAddress) \|\|
1247	MF.getFunction().hasFnAttribute(Kind: Attribute::SanitizeMemTag),
1248	/ForSimm=/false);
1249	}
1250
1251	StackOffset
1252	AArch64FrameLowering::getFrameIndexReferenceFromSP(const MachineFunction &MF,
1253	int FI) const {
1254	// This function serves to provide a comparable offset from a single reference
1255	// point (the value of SP at function entry) that can be used for analysis,
1256	// e.g. the stack-frame-layout analysis pass. It is not guaranteed to be
1257	// correct for all objects in the presence of VLA-area objects or dynamic
1258	// stack re-alignment.
1259
1260	const auto &MFI = MF.getFrameInfo();
1261
1262	int64_t ObjectOffset = MFI.getObjectOffset(ObjectIdx: FI);
1263	StackOffset ZPRStackSize = getZPRStackSize(MF);
1264	StackOffset PPRStackSize = getPPRStackSize(MF);
1265	StackOffset SVEStackSize = ZPRStackSize + PPRStackSize;
1266
1267	// For VLA-area objects, just emit an offset at the end of the stack frame.
1268	// Whilst not quite correct, these objects do live at the end of the frame and
1269	// so it is more useful for analysis for the offset to reflect this.
1270	if (MFI.isVariableSizedObjectIndex(ObjectIdx: FI)) {
1271	return StackOffset::getFixed(Fixed: -((int64_t)MFI.getStackSize())) - SVEStackSize;
1272	}
1273
1274	// This is correct in the absence of any SVE stack objects.
1275	if (!SVEStackSize)
1276	return StackOffset::getFixed(Fixed: ObjectOffset - getOffsetOfLocalArea());
1277
1278	const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
1279	bool FPAfterSVECalleeSaves = hasSVECalleeSavesAboveFrameRecord(MF);
1280	if (MFI.hasScalableStackID(ObjectIdx: FI)) {
1281	if (FPAfterSVECalleeSaves &&
1282	-ObjectOffset <= (int64_t)AFI->getSVECalleeSavedStackSize()) {
1283	assert(!AFI->hasSplitSVEObjects() &&
1284	"split-sve-objects not supported with FPAfterSVECalleeSaves");
1285	return StackOffset::getScalable(Scalable: ObjectOffset);
1286	}
1287	StackOffset AccessOffset{};
1288	// The scalable vectors are below (lower address) the scalable predicates
1289	// with split SVE objects, so we must subtract the size of the predicates.
1290	if (AFI->hasSplitSVEObjects() &&
1291	MFI.getStackID(ObjectIdx: FI) == TargetStackID::ScalableVector)
1292	AccessOffset = -PPRStackSize;
1293	return AccessOffset +
1294	StackOffset::get(Fixed: -((int64_t)AFI->getCalleeSavedStackSize()),
1295	Scalable: ObjectOffset);
1296	}
1297
1298	bool IsFixed = MFI.isFixedObjectIndex(ObjectIdx: FI);
1299	bool IsCSR =
1300	!IsFixed && ObjectOffset >= -((int)AFI->getCalleeSavedStackSize(MFI));
1301
1302	StackOffset ScalableOffset = {};
1303	if (!IsFixed && !IsCSR) {
1304	ScalableOffset = -SVEStackSize;
1305	} else if (FPAfterSVECalleeSaves && IsCSR) {
1306	ScalableOffset =
1307	-StackOffset::getScalable(Scalable: AFI->getSVECalleeSavedStackSize());
1308	}
1309
1310	return StackOffset::getFixed(Fixed: ObjectOffset) + ScalableOffset;
1311	}
1312
1313	StackOffset
1314	AArch64FrameLowering::getNonLocalFrameIndexReference(const MachineFunction &MF,
1315	int FI) const {
1316	return StackOffset::getFixed(Fixed: getSEHFrameIndexOffset(MF, FI));
1317	}
1318
1319	StackOffset AArch64FrameLowering::getFPOffset(const MachineFunction &MF,
1320	int64_t ObjectOffset) const {
1321	const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
1322	const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1323	const Function &F = MF.getFunction();
1324	bool IsWin64 = Subtarget.isCallingConvWin64(CC: F.getCallingConv(), IsVarArg: F.isVarArg());
1325	unsigned FixedObject =
1326	getFixedObjectSize(MF, AFI, IsWin64, /IsFunclet=/false);
1327	int64_t CalleeSaveSize = AFI->getCalleeSavedStackSize(MFI: MF.getFrameInfo());
1328	int64_t FPAdjust =
1329	CalleeSaveSize - AFI->getCalleeSaveBaseToFrameRecordOffset();
1330	return StackOffset::getFixed(Fixed: ObjectOffset + FixedObject + FPAdjust);
1331	}
1332
1333	StackOffset AArch64FrameLowering::getStackOffset(const MachineFunction &MF,
1334	int64_t ObjectOffset) const {
1335	const auto &MFI = MF.getFrameInfo();
1336	return StackOffset::getFixed(Fixed: ObjectOffset + (int64_t)MFI.getStackSize());
1337	}
1338
1339	// TODO: This function currently does not work for scalable vectors.
1340	int AArch64FrameLowering::getSEHFrameIndexOffset(const MachineFunction &MF,
1341	int FI) const {
1342	const AArch64RegisterInfo *RegInfo =
1343	MF.getSubtarget<AArch64Subtarget>().getRegisterInfo();
1344	int ObjectOffset = MF.getFrameInfo().getObjectOffset(ObjectIdx: FI);
1345	return RegInfo->getLocalAddressRegister(MF) == AArch64::FP
1346	? getFPOffset(MF, ObjectOffset).getFixed()
1347	: getStackOffset(MF, ObjectOffset).getFixed();
1348	}
1349
1350	StackOffset AArch64FrameLowering::resolveFrameIndexReference(
1351	const MachineFunction &MF, int FI, Register &FrameReg, bool PreferFP,
1352	bool ForSimm) const {
1353	const auto &MFI = MF.getFrameInfo();
1354	int64_t ObjectOffset = MFI.getObjectOffset(ObjectIdx: FI);
1355	bool isFixed = MFI.isFixedObjectIndex(ObjectIdx: FI);
1356	auto StackID = static_cast<TargetStackID::Value>(MFI.getStackID(ObjectIdx: FI));
1357	return resolveFrameOffsetReference(MF, ObjectOffset, isFixed, StackID,
1358	FrameReg, PreferFP, ForSimm);
1359	}
1360
1361	StackOffset AArch64FrameLowering::resolveFrameOffsetReference(
1362	const MachineFunction &MF, int64_t ObjectOffset, bool isFixed,
1363	TargetStackID::Value StackID, Register &FrameReg, bool PreferFP,
1364	bool ForSimm) const {
1365	const auto &MFI = MF.getFrameInfo();
1366	const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1367	const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
1368	const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
1369
1370	int64_t FPOffset = getFPOffset(MF, ObjectOffset).getFixed();
1371	int64_t Offset = getStackOffset(MF, ObjectOffset).getFixed();
1372	bool isCSR =
1373	!isFixed && ObjectOffset >= -((int)AFI->getCalleeSavedStackSize(MFI));
1374	bool isSVE = MFI.isScalableStackID(StackID);
1375
1376	StackOffset ZPRStackSize = getZPRStackSize(MF);
1377	StackOffset PPRStackSize = getPPRStackSize(MF);
1378	StackOffset SVEStackSize = ZPRStackSize + PPRStackSize;
1379
1380	// Use frame pointer to reference fixed objects. Use it for locals if
1381	// there are VLAs or a dynamically realigned SP (and thus the SP isn't
1382	// reliable as a base). Make sure useFPForScavengingIndex() does the
1383	// right thing for the emergency spill slot.
1384	bool UseFP = false;
1385	if (AFI->hasStackFrame() && !isSVE) {
1386	// We shouldn't prefer using the FP to access fixed-sized stack objects when
1387	// there are scalable (SVE) objects in between the FP and the fixed-sized
1388	// objects.
1389	PreferFP &= !SVEStackSize;
1390
1391	// Note: Keeping the following as multiple 'if' statements rather than
1392	// merging to a single expression for readability.
1393	//
1394	// Argument access should always use the FP.
1395	if (isFixed) {
1396	UseFP = hasFP(MF);
1397	} else if (isCSR && RegInfo->hasStackRealignment(MF)) {
1398	// References to the CSR area must use FP if we're re-aligning the stack
1399	// since the dynamically-sized alignment padding is between the SP/BP and
1400	// the CSR area.
1401	assert(hasFP(MF) && "Re-aligned stack must have frame pointer");
1402	UseFP = true;
1403	} else if (hasFP(MF) && !RegInfo->hasStackRealignment(MF)) {
1404	// If the FPOffset is negative and we're producing a signed immediate, we
1405	// have to keep in mind that the available offset range for negative
1406	// offsets is smaller than for positive ones. If an offset is available
1407	// via the FP and the SP, use whichever is closest.
1408	bool FPOffsetFits = !ForSimm \|\| FPOffset >= -`256`;
1409	PreferFP \|= Offset > -FPOffset && !SVEStackSize;
1410
1411	if (FPOffset >= `0`) {
1412	// If the FPOffset is positive, that'll always be best, as the SP/BP
1413	// will be even further away.
1414	UseFP = true;
1415	} else if (MFI.hasVarSizedObjects()) {
1416	// If we have variable sized objects, we can use either FP or BP, as the
1417	// SP offset is unknown. We can use the base pointer if we have one and
1418	// FP is not preferred. If not, we're stuck with using FP.
1419	bool CanUseBP = RegInfo->hasBasePointer(MF);
1420	if (FPOffsetFits && CanUseBP) // Both are ok. Pick the best.
1421	UseFP = PreferFP;
1422	else if (!CanUseBP) // Can't use BP. Forced to use FP.
1423	UseFP = true;
1424	// else we can use BP and FP, but the offset from FP won't fit.
1425	// That will make us scavenge registers which we can probably avoid by
1426	// using BP. If it won't fit for BP either, we'll scavenge anyway.
1427	} else if (MF.hasEHFunclets() && !RegInfo->hasBasePointer(MF)) {
1428	// Funclets access the locals contained in the parent's stack frame
1429	// via the frame pointer, so we have to use the FP in the parent
1430	// function.
1431	(void) Subtarget;
1432	assert(Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv(),
1433	MF.getFunction().isVarArg()) &&
1434	"Funclets should only be present on Win64");
1435	UseFP = true;
1436	} else {
1437	// We have the choice between FP and (SP or BP).
1438	if (FPOffsetFits && PreferFP) // If FP is the best fit, use it.
1439	UseFP = true;
1440	}
1441	}
1442	}
1443
1444	assert(
1445	((isFixed \|\| isCSR) \|\| !RegInfo->hasStackRealignment(MF) \|\| !UseFP) &&
1446	"In the presence of dynamic stack pointer realignment, "
1447	"non-argument/CSR objects cannot be accessed through the frame pointer");
1448
1449	bool FPAfterSVECalleeSaves = hasSVECalleeSavesAboveFrameRecord(MF);
1450
1451	if (isSVE) {
1452	StackOffset FPOffset = StackOffset::get(
1453	Fixed: -AFI->getCalleeSaveBaseToFrameRecordOffset(), Scalable: ObjectOffset);
1454	StackOffset SPOffset =
1455	SVEStackSize +
1456	StackOffset::get(Fixed: MFI.getStackSize() - AFI->getCalleeSavedStackSize(),
1457	Scalable: ObjectOffset);
1458
1459	// With split SVE objects the ObjectOffset is relative to the split area
1460	// (i.e. the PPR area or ZPR area respectively).
1461	if (AFI->hasSplitSVEObjects() && StackID == TargetStackID::ScalableVector) {
1462	// If we're accessing an SVE vector with split SVE objects...
1463	// - From the FP we need to move down past the PPR area:
1464	FPOffset -= PPRStackSize;
1465	// - From the SP we only need to move up to the ZPR area:
1466	SPOffset -= PPRStackSize;
1467	// Note: `SPOffset = SVEStackSize + ...`, so `-= PPRStackSize` results in
1468	// `SPOffset = ZPRStackSize + ...`.
1469	}
1470
1471	if (FPAfterSVECalleeSaves) {
1472	FPOffset += StackOffset::getScalable(Scalable: AFI->getSVECalleeSavedStackSize());
1473	if (-ObjectOffset <= (int64_t)AFI->getSVECalleeSavedStackSize()) {
1474	FPOffset += StackOffset::getFixed(Fixed: AFI->getCalleeSavedStackSize());
1475	SPOffset += StackOffset::getFixed(Fixed: AFI->getCalleeSavedStackSize());
1476	}
1477	}
1478
1479	// Always use the FP for SVE spills if available and beneficial.
1480	if (hasFP(MF) && (SPOffset.getFixed() \|\|
1481	FPOffset.getScalable() < SPOffset.getScalable() \|\|
1482	RegInfo->hasStackRealignment(MF))) {
1483	FrameReg = RegInfo->getFrameRegister(MF);
1484	return FPOffset;
1485	}
1486	FrameReg = RegInfo->hasBasePointer(MF) ? RegInfo->getBaseRegister()
1487	: MCRegister (AArch64::SP);
1488
1489	return SPOffset;
1490	}
1491
1492	StackOffset SVEAreaOffset = {};
1493	if (FPAfterSVECalleeSaves) {
1494	// In this stack layout, the FP is in between the callee saves and other
1495	// SVE allocations.
1496	StackOffset SVECalleeSavedStack =
1497	StackOffset::getScalable(Scalable: AFI->getSVECalleeSavedStackSize());
1498	if (UseFP) {
1499	if (isFixed)
1500	SVEAreaOffset = SVECalleeSavedStack;
1501	else if (!isCSR)
1502	SVEAreaOffset = SVECalleeSavedStack - SVEStackSize;
1503	} else {
1504	if (isFixed)
1505	SVEAreaOffset = SVEStackSize;
1506	else if (isCSR)
1507	SVEAreaOffset = SVEStackSize - SVECalleeSavedStack;
1508	}
1509	} else {
1510	if (UseFP && !(isFixed \|\| isCSR))
1511	SVEAreaOffset = -SVEStackSize;
1512	if (!UseFP && (isFixed \|\| isCSR))
1513	SVEAreaOffset = SVEStackSize;
1514	}
1515
1516	if (UseFP) {
1517	FrameReg = RegInfo->getFrameRegister(MF);
1518	return StackOffset::getFixed(Fixed: FPOffset) + SVEAreaOffset;
1519	}
1520
1521	// Use the base pointer if we have one.
1522	if (RegInfo->hasBasePointer(MF))
1523	FrameReg = RegInfo->getBaseRegister();
1524	else {
1525	assert(!MFI.hasVarSizedObjects() &&
1526	"Can't use SP when we have var sized objects.");
1527	FrameReg = AArch64::SP;
1528	// If we're using the red zone for this function, the SP won't actually
1529	// be adjusted, so the offsets will be negative. They're also all
1530	// within range of the signed 9-bit immediate instructions.
1531	if (canUseRedZone(MF))
1532	Offset -= AFI->getLocalStackSize();
1533	}
1534
1535	return StackOffset::getFixed(Fixed: Offset) + SVEAreaOffset;
1536	}
1537
1538	static RegState getPrologueDeath(MachineFunction &MF, unsigned Reg) {
1539	// Do not set a kill flag on values that are also marked as live-in. This
1540	// happens with the @llvm-returnaddress intrinsic and with arguments passed in
1541	// callee saved registers.
1542	// Omitting the kill flags is conservatively correct even if the live-in
1543	// is not used after all.
1544	bool IsLiveIn = MF.getRegInfo().isLiveIn(Reg);
1545	return getKillRegState(B: !IsLiveIn);
1546	}
1547
1548	static bool produceCompactUnwindFrame(const AArch64FrameLowering &AFL,
1549	MachineFunction &MF) {
1550	const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1551	AttributeList Attrs = MF.getFunction().getAttributes();
1552	AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1553	return Subtarget.isTargetMachO() &&
1554	!(Subtarget.getTargetLowering()->supportSwiftError() &&
1555	Attrs.hasAttrSomewhere(Kind: Attribute::SwiftError)) &&
1556	MF.getFunction().getCallingConv() != CallingConv::SwiftTail &&
1557	!AFL.requiresSaveVG(MF) && !AFI->isSVECC();
1558	}
1559
1560	static bool invalidateWindowsRegisterPairing(bool SpillExtendedVolatile,
1561	unsigned SpillCount, unsigned Reg1,
1562	unsigned Reg2, bool NeedsWinCFI,
1563	const TargetRegisterInfo *TRI) {
1564	// If we are generating register pairs for a Windows function that requires
1565	// EH support, then pair consecutive registers only. There are no unwind
1566	// opcodes for saves/restores of non-consecutive register pairs.
1567	// The unwind opcodes are save_regp, save_regp_x, save_fregp, save_frepg_x,
1568	// save_lrpair.
1569	// https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
1570
1571	if (Reg2 == AArch64::FP)
1572	return true;
1573	if (!NeedsWinCFI)
1574	return false;
1575
1576	// ARM64EC introduced `save_any_regp`, which expects 16-byte alignment.
1577	// This is handled by only allowing paired spills for registers spilled at
1578	// even positions (which should be 16-byte aligned, as other GPRs/FPRs are
1579	// 8-bytes). We carve out an exception for {FP,LR}, which does not require
1580	// 16-byte alignment in the uop representation.
1581	if (TRI->getEncodingValue(Reg: Reg2) == TRI->getEncodingValue(Reg: Reg1) + `1`)
1582	return SpillExtendedVolatile
1583	? !((Reg1 == AArch64::FP && Reg2 == AArch64::LR) \|\|
1584	(SpillCount % `2`) == `0`)
1585	: false;
1586
1587	// If pairing a GPR with LR, the pair can be described by the save_lrpair
1588	// opcode. The save_lrpair opcode requires the first register to be odd.
1589	if (Reg1 >= AArch64::X19 && Reg1 <= AArch64::X27 &&
1590	(Reg1 - AArch64::X19) % `2` == `0` && Reg2 == AArch64::LR)
1591	return false;
1592	return true;
1593	}
1594
1595	/// Returns true if Reg1 and Reg2 cannot be paired using a ldp/stp instruction.
1596	/// WindowsCFI requires that only consecutive registers can be paired.
1597	/// LR and FP need to be allocated together when the frame needs to save
1598	/// the frame-record. This means any other register pairing with LR is invalid.
1599	static bool invalidateRegisterPairing(bool SpillExtendedVolatile,
1600	unsigned SpillCount, unsigned Reg1,
1601	unsigned Reg2, bool UsesWinAAPCS,
1602	bool NeedsWinCFI, bool NeedsFrameRecord,
1603	const TargetRegisterInfo *TRI) {
1604	if (UsesWinAAPCS)
1605	return invalidateWindowsRegisterPairing(SpillExtendedVolatile, SpillCount,
1606	Reg1, Reg2, NeedsWinCFI, TRI);
1607
1608	// If we need to store the frame record, don't pair any register
1609	// with LR other than FP.
1610	if (NeedsFrameRecord)
1611	return Reg2 == AArch64::LR;
1612
1613	return false;
1614	}
1615
1616	namespace {
1617
1618	struct RegPairInfo {
1619	Register Reg1;
1620	Register Reg2;
1621	int FrameIdx;
1622	int Offset;
1623	enum RegType { GPR, FPR64, FPR128, PPR, ZPR, VG } Type;
1624	const TargetRegisterClass *RC;
1625
1626	RegPairInfo() = default;
1627
1628	bool isPaired() const { return Reg2.isValid(); }
1629
1630	bool isScalable() const { return Type == PPR \|\| Type == ZPR; }
1631	};
1632
1633	} // end anonymous namespace
1634
1635	MCRegister findFreePredicateReg(BitVector &SavedRegs) {
1636	for (unsigned PReg = AArch64::P8; PReg <= AArch64::P15; ++PReg) {
1637	if (SavedRegs.test(Idx: PReg)) {
1638	unsigned PNReg = PReg - AArch64::P0 + AArch64::PN0;
1639	return MCRegister (PNReg);
1640	}
1641	}
1642	return MCRegister ();
1643	}
1644
1645	// The multivector LD/ST are available only for SME or SVE2p1 targets
1646	bool enableMultiVectorSpillFill(const AArch64Subtarget &Subtarget,
1647	MachineFunction &MF) {
1648	if (DisableMultiVectorSpillFill)
1649	return false;
1650
1651	SMEAttrs FuncAttrs = MF.getInfo<AArch64FunctionInfo>()->getSMEFnAttrs();
1652	bool IsLocallyStreaming =
1653	FuncAttrs.hasStreamingBody() && !FuncAttrs.hasStreamingInterface();
1654
1655	// Only when in streaming mode SME2 instructions can be safely used.
1656	// It is not safe to use SME2 instructions when in streaming compatible or
1657	// locally streaming mode.
1658	return Subtarget.hasSVE2p1() \|\|
1659	(Subtarget.hasSME2() &&
1660	(!IsLocallyStreaming && Subtarget.isStreaming()));
1661	}
1662
1663	void computeCalleeSaveRegisterPairs(const AArch64FrameLowering &AFL,
1664	MachineFunction &MF,
1665	ArrayRef<CalleeSavedInfo> CSI,
1666	const TargetRegisterInfo *TRI,
1667	SmallVectorImpl<RegPairInfo> &RegPairs,
1668	bool NeedsFrameRecord) {
1669
1670	if (CSI.empty())
1671	return;
1672
1673	bool IsWindows = isTargetWindows(MF);
1674	AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1675	unsigned StackHazardSize = getStackHazardSize(MF);
1676	MachineFrameInfo &MFI = MF.getFrameInfo();
1677	CallingConv::ID CC = MF.getFunction().getCallingConv();
1678	unsigned Count = CSI.size();
1679	(void)CC;
1680	// MachO's compact unwind format relies on all registers being stored in
1681	// pairs.
1682	assert((!produceCompactUnwindFrame(AFL, MF) \|\|
1683	CC == CallingConv::PreserveMost \|\| CC == CallingConv::PreserveAll \|\|
1684	CC == CallingConv::CXX_FAST_TLS \|\| CC == CallingConv::Win64 \|\|
1685	(Count & `1`) == `0`) &&
1686	"Odd number of callee-saved regs to spill!");
1687	int ByteOffset = AFI->getCalleeSavedStackSize();
1688	int StackFillDir = -`1`;
1689	int RegInc = `1`;
1690	unsigned FirstReg = `0`;
1691	if (IsWindows) {
1692	// For WinCFI, fill the stack from the bottom up.
1693	ByteOffset = `0`;
1694	StackFillDir = `1`;
1695	// As the CSI array is reversed to match PrologEpilogInserter, iterate
1696	// backwards, to pair up registers starting from lower numbered registers.
1697	RegInc = -`1`;
1698	FirstReg = Count - `1`;
1699	}
1700
1701	bool FPAfterSVECalleeSaves = AFL.hasSVECalleeSavesAboveFrameRecord(MF);
1702	// Windows AAPCS has x9-x15 as volatile registers, x16-x17 as intra-procedural
1703	// scratch, x18 as platform reserved. However, clang has extended calling
1704	// convensions such as preserve_most and preserve_all which treat these as
1705	// CSR. As such, the ARM64 unwind uOPs bias registers by 19. We use ARM64EC
1706	// uOPs which have separate restrictions. We need to check for that.
1707	//
1708	// NOTE: we currently do not account for the D registers as LLVM does not
1709	// support non-ABI compliant D register spills.
1710	bool SpillExtendedVolatile =
1711	IsWindows && llvm::any_of(Range&: CSI, P: [](const CalleeSavedInfo &CSI) {
1712	const auto &Reg = CSI.getReg();
1713	return Reg >= AArch64::X0 && Reg <= AArch64::X18;
1714	});
1715
1716	int ZPRByteOffset = `0`;
1717	int PPRByteOffset = `0`;
1718	bool SplitPPRs = AFI->hasSplitSVEObjects();
1719	if (SplitPPRs) {
1720	ZPRByteOffset = AFI->getZPRCalleeSavedStackSize();
1721	PPRByteOffset = AFI->getPPRCalleeSavedStackSize();
1722	} else if (!FPAfterSVECalleeSaves) {
1723	ZPRByteOffset =
1724	AFI->getZPRCalleeSavedStackSize() + AFI->getPPRCalleeSavedStackSize();
1725	// Unused: Everything goes in ZPR space.
1726	PPRByteOffset = `0`;
1727	}
1728
1729	bool NeedGapToAlignStack = AFI->hasCalleeSaveStackFreeSpace();
1730	Register LastReg = `0`;
1731	bool HasCSHazardPadding = AFI->hasStackHazardSlotIndex() && !SplitPPRs;
1732
1733	// When iterating backwards, the loop condition relies on unsigned wraparound.
1734	for (unsigned i = FirstReg; i < Count; i += RegInc) {
1735	RegPairInfo RPI;
1736	RPI.Reg1 = CSI [i].getReg();
1737
1738	if (AArch64::GPR64RegClass.contains(Reg: RPI.Reg1)) {
1739	RPI.Type = RegPairInfo::GPR;
1740	RPI.RC = &AArch64::GPR64RegClass;
1741	} else if (AArch64::FPR64RegClass.contains(Reg: RPI.Reg1)) {
1742	RPI.Type = RegPairInfo::FPR64;
1743	RPI.RC = &AArch64::FPR64RegClass;
1744	} else if (AArch64::FPR128RegClass.contains(Reg: RPI.Reg1)) {
1745	RPI.Type = RegPairInfo::FPR128;
1746	RPI.RC = &AArch64::FPR128RegClass;
1747	} else if (AArch64::ZPRRegClass.contains(Reg: RPI.Reg1)) {
1748	RPI.Type = RegPairInfo::ZPR;
1749	RPI.RC = &AArch64::ZPRRegClass;
1750	} else if (AArch64::PPRRegClass.contains(Reg: RPI.Reg1)) {
1751	RPI.Type = RegPairInfo::PPR;
1752	RPI.RC = &AArch64::PPRRegClass;
1753	} else if (RPI.Reg1 == AArch64::VG) {
1754	RPI.Type = RegPairInfo::VG;
1755	RPI.RC = &AArch64::FIXED_REGSRegClass;
1756	} else {
1757	llvm_unreachable("Unsupported register class.");
1758	}
1759
1760	int &ScalableByteOffset = RPI.Type == RegPairInfo::PPR && SplitPPRs
1761	? PPRByteOffset
1762	: ZPRByteOffset;
1763
1764	// Add the stack hazard size as we transition from GPR->FPR CSRs.
1765	if (HasCSHazardPadding &&
1766	(!LastReg \|\| !AArch64InstrInfo::isFpOrNEON(Reg: LastReg)) &&
1767	AArch64InstrInfo::isFpOrNEON(Reg: RPI.Reg1))
1768	ByteOffset += StackFillDir * StackHazardSize;
1769	LastReg = RPI.Reg1;
1770
1771	bool NeedsWinCFI = AFL.needsWinCFI(MF);
1772	int Scale = TRI->getSpillSize(RC: *RPI.RC);
1773	// Add the next reg to the pair if it is in the same register class.
1774	if (unsigned(i + RegInc) < Count && !HasCSHazardPadding) {
1775	MCRegister NextReg = CSI [i + RegInc].getReg();
1776	unsigned SpillCount = NeedsWinCFI ? FirstReg - i : i;
1777	switch (RPI.Type) {
1778	case RegPairInfo::GPR:
1779	if (AArch64::GPR64RegClass.contains(Reg: NextReg) &&
1780	!invalidateRegisterPairing(SpillExtendedVolatile, SpillCount,
1781	Reg1: RPI.Reg1, Reg2: NextReg, UsesWinAAPCS: IsWindows,
1782	NeedsWinCFI, NeedsFrameRecord, TRI))
1783	RPI.Reg2 = NextReg;
1784	break;
1785	case RegPairInfo::FPR64:
1786	if (AArch64::FPR64RegClass.contains(Reg: NextReg) &&
1787	!invalidateRegisterPairing(SpillExtendedVolatile, SpillCount,
1788	Reg1: RPI.Reg1, Reg2: NextReg, UsesWinAAPCS: IsWindows,
1789	NeedsWinCFI, NeedsFrameRecord, TRI))
1790	RPI.Reg2 = NextReg;
1791	break;
1792	case RegPairInfo::FPR128:
1793	if (AArch64::FPR128RegClass.contains(Reg: NextReg))
1794	RPI.Reg2 = NextReg;
1795	break;
1796	case RegPairInfo::PPR:
1797	break;
1798	case RegPairInfo::ZPR:
1799	if (AFI->getPredicateRegForFillSpill() != `0` &&
1800	((RPI.Reg1 - AArch64::Z0) & `1`) == `0` && (NextReg == RPI.Reg1 + `1`)) {
1801	// Calculate offset of register pair to see if pair instruction can be
1802	// used.
1803	int Offset = (ScalableByteOffset + StackFillDir * `2` * Scale) / Scale;
1804	if ((-`16` <= Offset && Offset <= `14`) && (Offset % `2` == `0`))
1805	RPI.Reg2 = NextReg;
1806	}
1807	break;
1808	case RegPairInfo::VG:
1809	break;
1810	}
1811	}
1812
1813	// GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI
1814	// list to come in sorted by frame index so that we can issue the store
1815	// pair instructions directly. Assert if we see anything otherwise.
1816	//
1817	// The order of the registers in the list is controlled by
1818	// getCalleeSavedRegs(), so they will always be in-order, as well.
1819	assert((!RPI.isPaired() \|\|
1820	(CSI[i].getFrameIdx() + RegInc == CSI[i + RegInc].getFrameIdx())) &&
1821	"Out of order callee saved regs!");
1822
1823	assert((!RPI.isPaired() \|\| !NeedsFrameRecord \|\| RPI.Reg2 != AArch64::FP \|\|
1824	RPI.Reg1 == AArch64::LR) &&
1825	"FrameRecord must be allocated together with LR");
1826
1827	// Windows AAPCS has FP and LR reversed.
1828	assert((!RPI.isPaired() \|\| !NeedsFrameRecord \|\| RPI.Reg1 != AArch64::FP \|\|
1829	RPI.Reg2 == AArch64::LR) &&
1830	"FrameRecord must be allocated together with LR");
1831
1832	// MachO's compact unwind format relies on all registers being stored in
1833	// adjacent register pairs.
1834	assert((!produceCompactUnwindFrame(AFL, MF) \|\|
1835	CC == CallingConv::PreserveMost \|\| CC == CallingConv::PreserveAll \|\|
1836	CC == CallingConv::CXX_FAST_TLS \|\| CC == CallingConv::Win64 \|\|
1837	(RPI.isPaired() &&
1838	((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) \|\|
1839	RPI.Reg1 + `1` == RPI.Reg2))) &&
1840	"Callee-save registers not saved as adjacent register pair!");
1841
1842	RPI.FrameIdx = CSI [i].getFrameIdx();
1843	if (IsWindows &&
1844	RPI.isPaired()) // RPI.FrameIdx must be the lower index of the pair
1845	RPI.FrameIdx = CSI [i + RegInc].getFrameIdx();
1846
1847	// Realign the scalable offset if necessary. This is relevant when
1848	// spilling predicates on Windows.
1849	if (RPI.isScalable() && ScalableByteOffset % Scale != `0`) {
1850	ScalableByteOffset = alignTo(Value: ScalableByteOffset, Align: Scale);
1851	}
1852
1853	int OffsetPre = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
1854	assert(OffsetPre % Scale == `0`);
1855
1856	if (RPI.isScalable())
1857	ScalableByteOffset += StackFillDir * (RPI.isPaired() ? `2` * Scale : Scale);
1858	else
1859	ByteOffset += StackFillDir * (RPI.isPaired() ? `2` * Scale : Scale);
1860
1861	// Swift's async context is directly before FP, so allocate an extra
1862	// 8 bytes for it.
1863	if (NeedsFrameRecord && AFI->hasSwiftAsyncContext() &&
1864	((!IsWindows && RPI.Reg2 == AArch64::FP) \|\|
1865	(IsWindows && RPI.Reg2 == AArch64::LR)))
1866	ByteOffset += StackFillDir * `8`;
1867
1868	// Round up size of non-pair to pair size if we need to pad the
1869	// callee-save area to ensure 16-byte alignment.
1870	if (NeedGapToAlignStack && !IsWindows && !RPI.isScalable() &&
1871	RPI.Type != RegPairInfo::FPR128 && !RPI.isPaired() &&
1872	ByteOffset % `16` != `0`) {
1873	ByteOffset += `8` * StackFillDir;
1874	assert(MFI.getObjectAlign(RPI.FrameIdx) <= Align(`16`));
1875	// A stack frame with a gap looks like this, bottom up:
1876	// d9, d8. x21, gap, x20, x19.
1877	// Set extra alignment on the x21 object to create the gap above it.
1878	MFI.setObjectAlignment(ObjectIdx: RPI.FrameIdx, Alignment: Align (`16`));
1879	NeedGapToAlignStack = false;
1880	}
1881
1882	int OffsetPost = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
1883	assert(OffsetPost % Scale == `0`);
1884	// If filling top down (default), we want the offset after incrementing it.
1885	// If filling bottom up (WinCFI) we need the original offset.
1886	int Offset = IsWindows ? OffsetPre : OffsetPost;
1887
1888	// The FP, LR pair goes 8 bytes into our expanded 24-byte slot so that the
1889	// Swift context can directly precede FP.
1890	if (NeedsFrameRecord && AFI->hasSwiftAsyncContext() &&
1891	((!IsWindows && RPI.Reg2 == AArch64::FP) \|\|
1892	(IsWindows && RPI.Reg2 == AArch64::LR)))
1893	Offset += `8`;
1894	RPI.Offset = Offset / Scale;
1895
1896	assert((!RPI.isPaired() \|\|
1897	(!RPI.isScalable() && RPI.Offset >= -`64` && RPI.Offset <= `63`) \|\|
1898	(RPI.isScalable() && RPI.Offset >= -`256` && RPI.Offset <= `255`)) &&
1899	"Offset out of bounds for LDP/STP immediate");
1900
1901	auto isFrameRecord = [&] {
1902	if (RPI.isPaired())
1903	return IsWindows ? RPI.Reg1 == AArch64::FP && RPI.Reg2 == AArch64::LR
1904	: RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP;
1905	// Otherwise, look for the frame record as two unpaired registers. This is
1906	// needed for -aarch64-stack-hazard-size=<val>, which disables register
1907	// pairing (as the padding may be too large for the LDP/STP offset). Note:
1908	// On Windows, this check works out as current reg == FP, next reg == LR,
1909	// and on other platforms current reg == FP, previous reg == LR. This
1910	// works out as the correct pre-increment or post-increment offsets
1911	// respectively.
1912	return i > `0` && RPI.Reg1 == AArch64::FP &&
1913	CSI [i - `1`].getReg() == AArch64::LR;
1914	};
1915
1916	// Save the offset to frame record so that the FP register can point to the
1917	// innermost frame record (spilled FP and LR registers).
1918	if (NeedsFrameRecord && isFrameRecord ())
1919	AFI->setCalleeSaveBaseToFrameRecordOffset(Offset);
1920
1921	RegPairs.push_back(Elt: RPI);
1922	if (RPI.isPaired())
1923	i += RegInc;
1924	}
1925	if (IsWindows) {
1926	// If we need an alignment gap in the stack, align the topmost stack
1927	// object. A stack frame with a gap looks like this, bottom up:
1928	// x19, d8. d9, gap.
1929	// Set extra alignment on the topmost stack object (the first element in
1930	// CSI, which goes top down), to create the gap above it.
1931	if (AFI->hasCalleeSaveStackFreeSpace())
1932	MFI.setObjectAlignment(ObjectIdx: CSI [`0`].getFrameIdx(), Alignment: Align (`16`));
1933	// We iterated bottom up over the registers; flip RegPairs back to top
1934	// down order.
1935	std::reverse(first: RegPairs.begin(), last: RegPairs.end());
1936	}
1937	}
1938
1939	bool AArch64FrameLowering::spillCalleeSavedRegisters(
1940	MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
1941	ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo TRI) const* {
1942	MachineFunction &MF = *MBB.getParent();
1943	const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1944	auto &TLI = *Subtarget.getTargetLowering();
1945	const AArch64InstrInfo &TII = *Subtarget.getInstrInfo();
1946	bool NeedsWinCFI = needsWinCFI(MF);
1947	DebugLoc DL;
1948	SmallVector<RegPairInfo, `8`> RegPairs;
1949
1950	computeCalleeSaveRegisterPairs(AFL: *this, MF, CSI, TRI, RegPairs, NeedsFrameRecord: hasFP(MF));
1951
1952	MachineRegisterInfo &MRI = MF.getRegInfo();
1953	// Refresh the reserved regs in case there are any potential changes since the
1954	// last freeze.
1955	MRI.freezeReservedRegs();
1956
1957	if (homogeneousPrologEpilog(MF)) {
1958	auto MIB = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AArch64::HOM_Prolog))
1959	.setMIFlag(MachineInstr::FrameSetup);
1960
1961	for (auto &RPI : RegPairs) {
1962	MIB.addReg(RegNo: RPI.Reg1);
1963	MIB.addReg(RegNo: RPI.Reg2);
1964
1965	// Update register live in.
1966	if (!MRI.isReserved(PhysReg: RPI.Reg1))
1967	MBB.addLiveIn(PhysReg: RPI.Reg1);
1968	if (RPI.isPaired() && !MRI.isReserved(PhysReg: RPI.Reg2))
1969	MBB.addLiveIn(PhysReg: RPI.Reg2);
1970	}
1971	return true;
1972	}
1973	bool PTrueCreated = false;
1974	for (const RegPairInfo &RPI : llvm::reverse(C&: RegPairs)) {
1975	Register Reg1 = RPI.Reg1;
1976	Register Reg2 = RPI.Reg2;
1977	unsigned StrOpc;
1978
1979	// Issue sequence of spills for cs regs. The first spill may be converted
1980	// to a pre-decrement store later by emitPrologue if the callee-save stack
1981	// area allocation can't be combined with the local stack area allocation.
1982	// For example:
1983	// stp x22, x21, [sp, #0] // addImm(+0)
1984	// stp x20, x19, [sp, #16] // addImm(+2)
1985	// stp fp, lr, [sp, #32] // addImm(+4)
1986	// Rationale: This sequence saves uop updates compared to a sequence of
1987	// pre-increment spills like stp xi,xj,[sp,#-16]!
1988	// Note: Similar rationale and sequence for restores in epilog.
1989	unsigned Size = TRI->getSpillSize(RC: *RPI.RC);
1990	Align Alignment = TRI->getSpillAlign(RC: *RPI.RC);
1991	switch (RPI.Type) {
1992	case RegPairInfo::GPR:
1993	StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
1994	break;
1995	case RegPairInfo::FPR64:
1996	StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
1997	break;
1998	case RegPairInfo::FPR128:
1999	StrOpc = RPI.isPaired() ? AArch64::STPQi : AArch64::STRQui;
2000	break;
2001	case RegPairInfo::ZPR:
2002	StrOpc = RPI.isPaired() ? AArch64::ST1B_2Z_IMM : AArch64::STR_ZXI;
2003	break;
2004	case RegPairInfo::PPR:
2005	StrOpc = AArch64::STR_PXI;
2006	break;
2007	case RegPairInfo::VG:
2008	StrOpc = AArch64::STRXui;
2009	break;
2010	}
2011
2012	Register X0Scratch;
2013	llvm::scope_exit RestoreX0([&] {
2014	if (X0Scratch != AArch64::NoRegister)
2015	BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: AArch64::X0)
2016	.addReg(RegNo: X0Scratch)
2017	.setMIFlag(MachineInstr::FrameSetup);
2018	});
2019
2020	if (Reg1 == AArch64::VG) {
2021	// Find an available register to store value of VG to.
2022	Reg1 = findScratchNonCalleeSaveRegister(MBB: &MBB, HasCall: true);
2023	assert(Reg1 != AArch64::NoRegister);
2024	if (MF.getSubtarget<AArch64Subtarget>().hasSVE()) {
2025	BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AArch64::CNTD_XPiI), DestReg: Reg1)
2026	.addImm(Val: `31`)
2027	.addImm(Val: `1`)
2028	.setMIFlag(MachineInstr::FrameSetup);
2029	} else {
2030	const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
2031	if (any_of(Range: MBB.liveins(),
2032	P: [&STI](const MachineBasicBlock::RegisterMaskPair &LiveIn) {
2033	return STI.getRegisterInfo()->isSuperOrSubRegisterEq(
2034	RegA: AArch64::X0, RegB: LiveIn.PhysReg);
2035	})) {
2036	X0Scratch = Reg1;
2037	BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X0Scratch)
2038	.addReg(RegNo: AArch64::X0)
2039	.setMIFlag(MachineInstr::FrameSetup);
2040	}
2041
2042	RTLIB::Libcall LC = RTLIB::SMEABI_GET_CURRENT_VG;
2043	const uint32_t *RegMask =
2044	TRI->getCallPreservedMask(MF, TLI.getLibcallCallingConv(Call: LC));
2045	BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AArch64::BL))
2046	.addExternalSymbol(FnName: TLI.getLibcallName(Call: LC))
2047	.addRegMask(Mask: RegMask)
2048	.addReg(RegNo: AArch64::X0, Flags: RegState::ImplicitDefine)
2049	.setMIFlag(MachineInstr::FrameSetup);
2050	Reg1 = AArch64::X0;
2051	}
2052	}
2053
2054	LLVM_DEBUG({
2055	dbgs() << "CSR spill: (" << printReg(Reg1, TRI);
2056	if (RPI.isPaired())
2057	dbgs() << ", " << printReg(Reg2, TRI);
2058	dbgs() << ") -> fi#(" << RPI.FrameIdx;
2059	if (RPI.isPaired())
2060	dbgs() << ", " << RPI.FrameIdx + `1`;
2061	dbgs() << ")\n";
2062	});
2063
2064	assert((!isTargetWindows(MF) \|\|
2065	!(Reg1 == AArch64::LR && Reg2 == AArch64::FP)) &&
2066	"Windows unwdinding requires a consecutive (FP,LR) pair");
2067	// Windows unwind codes require consecutive registers if registers are
2068	// paired. Make the switch here, so that the code below will save (x,x+1)
2069	// and not (x+1,x).
2070	unsigned FrameIdxReg1 = RPI.FrameIdx;
2071	unsigned FrameIdxReg2 = RPI.FrameIdx + `1`;
2072	if (isTargetWindows(MF) && RPI.isPaired()) {
2073	std::swap(a&: Reg1, b&: Reg2);
2074	std::swap(a&: FrameIdxReg1, b&: FrameIdxReg2);
2075	}
2076
2077	if (RPI.isPaired() && RPI.isScalable()) {
2078	[[maybe_unused]] const AArch64Subtarget &Subtarget =
2079	MF.getSubtarget<AArch64Subtarget>();
2080	AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
2081	unsigned PnReg = AFI->getPredicateRegForFillSpill();
2082	assert((PnReg != `0` && enableMultiVectorSpillFill(Subtarget, MF)) &&
2083	"Expects SVE2.1 or SME2 target and a predicate register");
2084	#ifdef EXPENSIVE_CHECKS
2085	auto IsPPR = [](const RegPairInfo &c) {
2086	return c.Reg1 == RegPairInfo::PPR;
2087	};
2088	auto PPRBegin = std::find_if(RegPairs.begin(), RegPairs.end(), IsPPR);
2089	auto IsZPR = [](const RegPairInfo &c) {
2090	return c.Type == RegPairInfo::ZPR;
2091	};
2092	auto ZPRBegin = std::find_if(RegPairs.begin(), RegPairs.end(), IsZPR);
2093	assert(!(PPRBegin < ZPRBegin) &&
2094	"Expected callee save predicate to be handled first");
2095	#endif
2096	if (!PTrueCreated) {
2097	PTrueCreated = true;
2098	BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AArch64::PTRUE_C_B), DestReg: PnReg)
2099	.setMIFlags(MachineInstr::FrameSetup);
2100	}
2101	MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: StrOpc));
2102	if (!MRI.isReserved(PhysReg: Reg1))
2103	MBB.addLiveIn(PhysReg: Reg1);
2104	if (!MRI.isReserved(PhysReg: Reg2))
2105	MBB.addLiveIn(PhysReg: Reg2);
2106	MIB.addReg(/PairRegs/ RegNo: AArch64::Z0_Z1 + (RPI.Reg1 - AArch64::Z0));
2107	MIB.addMemOperand(MMO: MF.getMachineMemOperand(
2108	PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIdxReg2),
2109	F: MachineMemOperand::MOStore, Size, BaseAlignment: Alignment));
2110	MIB.addReg(RegNo: PnReg);
2111	MIB.addReg(RegNo: AArch64::SP)
2112	.addImm(Val: RPI.Offset / `2`) // [sp, #imm2vscale],
2113	// where 2vscale is implicit*
2114	.setMIFlag(MachineInstr::FrameSetup);
2115	MIB.addMemOperand(MMO: MF.getMachineMemOperand(
2116	PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIdxReg1),
2117	F: MachineMemOperand::MOStore, Size, BaseAlignment: Alignment));
2118	if (NeedsWinCFI)
2119	insertSEH(MBBI: MIB, TII, Flag: MachineInstr::FrameSetup);
2120	} else { // The code when the pair of ZReg is not present
2121	MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: StrOpc));
2122	if (!MRI.isReserved(PhysReg: Reg1))
2123	MBB.addLiveIn(PhysReg: Reg1);
2124	if (RPI.isPaired()) {
2125	if (!MRI.isReserved(PhysReg: Reg2))
2126	MBB.addLiveIn(PhysReg: Reg2);
2127	MIB.addReg(RegNo: Reg2, Flags: getPrologueDeath(MF, Reg: Reg2));
2128	MIB.addMemOperand(MMO: MF.getMachineMemOperand(
2129	PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIdxReg2),
2130	F: MachineMemOperand::MOStore, Size, BaseAlignment: Alignment));
2131	}
2132	MIB.addReg(RegNo: Reg1, Flags: getPrologueDeath(MF, Reg: Reg1))
2133	.addReg(RegNo: AArch64::SP)
2134	.addImm(Val: RPI.Offset) // [sp, #offsetvscale],*
2135	// where factorvscale is implicit*
2136	.setMIFlag(MachineInstr::FrameSetup);
2137	MIB.addMemOperand(MMO: MF.getMachineMemOperand(
2138	PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIdxReg1),
2139	F: MachineMemOperand::MOStore, Size, BaseAlignment: Alignment));
2140	if (NeedsWinCFI)
2141	insertSEH(MBBI: MIB, TII, Flag: MachineInstr::FrameSetup);
2142	}
2143	// Update the StackIDs of the SVE stack slots.
2144	MachineFrameInfo &MFI = MF.getFrameInfo();
2145	if (RPI.Type == RegPairInfo::ZPR) {
2146	MFI.setStackID(ObjectIdx: FrameIdxReg1, ID: TargetStackID::ScalableVector);
2147	if (RPI.isPaired())
2148	MFI.setStackID(ObjectIdx: FrameIdxReg2, ID: TargetStackID::ScalableVector);
2149	} else if (RPI.Type == RegPairInfo::PPR) {
2150	MFI.setStackID(ObjectIdx: FrameIdxReg1, ID: TargetStackID::ScalablePredicateVector);
2151	if (RPI.isPaired())
2152	MFI.setStackID(ObjectIdx: FrameIdxReg2, ID: TargetStackID::ScalablePredicateVector);
2153	}
2154	}
2155	return true;
2156	}
2157
2158	bool AArch64FrameLowering::restoreCalleeSavedRegisters(
2159	MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
2160	MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo TRI) const* {
2161	MachineFunction &MF = *MBB.getParent();
2162	const AArch64InstrInfo &TII =
2163	*MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
2164	DebugLoc DL;
2165	SmallVector<RegPairInfo, `8`> RegPairs;
2166	bool NeedsWinCFI = needsWinCFI(MF);
2167
2168	if (MBBI != MBB.end())
2169	DL = MBBI ->getDebugLoc();
2170
2171	computeCalleeSaveRegisterPairs(AFL: *this, MF, CSI, TRI, RegPairs, NeedsFrameRecord: hasFP(MF));
2172	if (homogeneousPrologEpilog(MF, Exit: &MBB)) {
2173	auto MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AArch64::HOM_Epilog))
2174	.setMIFlag(MachineInstr::FrameDestroy);
2175	for (auto &RPI : RegPairs) {
2176	MIB.addReg(RegNo: RPI.Reg1, Flags: RegState::Define);
2177	MIB.addReg(RegNo: RPI.Reg2, Flags: RegState::Define);
2178	}
2179	return true;
2180	}
2181
2182	// For performance reasons restore SVE register in increasing order
2183	auto IsPPR = [](const RegPairInfo &c) { return c.Type == RegPairInfo::PPR; };
2184	auto PPRBegin = llvm::find_if(Range&: RegPairs, P: IsPPR);
2185	auto PPREnd = std::find_if_not(first: PPRBegin, last: RegPairs.end(), pred: IsPPR);
2186	std::reverse(first: PPRBegin, last: PPREnd);
2187	auto IsZPR = [](const RegPairInfo &c) { return c.Type == RegPairInfo::ZPR; };
2188	auto ZPRBegin = llvm::find_if(Range&: RegPairs, P: IsZPR);
2189	auto ZPREnd = std::find_if_not(first: ZPRBegin, last: RegPairs.end(), pred: IsZPR);
2190	std::reverse(first: ZPRBegin, last: ZPREnd);
2191
2192	bool PTrueCreated = false;
2193	for (const RegPairInfo &RPI : RegPairs) {
2194	Register Reg1 = RPI.Reg1;
2195	Register Reg2 = RPI.Reg2;
2196
2197	// Issue sequence of restores for cs regs. The last restore may be converted
2198	// to a post-increment load later by emitEpilogue if the callee-save stack
2199	// area allocation can't be combined with the local stack area allocation.
2200	// For example:
2201	// ldp fp, lr, [sp, #32] // addImm(+4)
2202	// ldp x20, x19, [sp, #16] // addImm(+2)
2203	// ldp x22, x21, [sp, #0] // addImm(+0)
2204	// Note: see comment in spillCalleeSavedRegisters()
2205	unsigned LdrOpc;
2206	unsigned Size = TRI->getSpillSize(RC: *RPI.RC);
2207	Align Alignment = TRI->getSpillAlign(RC: *RPI.RC);
2208	switch (RPI.Type) {
2209	case RegPairInfo::GPR:
2210	LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
2211	break;
2212	case RegPairInfo::FPR64:
2213	LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
2214	break;
2215	case RegPairInfo::FPR128:
2216	LdrOpc = RPI.isPaired() ? AArch64::LDPQi : AArch64::LDRQui;
2217	break;
2218	case RegPairInfo::ZPR:
2219	LdrOpc = RPI.isPaired() ? AArch64::LD1B_2Z_IMM : AArch64::LDR_ZXI;
2220	break;
2221	case RegPairInfo::PPR:
2222	LdrOpc = AArch64::LDR_PXI;
2223	break;
2224	case RegPairInfo::VG:
2225	continue;
2226	}
2227	LLVM_DEBUG({
2228	dbgs() << "CSR restore: (" << printReg(Reg1, TRI);
2229	if (RPI.isPaired())
2230	dbgs() << ", " << printReg(Reg2, TRI);
2231	dbgs() << ") -> fi#(" << RPI.FrameIdx;
2232	if (RPI.isPaired())
2233	dbgs() << ", " << RPI.FrameIdx + `1`;
2234	dbgs() << ")\n";
2235	});
2236
2237	// Windows unwind codes require consecutive registers if registers are
2238	// paired. Make the switch here, so that the code below will save (x,x+1)
2239	// and not (x+1,x).
2240	unsigned FrameIdxReg1 = RPI.FrameIdx;
2241	unsigned FrameIdxReg2 = RPI.FrameIdx + `1`;
2242	if (isTargetWindows(MF) && RPI.isPaired()) {
2243	std::swap(a&: Reg1, b&: Reg2);
2244	std::swap(a&: FrameIdxReg1, b&: FrameIdxReg2);
2245	}
2246
2247	AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
2248	if (RPI.isPaired() && RPI.isScalable()) {
2249	[[maybe_unused]] const AArch64Subtarget &Subtarget =
2250	MF.getSubtarget<AArch64Subtarget>();
2251	unsigned PnReg = AFI->getPredicateRegForFillSpill();
2252	assert((PnReg != `0` && enableMultiVectorSpillFill(Subtarget, MF)) &&
2253	"Expects SVE2.1 or SME2 target and a predicate register");
2254	#ifdef EXPENSIVE_CHECKS
2255	assert(!(PPRBegin < ZPRBegin) &&
2256	"Expected callee save predicate to be handled first");
2257	#endif
2258	if (!PTrueCreated) {
2259	PTrueCreated = true;
2260	BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AArch64::PTRUE_C_B), DestReg: PnReg)
2261	.setMIFlags(MachineInstr::FrameDestroy);
2262	}
2263	MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: LdrOpc));
2264	MIB.addReg(/PairRegs/ RegNo: AArch64::Z0_Z1 + (RPI.Reg1 - AArch64::Z0),
2265	Flags: getDefRegState(B: true));
2266	MIB.addMemOperand(MMO: MF.getMachineMemOperand(
2267	PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIdxReg2),
2268	F: MachineMemOperand::MOLoad, Size, BaseAlignment: Alignment));
2269	MIB.addReg(RegNo: PnReg);
2270	MIB.addReg(RegNo: AArch64::SP)
2271	.addImm(Val: RPI.Offset / `2`) // [sp, #imm2vscale]
2272	// where 2vscale is implicit*
2273	.setMIFlag(MachineInstr::FrameDestroy);
2274	MIB.addMemOperand(MMO: MF.getMachineMemOperand(
2275	PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIdxReg1),
2276	F: MachineMemOperand::MOLoad, Size, BaseAlignment: Alignment));
2277	if (NeedsWinCFI)
2278	insertSEH(MBBI: MIB, TII, Flag: MachineInstr::FrameDestroy);
2279	} else {
2280	MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: LdrOpc));
2281	if (RPI.isPaired()) {
2282	MIB.addReg(RegNo: Reg2, Flags: getDefRegState(B: true));
2283	MIB.addMemOperand(MMO: MF.getMachineMemOperand(
2284	PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIdxReg2),
2285	F: MachineMemOperand::MOLoad, Size, BaseAlignment: Alignment));
2286	}
2287	MIB.addReg(RegNo: Reg1, Flags: getDefRegState(B: true));
2288	MIB.addReg(RegNo: AArch64::SP)
2289	.addImm(Val: RPI.Offset) // [sp, #offsetvscale]*
2290	// where factorvscale is implicit*
2291	.setMIFlag(MachineInstr::FrameDestroy);
2292	MIB.addMemOperand(MMO: MF.getMachineMemOperand(
2293	PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIdxReg1),
2294	F: MachineMemOperand::MOLoad, Size, BaseAlignment: Alignment));
2295	if (NeedsWinCFI)
2296	insertSEH(MBBI: MIB, TII, Flag: MachineInstr::FrameDestroy);
2297	}
2298	}
2299	return true;
2300	}
2301
2302	// Return the FrameID for a MMO.
2303	static std::optional<int> getMMOFrameID(MachineMemOperand *MMO,
2304	const MachineFrameInfo &MFI) {
2305	auto *PSV =
2306	dyn_cast_or_null<FixedStackPseudoSourceValue>(Val: MMO->getPseudoValue());
2307	if (PSV)
2308	return std::optional<int>(PSV->getFrameIndex());
2309
2310	if (MMO->getValue()) {
2311	if (auto *Al = dyn_cast<AllocaInst>(Val: getUnderlyingObject(V: MMO->getValue()))) {
2312	for (int FI = MFI.getObjectIndexBegin(); FI < MFI.getObjectIndexEnd();
2313	FI++)
2314	if (MFI.getObjectAllocation(ObjectIdx: FI) == Al)
2315	return FI;
2316	}
2317	}
2318
2319	return std::nullopt;
2320	}
2321
2322	// Return the FrameID for a Load/Store instruction by looking at the first MMO.
2323	static std::optional<int> getLdStFrameID(const MachineInstr &MI,
2324	const MachineFrameInfo &MFI) {
2325	if (!MI.mayLoadOrStore() \|\| MI.getNumMemOperands() < `1`)
2326	return std::nullopt;
2327
2328	return getMMOFrameID(MMO: *MI.memoperands_begin(), MFI);
2329	}
2330
2331	// Returns true if the LDST MachineInstr \p MI is a PPR access.
2332	static bool isPPRAccess(const MachineInstr &MI) {
2333	return AArch64::PPRRegClass.contains(Reg: MI.getOperand(i: `0`).getReg());
2334	}
2335
2336	// Check if a Hazard slot is needed for the current function, and if so create
2337	// one for it. The index is stored in AArch64FunctionInfo->StackHazardSlotIndex,
2338	// which can be used to determine if any hazard padding is needed.
2339	void AArch64FrameLowering::determineStackHazardSlot(
2340	MachineFunction &MF, BitVector &SavedRegs) const {
2341	unsigned StackHazardSize = getStackHazardSize(MF);
2342	auto *AFI = MF.getInfo<AArch64FunctionInfo>();
2343	if (StackHazardSize == `0` \|\| StackHazardSize % `16` != `0` \|\|
2344	AFI->hasStackHazardSlotIndex())
2345	return;
2346
2347	// Stack hazards are only needed in streaming functions.
2348	SMEAttrs Attrs = AFI->getSMEFnAttrs();
2349	if (!StackHazardInNonStreaming && Attrs.hasNonStreamingInterfaceAndBody())
2350	return;
2351
2352	MachineFrameInfo &MFI = MF.getFrameInfo();
2353
2354	// Add a hazard slot if there are any CSR FPR registers, or are any fp-only
2355	// stack objects.
2356	bool HasFPRCSRs = any_of(Range: SavedRegs.set_bits(), P: [](unsigned Reg) {
2357	return AArch64::FPR64RegClass.contains(Reg) \|\|
2358	AArch64::FPR128RegClass.contains(Reg) \|\|
2359	AArch64::ZPRRegClass.contains(Reg);
2360	});
2361	bool HasPPRCSRs = any_of(Range: SavedRegs.set_bits(), P: [](unsigned Reg) {
2362	return AArch64::PPRRegClass.contains(Reg);
2363	});
2364	bool HasFPRStackObjects = false;
2365	bool HasPPRStackObjects = false;
2366	if (!HasFPRCSRs \|\| SplitSVEObjects) {
2367	enum SlotType : uint8_t {
2368	Unknown = `0`,
2369	ZPRorFPR = `1` << `0`,
2370	PPR = `1` << `1`,
2371	GPR = `1` << `2`,
2372	LLVM_MARK_AS_BITMASK_ENUM(GPR)
2373	};
2374
2375	// Find stack slots solely used for one kind of register (ZPR, PPR, etc.),
2376	// based on the kinds of accesses used in the function.
2377	SmallVector<SlotType> SlotTypes(MFI.getObjectIndexEnd(), SlotType::Unknown);
2378	for (auto &MBB : MF) {
2379	for (auto &MI : MBB) {
2380	std::optional<int> FI = getLdStFrameID(MI, MFI);
2381	if (!FI \|\| FI < `0` \|\| FI > int(SlotTypes.size()))
2382	continue;
2383	if (MFI.hasScalableStackID(ObjectIdx: *FI)) {
2384	SlotTypes [*FI] \|=
2385	isPPRAccess(MI) ? SlotType::PPR : SlotType::ZPRorFPR;
2386	} else {
2387	SlotTypes [*FI] \|= AArch64InstrInfo::isFpOrNEON(MI)
2388	? SlotType::ZPRorFPR
2389	: SlotType::GPR;
2390	}
2391	}
2392	}
2393
2394	for (int FI = `0`; FI < int(SlotTypes.size()); ++FI) {
2395	HasFPRStackObjects \|= SlotTypes [FI] == SlotType::ZPRorFPR;
2396	// For SplitSVEObjects remember that this stack slot is a predicate, this
2397	// will be needed later when determining the frame layout.
2398	if (SlotTypes [FI] == SlotType::PPR) {
2399	MFI.setStackID(ObjectIdx: FI, ID: TargetStackID::ScalablePredicateVector);
2400	HasPPRStackObjects = true;
2401	}
2402	}
2403	}
2404
2405	if (HasFPRCSRs \|\| HasFPRStackObjects) {
2406	int ID = MFI.CreateStackObject(Size: StackHazardSize, Alignment: Align (`16`), isSpillSlot: false);
2407	LLVM_DEBUG(dbgs() << "Created Hazard slot at " << ID << " size "
2408	<< StackHazardSize << "\n");
2409	AFI->setStackHazardSlotIndex(ID);
2410	}
2411
2412	if (!AFI->hasStackHazardSlotIndex())
2413	return;
2414
2415	if (SplitSVEObjects) {
2416	CallingConv::ID CC = MF.getFunction().getCallingConv();
2417	if (AFI->isSVECC() \|\| CC == CallingConv::AArch64_SVE_VectorCall) {
2418	AFI->setSplitSVEObjects(true);
2419	LLVM_DEBUG(dbgs() << "Using SplitSVEObjects for SVE CC function\n");
2420	return;
2421	}
2422
2423	// We only use SplitSVEObjects in non-SVE CC functions if there's a
2424	// possibility of a stack hazard between PPRs and ZPRs/FPRs.
2425	LLVM_DEBUG(dbgs() << "Determining if SplitSVEObjects should be used in "
2426	"non-SVE CC function...\n");
2427
2428	// If another calling convention is explicitly set FPRs can't be promoted to
2429	// ZPR callee-saves.
2430	if (!is_contained(Set: {CallingConv::C, CallingConv::Fast}, Element: CC)) {
2431	LLVM_DEBUG(
2432	dbgs()
2433	<< "Calling convention is not supported with SplitSVEObjects\n");
2434	return;
2435	}
2436
2437	if (!HasPPRCSRs && !HasPPRStackObjects) {
2438	LLVM_DEBUG(
2439	dbgs() << "Not using SplitSVEObjects as no PPRs are on the stack\n");
2440	return;
2441	}
2442
2443	if (!HasFPRCSRs && !HasFPRStackObjects) {
2444	LLVM_DEBUG(
2445	dbgs()
2446	<< "Not using SplitSVEObjects as no FPRs or ZPRs are on the stack\n");
2447	return;
2448	}
2449
2450	[[maybe_unused]] const AArch64Subtarget &Subtarget =
2451	MF.getSubtarget<AArch64Subtarget>();
2452	assert(Subtarget.isSVEorStreamingSVEAvailable() &&
2453	"Expected SVE to be available for PPRs");
2454
2455	const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
2456	// With SplitSVEObjects the CS hazard padding is placed between the
2457	// PPRs and ZPRs. If there are any FPR CS there would be a hazard between
2458	// them and the CS GRPs. Avoid this by promoting all FPR CS to ZPRs.
2459	BitVector FPRZRegs(SavedRegs.size());
2460	for (size_t Reg = `0`, E = SavedRegs.size(); HasFPRCSRs && Reg < E; ++Reg) {
2461	BitVector::reference RegBit = SavedRegs [Reg];
2462	if (!RegBit)
2463	continue;
2464	unsigned SubRegIdx = `0`;
2465	if (AArch64::FPR64RegClass.contains(Reg))
2466	SubRegIdx = AArch64::dsub;
2467	else if (AArch64::FPR128RegClass.contains(Reg))
2468	SubRegIdx = AArch64::zsub;
2469	else
2470	continue;
2471	// Clear the bit for the FPR save.
2472	RegBit = false;
2473	// Mark that we should save the corresponding ZPR.
2474	Register ZReg =
2475	TRI->getMatchingSuperReg(Reg, SubIdx: SubRegIdx, RC: &AArch64::ZPRRegClass);
2476	FPRZRegs.set(ZReg);
2477	}
2478	SavedRegs \|= FPRZRegs;
2479
2480	AFI->setSplitSVEObjects(true);
2481	LLVM_DEBUG(dbgs() << "SplitSVEObjects enabled!\n");
2482	}
2483	}
2484
2485	void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
2486	BitVector &SavedRegs,
2487	RegScavenger RS) const* {
2488	// All calls are tail calls in GHC calling conv, and functions have no
2489	// prologue/epilogue.
2490	if (MF.getFunction().getCallingConv() == CallingConv::GHC)
2491	return;
2492
2493	const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
2494
2495	TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
2496	const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
2497	AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
2498	unsigned UnspilledCSGPR = AArch64::NoRegister;
2499	unsigned UnspilledCSGPRPaired = AArch64::NoRegister;
2500
2501	MachineFrameInfo &MFI = MF.getFrameInfo();
2502	const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
2503
2504	MCRegister BasePointerReg =
2505	RegInfo->hasBasePointer(MF) ? RegInfo->getBaseRegister() : MCRegister ();
2506
2507	unsigned ExtraCSSpill = `0`;
2508	bool HasUnpairedGPR64 = false;
2509	bool HasPairZReg = false;
2510	BitVector UserReservedRegs = RegInfo->getUserReservedRegs(MF);
2511	BitVector ReservedRegs = RegInfo->getReservedRegs(MF);
2512
2513	// Figure out which callee-saved registers to save/restore.
2514	for (unsigned i = `0`; CSRegs[i]; ++i) {
2515	const MCRegister Reg = CSRegs[i];
2516
2517	// Add the base pointer register to SavedRegs if it is callee-save.
2518	if (Reg == BasePointerReg)
2519	SavedRegs.set(Reg);
2520
2521	// Don't save manually reserved registers set through +reserve-x#i,
2522	// even for callee-saved registers, as per GCC's behavior.
2523	if (UserReservedRegs [Reg]) {
2524	SavedRegs.reset(Idx: Reg);
2525	continue;
2526	}
2527
2528	bool RegUsed = SavedRegs.test(Idx: Reg);
2529	MCRegister PairedReg;
2530	const bool RegIsGPR64 = AArch64::GPR64RegClass.contains(Reg);
2531	if (RegIsGPR64 \|\| AArch64::FPR64RegClass.contains(Reg) \|\|
2532	AArch64::FPR128RegClass.contains(Reg)) {
2533	// Compensate for odd numbers of GP CSRs.
2534	// For now, all the known cases of odd number of CSRs are of GPRs.
2535	if (HasUnpairedGPR64)
2536	PairedReg = CSRegs[i % `2` == `0` ? i - `1` : i + `1`];
2537	else
2538	PairedReg = CSRegs[i ^ `1`];
2539	}
2540
2541	// If the function requires all the GP registers to save (SavedRegs),
2542	// and there are an odd number of GP CSRs at the same time (CSRegs),
2543	// PairedReg could be in a different register class from Reg, which would
2544	// lead to a FPR (usually D8) accidentally being marked saved.
2545	if (RegIsGPR64 && !AArch64::GPR64RegClass.contains(Reg: PairedReg)) {
2546	PairedReg = AArch64::NoRegister;
2547	HasUnpairedGPR64 = true;
2548	}
2549	assert(PairedReg == AArch64::NoRegister \|\|
2550	AArch64::GPR64RegClass.contains(Reg, PairedReg) \|\|
2551	AArch64::FPR64RegClass.contains(Reg, PairedReg) \|\|
2552	AArch64::FPR128RegClass.contains(Reg, PairedReg));
2553
2554	if (!RegUsed) {
2555	if (AArch64::GPR64RegClass.contains(Reg) && !ReservedRegs [Reg]) {
2556	UnspilledCSGPR = Reg;
2557	UnspilledCSGPRPaired = PairedReg;
2558	}
2559	continue;
2560	}
2561
2562	// MachO's compact unwind format relies on all registers being stored in
2563	// pairs.
2564	// FIXME: the usual format is actually better if unwinding isn't needed.
2565	if (producePairRegisters(MF) && PairedReg != AArch64::NoRegister &&
2566	!SavedRegs.test(Idx: PairedReg)) {
2567	SavedRegs.set(PairedReg);
2568	if (AArch64::GPR64RegClass.contains(Reg: PairedReg) &&
2569	!ReservedRegs [PairedReg])
2570	ExtraCSSpill = PairedReg;
2571	}
2572	// Check if there is a pair of ZRegs, so it can select PReg for spill/fill
2573	HasPairZReg \|= (AArch64::ZPRRegClass.contains(Reg1: Reg, Reg2: CSRegs[i ^ `1`]) &&
2574	SavedRegs.test(Idx: CSRegs[i ^ `1`]));
2575	}
2576
2577	if (HasPairZReg && enableMultiVectorSpillFill(Subtarget, MF)) {
2578	AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
2579	// Find a suitable predicate register for the multi-vector spill/fill
2580	// instructions.
2581	MCRegister PnReg = findFreePredicateReg(SavedRegs);
2582	if (PnReg.isValid())
2583	AFI->setPredicateRegForFillSpill(PnReg);
2584	// If no free callee-save has been found assign one.
2585	if (!AFI->getPredicateRegForFillSpill() &&
2586	MF.getFunction().getCallingConv() ==
2587	CallingConv::AArch64_SVE_VectorCall) {
2588	SavedRegs.set(AArch64::P8);
2589	AFI->setPredicateRegForFillSpill(AArch64::PN8);
2590	}
2591
2592	assert(!ReservedRegs[AFI->getPredicateRegForFillSpill()] &&
2593	"Predicate cannot be a reserved register");
2594	}
2595
2596	if (MF.getFunction().getCallingConv() == CallingConv::Win64 &&
2597	!Subtarget.isTargetWindows()) {
2598	// For Windows calling convention on a non-windows OS, where X18 is treated
2599	// as reserved, back up X18 when entering non-windows code (marked with the
2600	// Windows calling convention) and restore when returning regardless of
2601	// whether the individual function uses it - it might call other functions
2602	// that clobber it.
2603	SavedRegs.set(AArch64::X18);
2604	}
2605
2606	// Determine if a Hazard slot should be used and where it should go.
2607	// If SplitSVEObjects is used, the hazard padding is placed between the PPRs
2608	// and ZPRs. Otherwise, it goes in the callee save area.
2609	determineStackHazardSlot(MF, SavedRegs);
2610
2611	// Calculates the callee saved stack size.
2612	unsigned CSStackSize = `0`;
2613	unsigned ZPRCSStackSize = `0`;
2614	unsigned PPRCSStackSize = `0`;
2615	const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
2616	for (unsigned Reg : SavedRegs.set_bits()) {
2617	auto *RC = TRI->getMinimalPhysRegClass(Reg: MCRegister (Reg));
2618	assert(RC && "expected register class!");
2619	auto SpillSize = TRI->getSpillSize(RC: *RC);
2620	bool IsZPR = AArch64::ZPRRegClass.contains(Reg);
2621	bool IsPPR = !IsZPR && AArch64::PPRRegClass.contains(Reg);
2622	if (IsZPR)
2623	ZPRCSStackSize += SpillSize;
2624	else if (IsPPR)
2625	PPRCSStackSize += SpillSize;
2626	else
2627	CSStackSize += SpillSize;
2628	}
2629
2630	// Save number of saved regs, so we can easily update CSStackSize later to
2631	// account for any additional 64-bit GPR saves. Note: After this point
2632	// only 64-bit GPRs can be added to SavedRegs.
2633	unsigned NumSavedRegs = SavedRegs.count();
2634
2635	// If we have hazard padding in the CS area add that to the size.
2636	if (AFI->isStackHazardIncludedInCalleeSaveArea())
2637	CSStackSize += getStackHazardSize(MF);
2638
2639	// Increase the callee-saved stack size if the function has streaming mode
2640	// changes, as we will need to spill the value of the VG register.
2641	if (requiresSaveVG(MF))
2642	CSStackSize += `8`;
2643
2644	// If we must call __arm_get_current_vg in the prologue preserve the LR.
2645	if (requiresSaveVG(MF) && !Subtarget.hasSVE())
2646	SavedRegs.set(AArch64::LR);
2647
2648	// The frame record needs to be created by saving the appropriate registers
2649	uint64_t EstimatedStackSize = MFI.estimateStackSize(MF);
2650	if (hasFP(MF) \|\|
2651	windowsRequiresStackProbe(MF, StackSizeInBytes: EstimatedStackSize + CSStackSize + `16`)) {
2652	SavedRegs.set(AArch64::FP);
2653	SavedRegs.set(AArch64::LR);
2654	}
2655
2656	LLVM_DEBUG({
2657	dbgs() << "*** determineCalleeSaves\nSaved CSRs:";
2658	for (unsigned Reg : SavedRegs.set_bits())
2659	dbgs() << `' '` << printReg(MCRegister(Reg), RegInfo);
2660	dbgs() << "\n";
2661	});
2662
2663	// If any callee-saved registers are used, the frame cannot be eliminated.
2664	auto [ZPRLocalStackSize, PPRLocalStackSize] =
2665	determineSVEStackSizes(MF, AssignOffsets: AssignObjectOffsets::No);
2666	uint64_t SVELocals = ZPRLocalStackSize + PPRLocalStackSize;
2667	uint64_t SVEStackSize =
2668	alignTo(Value: ZPRCSStackSize + PPRCSStackSize + SVELocals, Align: `16`);
2669	bool CanEliminateFrame = (SavedRegs.count() == `0`) && !SVEStackSize;
2670
2671	// The CSR spill slots have not been allocated yet, so estimateStackSize
2672	// won't include them.
2673	unsigned EstimatedStackSizeLimit = estimateRSStackSizeLimit(MF);
2674
2675	// We may address some of the stack above the canonical frame address, either
2676	// for our own arguments or during a call. Include that in calculating whether
2677	// we have complicated addressing concerns.
2678	int64_t CalleeStackUsed = `0`;
2679	for (int I = MFI.getObjectIndexBegin(); I != `0`; ++I) {
2680	int64_t FixedOff = MFI.getObjectOffset(ObjectIdx: I);
2681	if (FixedOff > CalleeStackUsed)
2682	CalleeStackUsed = FixedOff;
2683	}
2684
2685	// Conservatively always assume BigStack when there are SVE spills.
2686	bool BigStack = SVEStackSize \|\| (EstimatedStackSize + CSStackSize +
2687	CalleeStackUsed) > EstimatedStackSizeLimit;
2688	if (BigStack \|\| !CanEliminateFrame \|\| RegInfo->cannotEliminateFrame(MF))
2689	AFI->setHasStackFrame(true);
2690
2691	// Estimate if we might need to scavenge a register at some point in order
2692	// to materialize a stack offset. If so, either spill one additional
2693	// callee-saved register or reserve a special spill slot to facilitate
2694	// register scavenging. If we already spilled an extra callee-saved register
2695	// above to keep the number of spills even, we don't need to do anything else
2696	// here.
2697	if (BigStack) {
2698	if (!ExtraCSSpill && UnspilledCSGPR != AArch64::NoRegister) {
2699	LLVM_DEBUG(dbgs() << "Spilling " << printReg(UnspilledCSGPR, RegInfo)
2700	<< " to get a scratch register.\n");
2701	SavedRegs.set(UnspilledCSGPR);
2702	ExtraCSSpill = UnspilledCSGPR;
2703
2704	// MachO's compact unwind format relies on all registers being stored in
2705	// pairs, so if we need to spill one extra for BigStack, then we need to
2706	// store the pair.
2707	if (producePairRegisters(MF)) {
2708	if (UnspilledCSGPRPaired == AArch64::NoRegister) {
2709	// Failed to make a pair for compact unwind format, revert spilling.
2710	if (produceCompactUnwindFrame(AFL: *this, MF)) {
2711	SavedRegs.reset(Idx: UnspilledCSGPR);
2712	ExtraCSSpill = AArch64::NoRegister;
2713	}
2714	} else
2715	SavedRegs.set(UnspilledCSGPRPaired);
2716	}
2717	}
2718
2719	// If we didn't find an extra callee-saved register to spill, create
2720	// an emergency spill slot.
2721	if (!ExtraCSSpill \|\| MF.getRegInfo().isPhysRegUsed(PhysReg: ExtraCSSpill)) {
2722	const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
2723	const TargetRegisterClass &RC = AArch64::GPR64RegClass;
2724	unsigned Size = TRI->getSpillSize(RC);
2725	Align Alignment = TRI->getSpillAlign(RC);
2726	int FI = MFI.CreateSpillStackObject(Size, Alignment);
2727	RS->addScavengingFrameIndex(FI);
2728	LLVM_DEBUG(dbgs() << "No available CS registers, allocated fi#" << FI
2729	<< " as the emergency spill slot.\n");
2730	}
2731	}
2732
2733	// Adding the size of additional 64bit GPR saves.
2734	CSStackSize += `8` * (SavedRegs.count() - NumSavedRegs);
2735
2736	// A Swift asynchronous context extends the frame record with a pointer
2737	// directly before FP.
2738	if (hasFP(MF) && AFI->hasSwiftAsyncContext())
2739	CSStackSize += `8`;
2740
2741	uint64_t AlignedCSStackSize = alignTo(Value: CSStackSize, Align: `16`);
2742	LLVM_DEBUG(dbgs() << "Estimated stack frame size: "
2743	<< EstimatedStackSize + AlignedCSStackSize << " bytes.\n");
2744
2745	assert((!MFI.isCalleeSavedInfoValid() \|\|
2746	AFI->getCalleeSavedStackSize() == AlignedCSStackSize) &&
2747	"Should not invalidate callee saved info");
2748
2749	// Round up to register pair alignment to avoid additional SP adjustment
2750	// instructions.
2751	AFI->setCalleeSavedStackSize(AlignedCSStackSize);
2752	AFI->setCalleeSaveStackHasFreeSpace(AlignedCSStackSize != CSStackSize);
2753	AFI->setSVECalleeSavedStackSize(ZPR: ZPRCSStackSize, PPR: alignTo(Value: PPRCSStackSize, Align: `16`));
2754	}
2755
2756	bool AArch64FrameLowering::assignCalleeSavedSpillSlots(
2757	MachineFunction &MF, const TargetRegisterInfo *RegInfo,
2758	std::vector<CalleeSavedInfo> &CSI) const {
2759	bool IsWindows = isTargetWindows(MF);
2760	unsigned StackHazardSize = getStackHazardSize(MF);
2761	// To match the canonical windows frame layout, reverse the list of
2762	// callee saved registers to get them laid out by PrologEpilogInserter
2763	// in the right order. (PrologEpilogInserter allocates stack objects top
2764	// down. Windows canonical prologs store higher numbered registers at
2765	// the top, thus have the CSI array start from the highest registers.)
2766	if (IsWindows)
2767	std::reverse(first: CSI.begin(), last: CSI.end());
2768
2769	if (CSI.empty())
2770	return true; // Early exit if no callee saved registers are modified!
2771
2772	// Now that we know which registers need to be saved and restored, allocate
2773	// stack slots for them.
2774	MachineFrameInfo &MFI = MF.getFrameInfo();
2775	auto *AFI = MF.getInfo<AArch64FunctionInfo>();
2776
2777	if (IsWindows && hasFP(MF) && AFI->hasSwiftAsyncContext()) {
2778	int FrameIdx = MFI.CreateStackObject(Size: `8`, Alignment: Align (`16`), isSpillSlot: true);
2779	AFI->setSwiftAsyncContextFrameIdx(FrameIdx);
2780	MFI.setIsCalleeSavedObjectIndex(ObjectIdx: FrameIdx, IsCalleeSaved: true);
2781	}
2782
2783	// Insert VG into the list of CSRs, immediately before LR if saved.
2784	if (requiresSaveVG(MF)) {
2785	CalleeSavedInfo VGInfo(AArch64::VG);
2786	auto It =
2787	find_if(Range&: CSI, P: [](auto &Info) { return Info.getReg() == AArch64::LR; });
2788	if (It != CSI.end())
2789	CSI.insert(position: It, x: VGInfo);
2790	else
2791	CSI.push_back(x: VGInfo);
2792	}
2793
2794	Register LastReg = `0`;
2795	int HazardSlotIndex = std::numeric_limits<int>::max();
2796	for (auto &CS : CSI) {
2797	MCRegister Reg = CS.getReg();
2798	const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg);
2799
2800	// Create a hazard slot as we switch between GPR and FPR CSRs.
2801	if (AFI->isStackHazardIncludedInCalleeSaveArea() &&
2802	(!LastReg \|\| !AArch64InstrInfo::isFpOrNEON(Reg: LastReg)) &&
2803	AArch64InstrInfo::isFpOrNEON(Reg)) {
2804	assert(HazardSlotIndex == std::numeric_limits<int>::max() &&
2805	"Unexpected register order for hazard slot");
2806	HazardSlotIndex = MFI.CreateStackObject(Size: StackHazardSize, Alignment: Align (`8`), isSpillSlot: true);
2807	LLVM_DEBUG(dbgs() << "Created CSR Hazard at slot " << HazardSlotIndex
2808	<< "\n");
2809	AFI->setStackHazardCSRSlotIndex(HazardSlotIndex);
2810	MFI.setIsCalleeSavedObjectIndex(ObjectIdx: HazardSlotIndex, IsCalleeSaved: true);
2811	}
2812
2813	unsigned Size = RegInfo->getSpillSize(RC: *RC);
2814	Align Alignment(RegInfo->getSpillAlign(RC: *RC));
2815	int FrameIdx = MFI.CreateStackObject(Size, Alignment, isSpillSlot: true);
2816	CS.setFrameIdx(FrameIdx);
2817	MFI.setIsCalleeSavedObjectIndex(ObjectIdx: FrameIdx, IsCalleeSaved: true);
2818
2819	// Grab 8 bytes below FP for the extended asynchronous frame info.
2820	if (hasFP(MF) && AFI->hasSwiftAsyncContext() && !IsWindows &&
2821	Reg == AArch64::FP) {
2822	FrameIdx = MFI.CreateStackObject(Size: `8`, Alignment, isSpillSlot: true);
2823	AFI->setSwiftAsyncContextFrameIdx(FrameIdx);
2824	MFI.setIsCalleeSavedObjectIndex(ObjectIdx: FrameIdx, IsCalleeSaved: true);
2825	}
2826	LastReg = Reg;
2827	}
2828
2829	// Add hazard slot in the case where no FPR CSRs are present.
2830	if (AFI->isStackHazardIncludedInCalleeSaveArea() &&
2831	HazardSlotIndex == std::numeric_limits<int>::max()) {
2832	HazardSlotIndex = MFI.CreateStackObject(Size: StackHazardSize, Alignment: Align (`8`), isSpillSlot: true);
2833	LLVM_DEBUG(dbgs() << "Created CSR Hazard at slot " << HazardSlotIndex
2834	<< "\n");
2835	AFI->setStackHazardCSRSlotIndex(HazardSlotIndex);
2836	MFI.setIsCalleeSavedObjectIndex(ObjectIdx: HazardSlotIndex, IsCalleeSaved: true);
2837	}
2838
2839	return true;
2840	}
2841
2842	bool AArch64FrameLowering::enableStackSlotScavenging(
2843	const MachineFunction &MF) const {
2844	const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
2845	// If the function has streaming-mode changes, don't scavenge a
2846	// spillslot in the callee-save area, as that might require an
2847	// 'addvl' in the streaming-mode-changing call-sequence when the
2848	// function doesn't use a FP.
2849	if (AFI->hasStreamingModeChanges() && !hasFP(MF))
2850	return false;
2851	// Don't allow register salvaging with hazard slots, in case it moves objects
2852	// into the wrong place.
2853	if (AFI->hasStackHazardSlotIndex())
2854	return false;
2855	return AFI->hasCalleeSaveStackFreeSpace();
2856	}
2857
2858	/// returns true if there are any SVE callee saves.
2859	static bool getSVECalleeSaveSlotRange(const MachineFrameInfo &MFI,
2860	int &Min, int &Max) {
2861	Min = std::numeric_limits<int>::max();
2862	Max = std::numeric_limits<int>::min();
2863
2864	if (!MFI.isCalleeSavedInfoValid())
2865	return false;
2866
2867	const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
2868	for (auto &CS : CSI) {
2869	if (AArch64::ZPRRegClass.contains(Reg: CS.getReg()) \|\|
2870	AArch64::PPRRegClass.contains(Reg: CS.getReg())) {
2871	assert((Max == std::numeric_limits<int>::min() \|\|
2872	Max + `1` == CS.getFrameIdx()) &&
2873	"SVE CalleeSaves are not consecutive");
2874	Min = std::min(a: Min, b: CS.getFrameIdx());
2875	Max = std::max(a: Max, b: CS.getFrameIdx());
2876	}
2877	}
2878	return Min != std::numeric_limits<int>::max();
2879	}
2880
2881	static SVEStackSizes determineSVEStackSizes(MachineFunction &MF,
2882	AssignObjectOffsets AssignOffsets) {
2883	MachineFrameInfo &MFI = MF.getFrameInfo();
2884	auto *AFI = MF.getInfo<AArch64FunctionInfo>();
2885
2886	SVEStackSizes SVEStack{};
2887
2888	// With SplitSVEObjects we maintain separate stack offsets for predicates
2889	// (PPRs) and SVE vectors (ZPRs). When SplitSVEObjects is disabled predicates
2890	// are included in the SVE vector area.
2891	uint64_t &ZPRStackTop = SVEStack.ZPRStackSize;
2892	uint64_t &PPRStackTop =
2893	AFI->hasSplitSVEObjects() ? SVEStack.PPRStackSize : SVEStack.ZPRStackSize;
2894
2895	#ifndef NDEBUG
2896	// First process all fixed stack objects.
2897	for (int I = MFI.getObjectIndexBegin(); I != `0`; ++I)
2898	assert(!MFI.hasScalableStackID(I) &&
2899	"SVE vectors should never be passed on the stack by value, only by "
2900	"reference.");
2901	#endif
2902
2903	auto AllocateObject = [&](int FI) {
2904	uint64_t &StackTop = MFI.getStackID(ObjectIdx: FI) == TargetStackID::ScalableVector
2905	? ZPRStackTop
2906	: PPRStackTop;
2907
2908	// FIXME: Given that the length of SVE vectors is not necessarily a power of
2909	// two, we'd need to align every object dynamically at runtime if the
2910	// alignment is larger than 16. This is not yet supported.
2911	Align Alignment = MFI.getObjectAlign(ObjectIdx: FI);
2912	if (Alignment > Align (`16`))
2913	report_fatal_error(
2914	reason: "Alignment of scalable vectors > 16 bytes is not yet supported");
2915
2916	StackTop += MFI.getObjectSize(ObjectIdx: FI);
2917	StackTop = alignTo(Size: StackTop, A: Alignment);
2918
2919	assert(StackTop < (uint64_t)std::numeric_limits<int64_t>::max() &&
2920	"SVE StackTop far too large?!");
2921
2922	int64_t Offset = -int64_t(StackTop);
2923	if (AssignOffsets == AssignObjectOffsets::Yes)
2924	MFI.setObjectOffset(ObjectIdx: FI, SPOffset: Offset);
2925
2926	LLVM_DEBUG(dbgs() << "alloc FI(" << FI << ") at SP[" << Offset << "]\n");
2927	};
2928
2929	// Then process all callee saved slots.
2930	int MinCSFrameIndex, MaxCSFrameIndex;
2931	if (getSVECalleeSaveSlotRange(MFI, Min&: MinCSFrameIndex, Max&: MaxCSFrameIndex)) {
2932	for (int FI = MinCSFrameIndex; FI <= MaxCSFrameIndex; ++FI)
2933	AllocateObject (FI);
2934	}
2935
2936	// Ensure the CS area is 16-byte aligned.
2937	PPRStackTop = alignTo(Size: PPRStackTop, A: Align (`16U`));
2938	ZPRStackTop = alignTo(Size: ZPRStackTop, A: Align (`16U`));
2939
2940	// Create a buffer of SVE objects to allocate and sort it.
2941	SmallVector<int, `8`> ObjectsToAllocate;
2942	// If we have a stack protector, and we've previously decided that we have SVE
2943	// objects on the stack and thus need it to go in the SVE stack area, then it
2944	// needs to go first.
2945	int StackProtectorFI = -`1`;
2946	if (MFI.hasStackProtectorIndex()) {
2947	StackProtectorFI = MFI.getStackProtectorIndex();
2948	if (MFI.getStackID(ObjectIdx: StackProtectorFI) == TargetStackID::ScalableVector)
2949	ObjectsToAllocate.push_back(Elt: StackProtectorFI);
2950	}
2951
2952	for (int FI = `0`, E = MFI.getObjectIndexEnd(); FI != E; ++FI) {
2953	if (FI == StackProtectorFI \|\| MFI.isDeadObjectIndex(ObjectIdx: FI) \|\|
2954	MFI.isCalleeSavedObjectIndex(ObjectIdx: FI))
2955	continue;
2956
2957	if (MFI.getStackID(ObjectIdx: FI) != TargetStackID::ScalableVector &&
2958	MFI.getStackID(ObjectIdx: FI) != TargetStackID::ScalablePredicateVector)
2959	continue;
2960
2961	ObjectsToAllocate.push_back(Elt: FI);
2962	}
2963
2964	// Allocate all SVE locals and spills
2965	for (unsigned FI : ObjectsToAllocate)
2966	AllocateObject (FI);
2967
2968	PPRStackTop = alignTo(Size: PPRStackTop, A: Align (`16U`));
2969	ZPRStackTop = alignTo(Size: ZPRStackTop, A: Align (`16U`));
2970
2971	if (AssignOffsets == AssignObjectOffsets::Yes)
2972	AFI->setStackSizeSVE(ZPR: SVEStack.ZPRStackSize, PPR: SVEStack.PPRStackSize);
2973
2974	return SVEStack;
2975	}
2976
2977	void AArch64FrameLowering::processFunctionBeforeFrameFinalized(
2978	MachineFunction &MF, RegScavenger RS) const* {
2979	assert(getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown &&
2980	"Upwards growing stack unsupported");
2981
2982	(void)determineSVEStackSizes(MF, AssignOffsets: AssignObjectOffsets::Yes);
2983
2984	// If this function isn't doing Win64-style C++ EH, we don't need to do
2985	// anything.
2986	if (!MF.hasEHFunclets())
2987	return;
2988
2989	MachineFrameInfo &MFI = MF.getFrameInfo();
2990	auto *AFI = MF.getInfo<AArch64FunctionInfo>();
2991
2992	// Win64 C++ EH needs to allocate space for the catch objects in the fixed
2993	// object area right next to the UnwindHelp object.
2994	WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo();
2995	int64_t CurrentOffset =
2996	AFI->getVarArgsGPRSize() + AFI->getTailCallReservedStack();
2997	for (WinEHTryBlockMapEntry &TBME : EHInfo.TryBlockMap) {
2998	for (WinEHHandlerType &H : TBME.HandlerArray) {
2999	int FrameIndex = H.CatchObj.FrameIndex;
3000	if ((FrameIndex != INT_MAX) && MFI.getObjectOffset(ObjectIdx: FrameIndex) == `0`) {
3001	CurrentOffset =
3002	alignTo(Value: CurrentOffset, Align: MFI.getObjectAlign(ObjectIdx: FrameIndex).value());
3003	CurrentOffset += MFI.getObjectSize(ObjectIdx: FrameIndex);
3004	MFI.setObjectOffset(ObjectIdx: FrameIndex, SPOffset: -CurrentOffset);
3005	}
3006	}
3007	}
3008
3009	// Create an UnwindHelp object.
3010	// The UnwindHelp object is allocated at the start of the fixed object area
3011	int64_t UnwindHelpOffset = alignTo(Size: CurrentOffset + `8`, A: Align (`16`));
3012	assert(UnwindHelpOffset == getFixedObjectSize(MF, AFI, /IsWin64/ true,
3013	/IsFunclet/ false) &&
3014	"UnwindHelpOffset must be at the start of the fixed object area");
3015	int UnwindHelpFI = MFI.CreateFixedObject(/Size/ `8`, SPOffset: -UnwindHelpOffset,
3016	/IsImmutable=/false);
3017	EHInfo.UnwindHelpFrameIdx = UnwindHelpFI;
3018
3019	MachineBasicBlock &MBB = MF.front();
3020	auto MBBI = MBB.begin();
3021	while (MBBI != MBB.end() && MBBI ->getFlag(Flag: MachineInstr::FrameSetup))
3022	++MBBI;
3023
3024	// We need to store -2 into the UnwindHelp object at the start of the
3025	// function.
3026	DebugLoc DL;
3027	RS->enterBasicBlockEnd(MBB);
3028	RS->backward(I: MBBI);
3029	Register DstReg = RS->FindUnusedReg(RC: &AArch64::GPR64commonRegClass);
3030	assert(DstReg && "There must be a free register after frame setup");
3031	const AArch64InstrInfo &TII =
3032	*MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
3033	BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AArch64::MOVi64imm), DestReg: DstReg).addImm(Val: -`2`);
3034	BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AArch64::STURXi))
3035	.addReg(RegNo: DstReg, Flags: getKillRegState(B: true))
3036	.addFrameIndex(Idx: UnwindHelpFI)
3037	.addImm(Val: `0`);
3038	}
3039
3040	namespace {
3041	struct TagStoreInstr {
3042	MachineInstr *MI;
3043	int64_t Offset, Size;
3044	explicit TagStoreInstr(MachineInstr *MI, int64_t Offset, int64_t Size)
3045	: MI(MI), Offset(Offset), Size(Size) {}
3046	};
3047
3048	class TagStoreEdit {
3049	MachineFunction *MF;
3050	MachineBasicBlock *MBB;
3051	MachineRegisterInfo *MRI;
3052	// Tag store instructions that are being replaced.
3053	SmallVector<TagStoreInstr, `8`> TagStores;
3054	// Combined memref arguments of the above instructions.
3055	SmallVector<MachineMemOperand *, `8`> CombinedMemRefs;
3056
3057	// Replace allocation tags in [FrameReg + FrameRegOffset, FrameReg +
3058	// FrameRegOffset + Size) with the address tag of SP.
3059	Register FrameReg;
3060	StackOffset FrameRegOffset;
3061	int64_t Size;
3062	// If not std::nullopt, move FrameReg to (FrameReg + FrameRegUpdate) at the
3063	// end.
3064	std::optional<int64_t> FrameRegUpdate;
3065	// MIFlags for any FrameReg updating instructions.
3066	unsigned FrameRegUpdateFlags;
3067
3068	// Use zeroing instruction variants.
3069	bool ZeroData;
3070	DebugLoc DL;
3071
3072	void emitUnrolled(MachineBasicBlock::iterator InsertI);
3073	void emitLoop(MachineBasicBlock::iterator InsertI);
3074
3075	public:
3076	TagStoreEdit(MachineBasicBlock MBB, bool* ZeroData)
3077	: MBB(MBB), ZeroData(ZeroData) {
3078	MF = MBB->getParent();
3079	MRI = &MF->getRegInfo();
3080	}
3081	// Add an instruction to be replaced. Instructions must be added in the
3082	// ascending order of Offset, and have to be adjacent.
3083	void addInstruction(TagStoreInstr I) {
3084	assert((TagStores.empty() \|\|
3085	TagStores.back().Offset + TagStores.back().Size == I.Offset) &&
3086	"Non-adjacent tag store instructions.");
3087	TagStores.push_back(Elt: I);
3088	}
3089	void clear() { TagStores.clear(); }
3090	// Emit equivalent code at the given location, and erase the current set of
3091	// instructions. May skip if the replacement is not profitable. May invalidate
3092	// the input iterator and replace it with a valid one.
3093	void emitCode(MachineBasicBlock::iterator &InsertI,
3094	const AArch64FrameLowering TFI, bool* TryMergeSPUpdate);
3095	};
3096
3097	void TagStoreEdit::emitUnrolled(MachineBasicBlock::iterator InsertI) {
3098	const AArch64InstrInfo *TII =
3099	MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
3100
3101	const int64_t kMinOffset = -`256` * `16`;
3102	const int64_t kMaxOffset = `255` * `16`;
3103
3104	Register BaseReg = FrameReg;
3105	int64_t BaseRegOffsetBytes = FrameRegOffset.getFixed();
3106	if (BaseRegOffsetBytes < kMinOffset \|\|
3107	BaseRegOffsetBytes + (Size - Size % `32`) > kMaxOffset \|\|
3108	// BaseReg can be FP, which is not necessarily aligned to 16-bytes. In
3109	// that case, BaseRegOffsetBytes will not be aligned to 16 bytes, which
3110	// is required for the offset of ST2G.
3111	BaseRegOffsetBytes % `16` != `0`) {
3112	Register ScratchReg = MRI->createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3113	emitFrameOffset(MBB&: *MBB, MBBI: InsertI, DL, DestReg: ScratchReg, SrcReg: BaseReg,
3114	Offset: StackOffset::getFixed(Fixed: BaseRegOffsetBytes), TII);
3115	BaseReg = ScratchReg;
3116	BaseRegOffsetBytes = `0`;
3117	}
3118
3119	MachineInstr LastI = nullptr*;
3120	while (Size) {
3121	int64_t InstrSize = (Size > `16`) ? `32` : `16`;
3122	unsigned Opcode =
3123	InstrSize == `16`
3124	? (ZeroData ? AArch64::STZGi : AArch64::STGi)
3125	: (ZeroData ? AArch64::STZ2Gi : AArch64::ST2Gi);
3126	assert(BaseRegOffsetBytes % `16` == `0`);
3127	MachineInstr I = BuildMI(BB&: MBB, I: InsertI, MIMD: DL, MCID: TII->get(Opcode))
3128	.addReg(RegNo: AArch64::SP)
3129	.addReg(RegNo: BaseReg)
3130	.addImm(Val: BaseRegOffsetBytes / `16`)
3131	.setMemRefs(CombinedMemRefs);
3132	// A store to [BaseReg, #0] should go last for an opportunity to fold the
3133	// final SP adjustment in the epilogue.
3134	if (BaseRegOffsetBytes == `0`)
3135	LastI = I;
3136	BaseRegOffsetBytes += InstrSize;
3137	Size -= InstrSize;
3138	}
3139
3140	if (LastI)
3141	MBB->splice(Where: InsertI, Other: MBB, From: LastI);
3142	}
3143
3144	void TagStoreEdit::emitLoop(MachineBasicBlock::iterator InsertI) {
3145	const AArch64InstrInfo *TII =
3146	MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
3147
3148	Register BaseReg = FrameRegUpdate
3149	? FrameReg
3150	: MRI->createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3151	Register SizeReg = MRI->createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3152
3153	emitFrameOffset(MBB&: *MBB, MBBI: InsertI, DL, DestReg: BaseReg, SrcReg: FrameReg, Offset: FrameRegOffset, TII);
3154
3155	int64_t LoopSize = Size;
3156	// If the loop size is not a multiple of 32, split off one 16-byte store at
3157	// the end to fold BaseReg update into.
3158	if (FrameRegUpdate && *FrameRegUpdate)
3159	LoopSize -= LoopSize % `32`;
3160	MachineInstr LoopI = BuildMI(BB&: MBB, I: InsertI, MIMD: DL,
3161	MCID: TII->get(Opcode: ZeroData ? AArch64::STZGloop_wback
3162	: AArch64::STGloop_wback))
3163	.addDef(RegNo: SizeReg)
3164	.addDef(RegNo: BaseReg)
3165	.addImm(Val: LoopSize)
3166	.addReg(RegNo: BaseReg)
3167	.setMemRefs(CombinedMemRefs);
3168	if (FrameRegUpdate)
3169	LoopI->setFlags(FrameRegUpdateFlags);
3170
3171	int64_t ExtraBaseRegUpdate =
3172	FrameRegUpdate ? (*FrameRegUpdate - FrameRegOffset.getFixed() - Size) : `0`;
3173	LLVM_DEBUG(dbgs() << "TagStoreEdit::emitLoop: LoopSize=" << LoopSize
3174	<< ", Size=" << Size
3175	<< ", ExtraBaseRegUpdate=" << ExtraBaseRegUpdate
3176	<< ", FrameRegUpdate=" << FrameRegUpdate
3177	<< ", FrameRegOffset.getFixed()="
3178	<< FrameRegOffset.getFixed() << "\n");
3179	if (LoopSize < Size) {
3180	assert(FrameRegUpdate);
3181	assert(Size - LoopSize == `16`);
3182	// Tag 16 more bytes at BaseReg and update BaseReg.
3183	int64_t STGOffset = ExtraBaseRegUpdate + `16`;
3184	assert(STGOffset % `16` == `0` && STGOffset >= -`4096` && STGOffset <= `4080` &&
3185	"STG immediate out of range");
3186	BuildMI(BB&: *MBB, I: InsertI, MIMD: DL,
3187	MCID: TII->get(Opcode: ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex))
3188	.addDef(RegNo: BaseReg)
3189	.addReg(RegNo: BaseReg)
3190	.addReg(RegNo: BaseReg)
3191	.addImm(Val: STGOffset / `16`)
3192	.setMemRefs(CombinedMemRefs)
3193	.setMIFlags(FrameRegUpdateFlags);
3194	} else if (ExtraBaseRegUpdate) {
3195	// Update BaseReg.
3196	int64_t AddSubOffset = std::abs(i: ExtraBaseRegUpdate);
3197	assert(AddSubOffset <= `4095` && "ADD/SUB immediate out of range");
3198	BuildMI(
3199	BB&: *MBB, I: InsertI, MIMD: DL,
3200	MCID: TII->get(Opcode: ExtraBaseRegUpdate > `0` ? AArch64::ADDXri : AArch64::SUBXri))
3201	.addDef(RegNo: BaseReg)
3202	.addReg(RegNo: BaseReg)
3203	.addImm(Val: AddSubOffset)
3204	.addImm(Val: `0`)
3205	.setMIFlags(FrameRegUpdateFlags);
3206	}
3207	}
3208
3209	// Check if II is a register update that can be merged into STGloop that ends*
3210	// at (Reg + Size). RemainingOffset is the required adjustment to Reg after the
3211	// end of the loop.
3212	bool canMergeRegUpdate(MachineBasicBlock::iterator II, unsigned Reg,
3213	int64_t Size, int64_t *TotalOffset) {
3214	MachineInstr &MI = *II;
3215	if ((MI.getOpcode() == AArch64::ADDXri \|\|
3216	MI.getOpcode() == AArch64::SUBXri) &&
3217	MI.getOperand(i: `0`).getReg() == Reg && MI.getOperand(i: `1`).getReg() == Reg) {
3218	unsigned Shift = AArch64_AM::getShiftValue(Imm: MI.getOperand(i: `3`).getImm());
3219	int64_t Offset = MI.getOperand(i: `2`).getImm() << Shift;
3220	if (MI.getOpcode() == AArch64::SUBXri)
3221	Offset = -Offset;
3222	int64_t PostOffset = Offset - Size;
3223	// TagStoreEdit::emitLoop might emit either an ADD/SUB after the loop, or
3224	// an STGPostIndex which does the last 16 bytes of tag write. Which one is
3225	// chosen depends on the alignment of the loop size, but the difference
3226	// between the valid ranges for the two instructions is small, so we
3227	// conservatively assume that it could be either case here.
3228	//
3229	// Max offset of STGPostIndex, minus the 16 byte tag write folded into that
3230	// instruction.
3231	const int64_t kMaxOffset = `4080` - `16`;
3232	// Max offset of SUBXri.
3233	const int64_t kMinOffset = -`4095`;
3234	if (PostOffset <= kMaxOffset && PostOffset >= kMinOffset &&
3235	PostOffset % `16` == `0`) {
3236	*TotalOffset = Offset;
3237	return true;
3238	}
3239	}
3240	return false;
3241	}
3242
3243	void mergeMemRefs(const SmallVectorImpl<TagStoreInstr> &TSE,
3244	SmallVectorImpl<MachineMemOperand *> &MemRefs) {
3245	MemRefs.clear();
3246	for (auto &TS : TSE) {
3247	MachineInstr *MI = TS.MI;
3248	// An instruction without memory operands may access anything. Be
3249	// conservative and return an empty list.
3250	if (MI->memoperands_empty()) {
3251	MemRefs.clear();
3252	return;
3253	}
3254	MemRefs.append(in_start: MI->memoperands_begin(), in_end: MI->memoperands_end());
3255	}
3256	}
3257
3258	void TagStoreEdit::emitCode(MachineBasicBlock::iterator &InsertI,
3259	const AArch64FrameLowering *TFI,
3260	bool TryMergeSPUpdate) {
3261	if (TagStores.empty())
3262	return;
3263	TagStoreInstr &FirstTagStore = TagStores [`0`];
3264	TagStoreInstr &LastTagStore = TagStores [TagStores.size() - `1`];
3265	Size = LastTagStore.Offset - FirstTagStore.Offset + LastTagStore.Size;
3266	DL = TagStores [`0`].MI->getDebugLoc();
3267
3268	Register Reg;
3269	FrameRegOffset = TFI->resolveFrameOffsetReference(
3270	MF: MF, ObjectOffset: FirstTagStore.Offset, isFixed: false* /isFixed/,
3271	StackID: TargetStackID::Default /StackID/, FrameReg&: Reg,
3272	/PreferFP=/false, /ForSimm=/true);
3273	FrameReg = Reg;
3274	FrameRegUpdate = std::nullopt;
3275
3276	mergeMemRefs(TSE: TagStores, MemRefs&: CombinedMemRefs);
3277
3278	LLVM_DEBUG({
3279	dbgs() << "Replacing adjacent STG instructions:\n";
3280	for (const auto &Instr : TagStores) {
3281	dbgs() << " " << *Instr.MI;
3282	}
3283	});
3284
3285	// Size threshold where a loop becomes shorter than a linear sequence of
3286	// tagging instructions.
3287	const int kSetTagLoopThreshold = `176`;
3288	if (Size < kSetTagLoopThreshold) {
3289	if (TagStores.size() < `2`)
3290	return;
3291	emitUnrolled(InsertI);
3292	} else {
3293	MachineInstr UpdateInstr = nullptr*;
3294	int64_t TotalOffset = `0`;
3295	if (TryMergeSPUpdate) {
3296	// See if we can merge base register update into the STGloop.
3297	// This is done in AArch64LoadStoreOptimizer for "normal" stores,
3298	// but STGloop is way too unusual for that, and also it only
3299	// realistically happens in function epilogue. Also, STGloop is expanded
3300	// before that pass.
3301	if (InsertI != MBB->end() &&
3302	canMergeRegUpdate(II: InsertI, Reg: FrameReg, Size: FrameRegOffset.getFixed() + Size,
3303	TotalOffset: &TotalOffset)) {
3304	UpdateInstr = &*InsertI ++;
3305	LLVM_DEBUG(dbgs() << "Folding SP update into loop:\n "
3306	<< *UpdateInstr);
3307	}
3308	}
3309
3310	if (!UpdateInstr && TagStores.size() < `2`)
3311	return;
3312
3313	if (UpdateInstr) {
3314	FrameRegUpdate = TotalOffset;
3315	FrameRegUpdateFlags = UpdateInstr->getFlags();
3316	}
3317	emitLoop(InsertI);
3318	if (UpdateInstr)
3319	UpdateInstr->eraseFromParent();
3320	}
3321
3322	for (auto &TS : TagStores)
3323	TS.MI->eraseFromParent();
3324	}
3325
3326	bool isMergeableStackTaggingInstruction(MachineInstr &MI, int64_t &Offset,
3327	int64_t &Size, bool &ZeroData) {
3328	MachineFunction &MF = *MI.getParent()->getParent();
3329	const MachineFrameInfo &MFI = MF.getFrameInfo();
3330
3331	unsigned Opcode = MI.getOpcode();
3332	ZeroData = (Opcode == AArch64::STZGloop \|\| Opcode == AArch64::STZGi \|\|
3333	Opcode == AArch64::STZ2Gi);
3334
3335	if (Opcode == AArch64::STGloop \|\| Opcode == AArch64::STZGloop) {
3336	if (!MI.getOperand(i: `0`).isDead() \|\| !MI.getOperand(i: `1`).isDead())
3337	return false;
3338	if (!MI.getOperand(i: `2`).isImm() \|\| !MI.getOperand(i: `3`).isFI())
3339	return false;
3340	Offset = MFI.getObjectOffset(ObjectIdx: MI.getOperand(i: `3`).getIndex());
3341	Size = MI.getOperand(i: `2`).getImm();
3342	return true;
3343	}
3344
3345	if (Opcode == AArch64::STGi \|\| Opcode == AArch64::STZGi)
3346	Size = `16`;
3347	else if (Opcode == AArch64::ST2Gi \|\| Opcode == AArch64::STZ2Gi)
3348	Size = `32`;
3349	else
3350	return false;
3351
3352	if (MI.getOperand(i: `0`).getReg() != AArch64::SP \|\| !MI.getOperand(i: `1`).isFI())
3353	return false;
3354
3355	Offset = MFI.getObjectOffset(ObjectIdx: MI.getOperand(i: `1`).getIndex()) +
3356	`16` * MI.getOperand(i: `2`).getImm();
3357	return true;
3358	}
3359
3360	// Detect a run of memory tagging instructions for adjacent stack frame slots,
3361	// and replace them with a shorter instruction sequence:
3362	// replace STG + STG with ST2G*
3363	// replace STGloop + STGloop with STGloop*
3364	// This code needs to run when stack slot offsets are already known, but before
3365	// FrameIndex operands in STG instructions are eliminated.
3366	MachineBasicBlock::iterator tryMergeAdjacentSTG(MachineBasicBlock::iterator II,
3367	const AArch64FrameLowering *TFI,
3368	RegScavenger *RS) {
3369	bool FirstZeroData;
3370	int64_t Size, Offset;
3371	MachineInstr &MI = *II;
3372	MachineBasicBlock *MBB = MI.getParent();
3373	MachineBasicBlock::iterator NextI = ++II;
3374	if (&MI == &MBB->instr_back())
3375	return II;
3376	if (!isMergeableStackTaggingInstruction(MI, Offset, Size, ZeroData&: FirstZeroData))
3377	return II;
3378
3379	SmallVector<TagStoreInstr, `4`> Instrs;
3380	Instrs.emplace_back(Args: &MI, Args&: Offset, Args&: Size);
3381
3382	constexpr int kScanLimit = `10`;
3383	int Count = `0`;
3384	for (MachineBasicBlock::iterator E = MBB->end();
3385	NextI != E && Count < kScanLimit; ++NextI) {
3386	MachineInstr &MI = *NextI;
3387	bool ZeroData;
3388	int64_t Size, Offset;
3389	// Collect instructions that update memory tags with a FrameIndex operand
3390	// and (when applicable) constant size, and whose output registers are dead
3391	// (the latter is almost always the case in practice). Since these
3392	// instructions effectively have no inputs or outputs, we are free to skip
3393	// any non-aliasing instructions in between without tracking used registers.
3394	if (isMergeableStackTaggingInstruction(MI, Offset, Size, ZeroData)) {
3395	if (ZeroData != FirstZeroData)
3396	break;
3397	Instrs.emplace_back(Args: &MI, Args&: Offset, Args&: Size);
3398	continue;
3399	}
3400
3401	// Only count non-transient, non-tagging instructions toward the scan
3402	// limit.
3403	if (!MI.isTransient())
3404	++Count;
3405
3406	// Just in case, stop before the epilogue code starts.
3407	if (MI.getFlag(Flag: MachineInstr::FrameSetup) \|\|
3408	MI.getFlag(Flag: MachineInstr::FrameDestroy))
3409	break;
3410
3411	// Reject anything that may alias the collected instructions.
3412	if (MI.mayLoadOrStore() \|\| MI.hasUnmodeledSideEffects() \|\| MI.isCall())
3413	break;
3414	}
3415
3416	// New code will be inserted after the last tagging instruction we've found.
3417	MachineBasicBlock::iterator InsertI = Instrs.back().MI;
3418
3419	// All the gathered stack tag instructions are merged and placed after
3420	// last tag store in the list. The check should be made if the nzcv
3421	// flag is live at the point where we are trying to insert. Otherwise
3422	// the nzcv flag might get clobbered if any stg loops are present.
3423
3424	// FIXME : This approach of bailing out from merge is conservative in
3425	// some ways like even if stg loops are not present after merge the
3426	// insert list, this liveness check is done (which is not needed).
3427	LivePhysRegs LiveRegs(*(MBB->getParent()->getSubtarget().getRegisterInfo()));
3428	LiveRegs.addLiveOuts(MBB: *MBB);
3429	for (auto I = MBB->rbegin();; ++I) {
3430	MachineInstr &MI = *I;
3431	if (MI == InsertI)
3432	break;
3433	LiveRegs.stepBackward(MI: *I);
3434	}
3435	InsertI ++;
3436	if (LiveRegs.contains(Reg: AArch64::NZCV))
3437	return InsertI;
3438
3439	llvm::stable_sort(Range&: Instrs,
3440	C: [](const TagStoreInstr &Left, const TagStoreInstr &Right) {
3441	return Left.Offset < Right.Offset;
3442	});
3443
3444	// Make sure that we don't have any overlapping stores.
3445	int64_t CurOffset = Instrs [`0`].Offset;
3446	for (auto &Instr : Instrs) {
3447	if (CurOffset > Instr.Offset)
3448	return NextI;
3449	CurOffset = Instr.Offset + Instr.Size;
3450	}
3451
3452	// Find contiguous runs of tagged memory and emit shorter instruction
3453	// sequences for them when possible.
3454	TagStoreEdit TSE(MBB, FirstZeroData);
3455	std::optional<int64_t> EndOffset;
3456	for (auto &Instr : Instrs) {
3457	if (EndOffset && *EndOffset != Instr.Offset) {
3458	// Found a gap.
3459	TSE.emitCode(InsertI, TFI, /TryMergeSPUpdate = / false);
3460	TSE.clear();
3461	}
3462
3463	TSE.addInstruction(I: Instr);
3464	EndOffset = Instr.Offset + Instr.Size;
3465	}
3466
3467	const MachineFunction *MF = MBB->getParent();
3468	// Multiple FP/SP updates in a loop cannot be described by CFI instructions.
3469	TSE.emitCode(
3470	InsertI, TFI, /TryMergeSPUpdate = /
3471	!MF->getInfo<AArch64FunctionInfo>()->needsAsyncDwarfUnwindInfo(MF: *MF));
3472
3473	return InsertI;
3474	}
3475	} // namespace
3476
3477	void AArch64FrameLowering::processFunctionBeforeFrameIndicesReplaced(
3478	MachineFunction &MF, RegScavenger RS = nullptr) const* {
3479	for (auto &BB : MF)
3480	for (MachineBasicBlock::iterator II = BB.begin(); II != BB.end();) {
3481	if (StackTaggingMergeSetTag)
3482	II = tryMergeAdjacentSTG(II, TFI: this, RS);
3483	}
3484
3485	// By the time this method is called, most of the prologue/epilogue code is
3486	// already emitted, whether its location was affected by the shrink-wrapping
3487	// optimization or not.
3488	if (!MF.getFunction().hasFnAttribute(Kind: Attribute::Naked) &&
3489	shouldSignReturnAddressEverywhere(MF))
3490	emitPacRetPlusLeafHardening(MF);
3491	}
3492
3493	/// For Win64 AArch64 EH, the offset to the Unwind object is from the SP
3494	/// before the update. This is easily retrieved as it is exactly the offset
3495	/// that is set in processFunctionBeforeFrameFinalized.
3496	StackOffset AArch64FrameLowering::getFrameIndexReferencePreferSP(
3497	const MachineFunction &MF, int FI, Register &FrameReg,
3498	bool IgnoreSPUpdates) const {
3499	const MachineFrameInfo &MFI = MF.getFrameInfo();
3500	if (IgnoreSPUpdates) {
3501	LLVM_DEBUG(dbgs() << "Offset from the SP for " << FI << " is "
3502	<< MFI.getObjectOffset(FI) << "\n");
3503	FrameReg = AArch64::SP;
3504	return StackOffset::getFixed(Fixed: MFI.getObjectOffset(ObjectIdx: FI));
3505	}
3506
3507	// Go to common code if we cannot provide sp + offset.
3508	if (MFI.hasVarSizedObjects() \|\|
3509	MF.getInfo<AArch64FunctionInfo>()->hasSVEStackSize() \|\|
3510	MF.getSubtarget().getRegisterInfo()->hasStackRealignment(MF))
3511	return getFrameIndexReference(MF, FI, FrameReg);
3512
3513	FrameReg = AArch64::SP;
3514	return getStackOffset(MF, ObjectOffset: MFI.getObjectOffset(ObjectIdx: FI));
3515	}
3516
3517	/// The parent frame offset (aka dispFrame) is only used on X86_64 to retrieve
3518	/// the parent's frame pointer
3519	unsigned AArch64FrameLowering::getWinEHParentFrameOffset(
3520	const MachineFunction &MF) const {
3521	return `0`;
3522	}
3523
3524	/// Funclets only need to account for space for the callee saved registers,
3525	/// as the locals are accounted for in the parent's stack frame.
3526	unsigned AArch64FrameLowering::getWinEHFuncletFrameSize(
3527	const MachineFunction &MF) const {
3528	// This is the size of the pushed CSRs.
3529	unsigned CSSize =
3530	MF.getInfo<AArch64FunctionInfo>()->getCalleeSavedStackSize();
3531	// This is the amount of stack a funclet needs to allocate.
3532	return alignTo(Size: CSSize + MF.getFrameInfo().getMaxCallFrameSize(),
3533	A: getStackAlign());
3534	}
3535
3536	namespace {
3537	struct FrameObject {
3538	bool IsValid = false;
3539	// Index of the object in MFI.
3540	int ObjectIndex = `0`;
3541	// Group ID this object belongs to.
3542	int GroupIndex = -`1`;
3543	// This object should be placed first (closest to SP).
3544	bool ObjectFirst = false;
3545	// This object's group (which always contains the object with
3546	// ObjectFirst==true) should be placed first.
3547	bool GroupFirst = false;
3548
3549	// Used to distinguish between FP and GPR accesses. The values are decided so
3550	// that they sort FPR < Hazard < GPR and they can be or'd together.
3551	unsigned Accesses = `0`;
3552	enum { AccessFPR = `1`, AccessHazard = `2`, AccessGPR = `4` };
3553	};
3554
3555	class GroupBuilder {
3556	SmallVector<int, `8`> CurrentMembers;
3557	int NextGroupIndex = `0`;
3558	std::vector<FrameObject> &Objects;
3559
3560	public:
3561	GroupBuilder(std::vector<FrameObject> &Objects) : Objects(Objects) {}
3562	void AddMember(int Index) { CurrentMembers.push_back(Elt: Index); }
3563	void EndCurrentGroup() {
3564	if (CurrentMembers.size() > `1`) {
3565	// Create a new group with the current member list. This might remove them
3566	// from their pre-existing groups. That's OK, dealing with overlapping
3567	// groups is too hard and unlikely to make a difference.
3568	LLVM_DEBUG(dbgs() << "group:");
3569	for (int Index : CurrentMembers) {
3570	Objects [Index].GroupIndex = NextGroupIndex;
3571	LLVM_DEBUG(dbgs() << " " << Index);
3572	}
3573	LLVM_DEBUG(dbgs() << "\n");
3574	NextGroupIndex++;
3575	}
3576	CurrentMembers.clear();
3577	}
3578	};
3579
3580	bool FrameObjectCompare(const FrameObject &A, const FrameObject &B) {
3581	// Objects at a lower index are closer to FP; objects at a higher index are
3582	// closer to SP.
3583	//
3584	// For consistency in our comparison, all invalid objects are placed
3585	// at the end. This also allows us to stop walking when we hit the
3586	// first invalid item after it's all sorted.
3587	//
3588	// If we want to include a stack hazard region, order FPR accesses < the
3589	// hazard object < GPRs accesses in order to create a separation between the
3590	// two. For the Accesses field 1 = FPR, 2 = Hazard Object, 4 = GPR.
3591	//
3592	// Otherwise the "first" object goes first (closest to SP), followed by the
3593	// members of the "first" group.
3594	//
3595	// The rest are sorted by the group index to keep the groups together.
3596	// Higher numbered groups are more likely to be around longer (i.e. untagged
3597	// in the function epilogue and not at some earlier point). Place them closer
3598	// to SP.
3599	//
3600	// If all else equal, sort by the object index to keep the objects in the
3601	// original order.
3602	return std::make_tuple(args: !A.IsValid, args: A.Accesses, args: A.ObjectFirst, args: A.GroupFirst,
3603	args: A.GroupIndex, args: A.ObjectIndex) <
3604	std::make_tuple(args: !B.IsValid, args: B.Accesses, args: B.ObjectFirst, args: B.GroupFirst,
3605	args: B.GroupIndex, args: B.ObjectIndex);
3606	}
3607	} // namespace
3608
3609	void AArch64FrameLowering::orderFrameObjects(
3610	const MachineFunction &MF, SmallVectorImpl<int> &ObjectsToAllocate) const {
3611	const AArch64FunctionInfo &AFI = *MF.getInfo<AArch64FunctionInfo>();
3612
3613	if ((!OrderFrameObjects && !AFI.hasSplitSVEObjects()) \|\|
3614	ObjectsToAllocate.empty())
3615	return;
3616
3617	const MachineFrameInfo &MFI = MF.getFrameInfo();
3618	std::vector<FrameObject> FrameObjects(MFI.getObjectIndexEnd());
3619	for (auto &Obj : ObjectsToAllocate) {
3620	FrameObjects [Obj].IsValid = true;
3621	FrameObjects [Obj].ObjectIndex = Obj;
3622	}
3623
3624	// Identify FPR vs GPR slots for hazards, and stack slots that are tagged at
3625	// the same time.
3626	GroupBuilder GB(FrameObjects);
3627	for (auto &MBB : MF) {
3628	for (auto &MI : MBB) {
3629	if (MI.isDebugInstr())
3630	continue;
3631
3632	if (AFI.hasStackHazardSlotIndex()) {
3633	std::optional<int> FI = getLdStFrameID(MI, MFI);
3634	if (FI && FI >= `0` && FI < (int)FrameObjects.size()) {
3635	if (MFI.getStackID(ObjectIdx: *FI) == TargetStackID::ScalableVector \|\|
3636	AArch64InstrInfo::isFpOrNEON(MI))
3637	FrameObjects [*FI].Accesses \|= FrameObject::AccessFPR;
3638	else
3639	FrameObjects [*FI].Accesses \|= FrameObject::AccessGPR;
3640	}
3641	}
3642
3643	int OpIndex;
3644	switch (MI.getOpcode()) {
3645	case AArch64::STGloop:
3646	case AArch64::STZGloop:
3647	OpIndex = `3`;
3648	break;
3649	case AArch64::STGi:
3650	case AArch64::STZGi:
3651	case AArch64::ST2Gi:
3652	case AArch64::STZ2Gi:
3653	OpIndex = `1`;
3654	break;
3655	default:
3656	OpIndex = -`1`;
3657	}
3658
3659	int TaggedFI = -`1`;
3660	if (OpIndex >= `0`) {
3661	const MachineOperand &MO = MI.getOperand(i: OpIndex);
3662	if (MO.isFI()) {
3663	int FI = MO.getIndex();
3664	if (FI >= `0` && FI < MFI.getObjectIndexEnd() &&
3665	FrameObjects [FI].IsValid)
3666	TaggedFI = FI;
3667	}
3668	}
3669
3670	// If this is a stack tagging instruction for a slot that is not part of a
3671	// group yet, either start a new group or add it to the current one.
3672	if (TaggedFI >= `0`)
3673	GB.AddMember(Index: TaggedFI);
3674	else
3675	GB.EndCurrentGroup();
3676	}
3677	// Groups should never span multiple basic blocks.
3678	GB.EndCurrentGroup();
3679	}
3680
3681	if (AFI.hasStackHazardSlotIndex()) {
3682	FrameObjects [AFI.getStackHazardSlotIndex()].Accesses =
3683	FrameObject::AccessHazard;
3684	// If a stack object is unknown or both GPR and FPR, sort it into GPR.
3685	for (auto &Obj : FrameObjects)
3686	if (!Obj.Accesses \|\|
3687	Obj.Accesses == (FrameObject::AccessGPR \| FrameObject::AccessFPR))
3688	Obj.Accesses = FrameObject::AccessGPR;
3689	}
3690
3691	// If the function's tagged base pointer is pinned to a stack slot, we want to
3692	// put that slot first when possible. This will likely place it at SP + 0,
3693	// and save one instruction when generating the base pointer because IRG does
3694	// not allow an immediate offset.
3695	std::optional<int> TBPI = AFI.getTaggedBasePointerIndex();
3696	if (TBPI) {
3697	FrameObjects [TBPI].ObjectFirst = true*;
3698	FrameObjects [TBPI].GroupFirst = true*;
3699	int FirstGroupIndex = FrameObjects [*TBPI].GroupIndex;
3700	if (FirstGroupIndex >= `0`)
3701	for (FrameObject &Object : FrameObjects)
3702	if (Object.GroupIndex == FirstGroupIndex)
3703	Object.GroupFirst = true;
3704	}
3705
3706	llvm::stable_sort(Range&: FrameObjects, C: FrameObjectCompare);
3707
3708	int i = `0`;
3709	for (auto &Obj : FrameObjects) {
3710	// All invalid items are sorted at the end, so it's safe to stop.
3711	if (!Obj.IsValid)
3712	break;
3713	ObjectsToAllocate [i++] = Obj.ObjectIndex;
3714	}
3715
3716	LLVM_DEBUG({
3717	dbgs() << "Final frame order:\n";
3718	for (auto &Obj : FrameObjects) {
3719	if (!Obj.IsValid)
3720	break;
3721	dbgs() << " " << Obj.ObjectIndex << ": group " << Obj.GroupIndex;
3722	if (Obj.ObjectFirst)
3723	dbgs() << ", first";
3724	if (Obj.GroupFirst)
3725	dbgs() << ", group-first";
3726	dbgs() << "\n";
3727	}
3728	});
3729	}
3730
3731	/// Emit a loop to decrement SP until it is equal to TargetReg, with probes at
3732	/// least every ProbeSize bytes. Returns an iterator of the first instruction
3733	/// after the loop. The difference between SP and TargetReg must be an exact
3734	/// multiple of ProbeSize.
3735	MachineBasicBlock::iterator
3736	AArch64FrameLowering::inlineStackProbeLoopExactMultiple(
3737	MachineBasicBlock::iterator MBBI, int64_t ProbeSize,
3738	Register TargetReg) const {
3739	MachineBasicBlock &MBB = *MBBI ->getParent();
3740	MachineFunction &MF = *MBB.getParent();
3741	const AArch64InstrInfo *TII =
3742	MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
3743	DebugLoc DL = MBB.findDebugLoc(MBBI);
3744
3745	MachineFunction::iterator MBBInsertPoint = std::next(x: MBB.getIterator());
3746	MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
3747	MF.insert(MBBI: MBBInsertPoint, MBB: LoopMBB);
3748	MachineBasicBlock *ExitMBB = MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
3749	MF.insert(MBBI: MBBInsertPoint, MBB: ExitMBB);
3750
3751	// SUB SP, SP, #ProbeSize (or equivalent if ProbeSize is not encodable
3752	// in SUB).
3753	emitFrameOffset(MBB&: *LoopMBB, MBBI: LoopMBB->end(), DL, DestReg: AArch64::SP, SrcReg: AArch64::SP,
3754	Offset: StackOffset::getFixed(Fixed: -ProbeSize), TII,
3755	MachineInstr::FrameSetup);
3756	// LDR XZR, [SP]
3757	BuildMI(BB&: *LoopMBB, I: LoopMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::LDRXui))
3758	.addDef(RegNo: AArch64::XZR)
3759	.addReg(RegNo: AArch64::SP)
3760	.addImm(Val: `0`)
3761	.addMemOperand(MMO: MF.getMachineMemOperand(
3762	PtrInfo: MachinePointerInfo::getUnknownStack(MF),
3763	F: MachineMemOperand::MOLoad \| MachineMemOperand::MOVolatile, Size: `8`,
3764	BaseAlignment: Align (`8`)))
3765	.setMIFlags(MachineInstr::FrameSetup);
3766	// CMP SP, TargetReg
3767	BuildMI(BB&: *LoopMBB, I: LoopMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::SUBSXrx64),
3768	DestReg: AArch64::XZR)
3769	.addReg(RegNo: AArch64::SP)
3770	.addReg(RegNo: TargetReg)
3771	.addImm(Val: AArch64_AM::getArithExtendImm(ET: AArch64_AM::UXTX, Imm: `0`))
3772	.setMIFlags(MachineInstr::FrameSetup);
3773	// B.CC Loop
3774	BuildMI(BB&: *LoopMBB, I: LoopMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::Bcc))
3775	.addImm(Val: AArch64CC::NE)
3776	.addMBB(MBB: LoopMBB)
3777	.setMIFlags(MachineInstr::FrameSetup);
3778
3779	LoopMBB->addSuccessor(Succ: ExitMBB);
3780	LoopMBB->addSuccessor(Succ: LoopMBB);
3781	// Synthesize the exit MBB.
3782	ExitMBB->splice(Where: ExitMBB->end(), Other: &MBB, From: MBBI, To: MBB.end());
3783	ExitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
3784	MBB.addSuccessor(Succ: LoopMBB);
3785	// Update liveins.
3786	fullyRecomputeLiveIns(MBBs: {ExitMBB, LoopMBB});
3787
3788	return ExitMBB->begin();
3789	}
3790
3791	void AArch64FrameLowering::inlineStackProbeFixed(
3792	MachineBasicBlock::iterator MBBI, Register ScratchReg, int64_t FrameSize,
3793	StackOffset CFAOffset) const {
3794	MachineBasicBlock *MBB = MBBI ->getParent();
3795	MachineFunction &MF = *MBB->getParent();
3796	const AArch64InstrInfo *TII =
3797	MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
3798	AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
3799	bool EmitAsyncCFI = AFI->needsAsyncDwarfUnwindInfo(MF);
3800	bool HasFP = hasFP(MF);
3801
3802	DebugLoc DL;
3803	int64_t ProbeSize = MF.getInfo<AArch64FunctionInfo>()->getStackProbeSize();
3804	int64_t NumBlocks = FrameSize / ProbeSize;
3805	int64_t ResidualSize = FrameSize % ProbeSize;
3806
3807	LLVM_DEBUG(dbgs() << "Stack probing: total " << FrameSize << " bytes, "
3808	<< NumBlocks << " blocks of " << ProbeSize
3809	<< " bytes, plus " << ResidualSize << " bytes\n");
3810
3811	// Decrement SP by NumBlock ProbeSize bytes, with either unrolled or*
3812	// ordinary loop.
3813	if (NumBlocks <= AArch64::StackProbeMaxLoopUnroll) {
3814	for (int i = `0`; i < NumBlocks; ++i) {
3815	// SUB SP, SP, #ProbeSize (or equivalent if ProbeSize is not
3816	// encodable in a SUB).
3817	emitFrameOffset(MBB&: *MBB, MBBI, DL, DestReg: AArch64::SP, SrcReg: AArch64::SP,
3818	Offset: StackOffset::getFixed(Fixed: -ProbeSize), TII,
3819	MachineInstr::FrameSetup, SetNZCV: false, NeedsWinCFI: false, HasWinCFI: nullptr,
3820	EmitCFAOffset: EmitAsyncCFI && !HasFP, InitialOffset: CFAOffset);
3821	CFAOffset += StackOffset::getFixed(Fixed: ProbeSize);
3822	// LDR XZR, [SP]
3823	BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::LDRXui))
3824	.addDef(RegNo: AArch64::XZR)
3825	.addReg(RegNo: AArch64::SP)
3826	.addImm(Val: `0`)
3827	.addMemOperand(MMO: MF.getMachineMemOperand(
3828	PtrInfo: MachinePointerInfo::getUnknownStack(MF),
3829	F: MachineMemOperand::MOLoad \| MachineMemOperand::MOVolatile, Size: `8`,
3830	BaseAlignment: Align (`8`)))
3831	.setMIFlags(MachineInstr::FrameSetup);
3832	}
3833	} else if (NumBlocks != `0`) {
3834	// SUB ScratchReg, SP, #FrameSize (or equivalent if FrameSize is not
3835	// encodable in ADD). ScrathReg may temporarily become the CFA register.
3836	emitFrameOffset(MBB&: *MBB, MBBI, DL, DestReg: ScratchReg, SrcReg: AArch64::SP,
3837	Offset: StackOffset::getFixed(Fixed: -ProbeSize * NumBlocks), TII,
3838	MachineInstr::FrameSetup, SetNZCV: false, NeedsWinCFI: false, HasWinCFI: nullptr,
3839	EmitCFAOffset: EmitAsyncCFI && !HasFP, InitialOffset: CFAOffset);
3840	CFAOffset += StackOffset::getFixed(Fixed: ProbeSize * NumBlocks);
3841	MBBI = inlineStackProbeLoopExactMultiple(MBBI, ProbeSize, TargetReg: ScratchReg);
3842	MBB = MBBI ->getParent();
3843	if (EmitAsyncCFI && !HasFP) {
3844	// Set the CFA register back to SP.
3845	CFIInstBuilder (*MBB, MBBI, MachineInstr::FrameSetup)
3846	.buildDefCFARegister(Reg: AArch64::SP);
3847	}
3848	}
3849
3850	if (ResidualSize != `0`) {
3851	// SUB SP, SP, #ResidualSize (or equivalent if ResidualSize is not encodable
3852	// in SUB).
3853	emitFrameOffset(MBB&: *MBB, MBBI, DL, DestReg: AArch64::SP, SrcReg: AArch64::SP,
3854	Offset: StackOffset::getFixed(Fixed: -ResidualSize), TII,
3855	MachineInstr::FrameSetup, SetNZCV: false, NeedsWinCFI: false, HasWinCFI: nullptr,
3856	EmitCFAOffset: EmitAsyncCFI && !HasFP, InitialOffset: CFAOffset);
3857	if (ResidualSize > AArch64::StackProbeMaxUnprobedStack) {
3858	// LDR XZR, [SP]
3859	BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::LDRXui))
3860	.addDef(RegNo: AArch64::XZR)
3861	.addReg(RegNo: AArch64::SP)
3862	.addImm(Val: `0`)
3863	.addMemOperand(MMO: MF.getMachineMemOperand(
3864	PtrInfo: MachinePointerInfo::getUnknownStack(MF),
3865	F: MachineMemOperand::MOLoad \| MachineMemOperand::MOVolatile, Size: `8`,
3866	BaseAlignment: Align (`8`)))
3867	.setMIFlags(MachineInstr::FrameSetup);
3868	}
3869	}
3870	}
3871
3872	void AArch64FrameLowering::inlineStackProbe(MachineFunction &MF,
3873	MachineBasicBlock &MBB) const {
3874	// Get the instructions that need to be replaced. We emit at most two of
3875	// these. Remember them in order to avoid complications coming from the need
3876	// to traverse the block while potentially creating more blocks.
3877	SmallVector<MachineInstr *, `4`> ToReplace;
3878	for (MachineInstr &MI : MBB)
3879	if (MI.getOpcode() == AArch64::PROBED_STACKALLOC \|\|
3880	MI.getOpcode() == AArch64::PROBED_STACKALLOC_VAR)
3881	ToReplace.push_back(Elt: &MI);
3882
3883	for (MachineInstr *MI : ToReplace) {
3884	if (MI->getOpcode() == AArch64::PROBED_STACKALLOC) {
3885	Register ScratchReg = MI->getOperand(i: `0`).getReg();
3886	int64_t FrameSize = MI->getOperand(i: `1`).getImm();
3887	StackOffset CFAOffset = StackOffset::get(Fixed: MI->getOperand(i: `2`).getImm(),
3888	Scalable: MI->getOperand(i: `3`).getImm());
3889	inlineStackProbeFixed(MBBI: MI->getIterator(), ScratchReg, FrameSize,
3890	CFAOffset);
3891	} else {
3892	assert(MI->getOpcode() == AArch64::PROBED_STACKALLOC_VAR &&
3893	"Stack probe pseudo-instruction expected");
3894	const AArch64InstrInfo *TII =
3895	MI->getMF()->getSubtarget<AArch64Subtarget>().getInstrInfo();
3896	Register TargetReg = MI->getOperand(i: `0`).getReg();
3897	(void)TII->probedStackAlloc(MBBI: MI->getIterator(), TargetReg, FrameSetup: true);
3898	}
3899	MI->eraseFromParent();
3900	}
3901	}
3902
3903	struct StackAccess {
3904	enum AccessType {
3905	NotAccessed = `0`, // Stack object not accessed by load/store instructions.
3906	GPR = `1` << `0`, // A general purpose register.
3907	PPR = `1` << `1`, // A predicate register.
3908	FPR = `1` << `2`, // A floating point/Neon/SVE register.
3909	};
3910
3911	int Idx;
3912	StackOffset Offset;
3913	int64_t Size;
3914	unsigned AccessTypes;
3915
3916	StackAccess() : Idx(`0`), Offset (), Size(`0`), AccessTypes(NotAccessed) {}
3917
3918	bool operator<(const StackAccess &Rhs) const {
3919	return std::make_tuple(args: start(), args: Idx) <
3920	std::make_tuple(args: Rhs.start(), args: Rhs.Idx);
3921	}
3922
3923	bool isCPU() const {
3924	// Predicate register load and store instructions execute on the CPU.
3925	return AccessTypes & (AccessType::GPR \| AccessType::PPR);
3926	}
3927	bool isSME() const { return AccessTypes & AccessType::FPR; }
3928	bool isMixed() const { return isCPU() && isSME(); }
3929
3930	int64_t start() const { return Offset.getFixed() + Offset.getScalable(); }
3931	int64_t end() const { return start() + Size; }
3932
3933	std::string getTypeString() const {
3934	switch (AccessTypes) {
3935	case AccessType::FPR:
3936	return "FPR";
3937	case AccessType::PPR:
3938	return "PPR";
3939	case AccessType::GPR:
3940	return "GPR";
3941	case AccessType::NotAccessed:
3942	return "NA";
3943	default:
3944	return "Mixed";
3945	}
3946	}
3947
3948	void print(raw_ostream &OS) const {
3949	OS << getTypeString() << " stack object at [SP"
3950	<< (Offset.getFixed() < `0` ? "" : "+") << Offset.getFixed();
3951	if (Offset.getScalable())
3952	OS << (Offset.getScalable() < `0` ? "" : "+") << Offset.getScalable()
3953	<< " * vscale";
3954	OS << "]";
3955	}
3956	};
3957
3958	static inline raw_ostream &operator<<(raw_ostream &OS, const StackAccess &SA) {
3959	SA.print(OS);
3960	return OS;
3961	}
3962
3963	void AArch64FrameLowering::emitRemarks(
3964	const MachineFunction &MF, MachineOptimizationRemarkEmitter ORE) const* {
3965
3966	auto *AFI = MF.getInfo<AArch64FunctionInfo>();
3967	if (AFI->getSMEFnAttrs().hasNonStreamingInterfaceAndBody())
3968	return;
3969
3970	unsigned StackHazardSize = getStackHazardSize(MF);
3971	const uint64_t HazardSize =
3972	(StackHazardSize) ? StackHazardSize : StackHazardRemarkSize;
3973
3974	if (HazardSize == `0`)
3975	return;
3976
3977	const MachineFrameInfo &MFI = MF.getFrameInfo();
3978	// Bail if function has no stack objects.
3979	if (!MFI.hasStackObjects())
3980	return;
3981
3982	std::vector<StackAccess> StackAccesses(MFI.getNumObjects());
3983
3984	size_t NumFPLdSt = `0`;
3985	size_t NumNonFPLdSt = `0`;
3986
3987	// Collect stack accesses via Load/Store instructions.
3988	for (const MachineBasicBlock &MBB : MF) {
3989	for (const MachineInstr &MI : MBB) {
3990	if (!MI.mayLoadOrStore() \|\| MI.getNumMemOperands() < `1`)
3991	continue;
3992	for (MachineMemOperand *MMO : MI.memoperands()) {
3993	std::optional<int> FI = getMMOFrameID(MMO, MFI);
3994	if (FI && !MFI.isDeadObjectIndex(ObjectIdx: *FI)) {
3995	int FrameIdx = *FI;
3996
3997	size_t ArrIdx = FrameIdx + MFI.getNumFixedObjects();
3998	if (StackAccesses [ArrIdx].AccessTypes == StackAccess::NotAccessed) {
3999	StackAccesses [ArrIdx].Idx = FrameIdx;
4000	StackAccesses [ArrIdx].Offset =
4001	getFrameIndexReferenceFromSP(MF, FI: FrameIdx);
4002	StackAccesses [ArrIdx].Size = MFI.getObjectSize(ObjectIdx: FrameIdx);
4003	}
4004
4005	unsigned RegTy = StackAccess::AccessType::GPR;
4006	if (MFI.hasScalableStackID(ObjectIdx: FrameIdx))
4007	RegTy = isPPRAccess(MI) ? StackAccess::PPR : StackAccess::FPR;
4008	else if (AArch64InstrInfo::isFpOrNEON(MI))
4009	RegTy = StackAccess::FPR;
4010
4011	StackAccesses [ArrIdx].AccessTypes \|= RegTy;
4012
4013	if (RegTy == StackAccess::FPR)
4014	++NumFPLdSt;
4015	else
4016	++NumNonFPLdSt;
4017	}
4018	}
4019	}
4020	}
4021
4022	if (NumFPLdSt == `0` \|\| NumNonFPLdSt == `0`)
4023	return;
4024
4025	llvm::sort(C&: StackAccesses);
4026	llvm::erase_if(C&: StackAccesses, P: [](const StackAccess &S) {
4027	return S.AccessTypes == StackAccess::NotAccessed;
4028	});
4029
4030	SmallVector<const StackAccess *> MixedObjects;
4031	SmallVector<std::pair<const StackAccess , const* StackAccess *>> HazardPairs;
4032
4033	if (StackAccesses.front().isMixed())
4034	MixedObjects.push_back(Elt: &StackAccesses.front());
4035
4036	for (auto It = StackAccesses.begin(), End = std::prev(x: StackAccesses.end());
4037	It != End; ++It) {
4038	const auto &First = *It;
4039	const auto &Second = *(It + `1`);
4040
4041	if (Second.isMixed())
4042	MixedObjects.push_back(Elt: &Second);
4043
4044	if ((First.isSME() && Second.isCPU()) \|\|
4045	(First.isCPU() && Second.isSME())) {
4046	uint64_t Distance = static_cast<uint64_t>(Second.start() - First.end());
4047	if (Distance < HazardSize)
4048	HazardPairs.emplace_back(Args: &First, Args: &Second);
4049	}
4050	}
4051
4052	auto EmitRemark = [&](llvm::StringRef Str) {
4053	ORE->emit(RemarkBuilder: [&]() {
4054	auto R = MachineOptimizationRemarkAnalysis (
4055	"sme", "StackHazard", MF.getFunction().getSubprogram(), &MF.front());
4056	return R << formatv(Fmt: "stack hazard in '{0}': ", Vals: MF.getName()).str() << Str;
4057	});
4058	};
4059
4060	for (const auto &P : HazardPairs)
4061	EmitRemark (formatv(Fmt: "{0} is too close to {1}", Vals: P.first, Vals: P.second).str());
4062
4063	for (const auto *Obj : MixedObjects)
4064	EmitRemark (
4065	formatv(Fmt: "{0} accessed by both GP and FP instructions", Vals: *Obj).str());
4066	}
4067

Browse the source code of llvm_projects/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp