ARMFrameLowering.cpp source code [llvm_projects/llvm/lib/Target/ARM/ARMFrameLowering.cpp]

1	//===- ARMFrameLowering.cpp - ARM Frame Information -----------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file contains the ARM implementation of TargetFrameLowering class.
10	//
11	//===----------------------------------------------------------------------===//
12	//
13	// This file contains the ARM implementation of TargetFrameLowering class.
14	//
15	// On ARM, stack frames are structured as follows:
16	//
17	// The stack grows downward.
18	//
19	// All of the individual frame areas on the frame below are optional, i.e. it's
20	// possible to create a function so that the particular area isn't present
21	// in the frame.
22	//
23	// At function entry, the "frame" looks as follows:
24	//
25	// \| \| Higher address
26	// \|-----------------------------------\|
27	// \| \|
28	// \| arguments passed on the stack \|
29	// \| \|
30	// \|-----------------------------------\| <- sp
31	// \| \| Lower address
32	//
33	//
34	// After the prologue has run, the frame has the following general structure.
35	// Technically the last frame area (VLAs) doesn't get created until in the
36	// main function body, after the prologue is run. However, it's depicted here
37	// for completeness.
38	//
39	// \| \| Higher address
40	// \|-----------------------------------\|
41	// \| \|
42	// \| arguments passed on the stack \|
43	// \| \|
44	// \|-----------------------------------\| <- (sp at function entry)
45	// \| \|
46	// \| varargs from registers \|
47	// \| \|
48	// \|-----------------------------------\|
49	// \| \|
50	// \| prev_lr \|
51	// \| prev_fp \|
52	// \| (a.k.a. "frame record") \|
53	// \| \|
54	// \|- - - - - - - - - - - - - - - - - -\| <- fp (r7 or r11)
55	// \| \|
56	// \| callee-saved gpr registers \|
57	// \| \|
58	// \|-----------------------------------\|
59	// \| \|
60	// \| callee-saved fp/simd regs \|
61	// \| \|
62	// \|-----------------------------------\|
63	// \|.empty.space.to.make.part.below....\|
64	// \|.aligned.in.case.it.needs.more.than\| (size of this area is unknown at
65	// \|.the.standard.8-byte.alignment.....\| compile time; if present)
66	// \|-----------------------------------\|
67	// \| \|
68	// \| local variables of fixed size \|
69	// \| including spill slots \|
70	// \|-----------------------------------\| <- base pointer (not defined by ABI,
71	// \|.variable-sized.local.variables....\| LLVM chooses r6)
72	// \|.(VLAs)............................\| (size of this area is unknown at
73	// \|...................................\| compile time)
74	// \|-----------------------------------\| <- sp
75	// \| \| Lower address
76	//
77	//
78	// To access the data in a frame, at-compile time, a constant offset must be
79	// computable from one of the pointers (fp, bp, sp) to access it. The size
80	// of the areas with a dotted background cannot be computed at compile-time
81	// if they are present, making it required to have all three of fp, bp and
82	// sp to be set up to be able to access all contents in the frame areas,
83	// assuming all of the frame areas are non-empty.
84	//
85	// For most functions, some of the frame areas are empty. For those functions,
86	// it may not be necessary to set up fp or bp:
87	// A base pointer is definitely needed when there are both VLAs and local*
88	// variables with more-than-default alignment requirements.
89	// A frame pointer is definitely needed when there are local variables with*
90	// more-than-default alignment requirements.
91	//
92	// In some cases when a base pointer is not strictly needed, it is generated
93	// anyway when offsets from the frame pointer to access local variables become
94	// so large that the offset can't be encoded in the immediate fields of loads
95	// or stores.
96	//
97	// The frame pointer might be chosen to be r7 or r11, depending on the target
98	// architecture and operating system. See ARMSubtarget::getFramePointerReg for
99	// details.
100	//
101	// Outgoing function arguments must be at the bottom of the stack frame when
102	// calling another function. If we do not have variable-sized stack objects, we
103	// can allocate a "reserved call frame" area at the bottom of the local
104	// variable area, large enough for all outgoing calls. If we do have VLAs, then
105	// the stack pointer must be decremented and incremented around each call to
106	// make space for the arguments below the VLAs.
107	//
108	//===----------------------------------------------------------------------===//
109
110	#include "ARMFrameLowering.h"
111	#include "ARMBaseInstrInfo.h"
112	#include "ARMBaseRegisterInfo.h"
113	#include "ARMConstantPoolValue.h"
114	#include "ARMMachineFunctionInfo.h"
115	#include "ARMSubtarget.h"
116	#include "MCTargetDesc/ARMAddressingModes.h"
117	#include "MCTargetDesc/ARMBaseInfo.h"
118	#include "Utils/ARMBaseInfo.h"
119	#include "llvm/ADT/BitVector.h"
120	#include "llvm/ADT/STLExtras.h"
121	#include "llvm/ADT/SmallPtrSet.h"
122	#include "llvm/ADT/SmallVector.h"
123	#include "llvm/CodeGen/CFIInstBuilder.h"
124	#include "llvm/CodeGen/MachineBasicBlock.h"
125	#include "llvm/CodeGen/MachineConstantPool.h"
126	#include "llvm/CodeGen/MachineFrameInfo.h"
127	#include "llvm/CodeGen/MachineFunction.h"
128	#include "llvm/CodeGen/MachineInstr.h"
129	#include "llvm/CodeGen/MachineInstrBuilder.h"
130	#include "llvm/CodeGen/MachineJumpTableInfo.h"
131	#include "llvm/CodeGen/MachineModuleInfo.h"
132	#include "llvm/CodeGen/MachineOperand.h"
133	#include "llvm/CodeGen/MachineRegisterInfo.h"
134	#include "llvm/CodeGen/RegisterScavenging.h"
135	#include "llvm/CodeGen/TargetInstrInfo.h"
136	#include "llvm/CodeGen/TargetRegisterInfo.h"
137	#include "llvm/CodeGen/TargetSubtargetInfo.h"
138	#include "llvm/IR/Attributes.h"
139	#include "llvm/IR/CallingConv.h"
140	#include "llvm/IR/DebugLoc.h"
141	#include "llvm/IR/Function.h"
142	#include "llvm/IR/Module.h"
143	#include "llvm/MC/MCAsmInfo.h"
144	#include "llvm/MC/MCInstrDesc.h"
145	#include "llvm/Support/CodeGen.h"
146	#include "llvm/Support/CommandLine.h"
147	#include "llvm/Support/Compiler.h"
148	#include "llvm/Support/Debug.h"
149	#include "llvm/Support/ErrorHandling.h"
150	#include "llvm/Support/raw_ostream.h"
151	#include "llvm/Target/TargetMachine.h"
152	#include <algorithm>
153	#include <cassert>
154	#include <cstddef>
155	#include <cstdint>
156	#include <iterator>
157	#include <utility>
158	#include <vector>
159
160	#define DEBUG_TYPE "arm-frame-lowering"
161
162	using namespace llvm;
163
164	static cl::opt<bool>
165	SpillAlignedNEONRegs("align-neon-spills", cl::Hidden, cl::init(Val: true),
166	cl::desc("Align ARM NEON spills in prolog and epilog"));
167
168	static MachineBasicBlock::iterator
169	skipAlignedDPRCS2Spills(MachineBasicBlock::iterator MI,
170	unsigned NumAlignedDPRCS2Regs);
171
172	enum class SpillArea {
173	GPRCS1,
174	GPRCS2,
175	FPStatus,
176	DPRCS1,
177	DPRCS2,
178	GPRCS3,
179	FPCXT,
180	};
181
182	/// Get the spill area that Reg should be saved into in the prologue.
183	SpillArea getSpillArea(Register Reg,
184	ARMSubtarget::PushPopSplitVariation Variation,
185	unsigned NumAlignedDPRCS2Regs,
186	const ARMBaseRegisterInfo *RegInfo) {
187	// NoSplit:
188	// push {r0-r12, lr} GPRCS1
189	// vpush {r8-d15} DPRCS1
190	//
191	// SplitR7:
192	// push {r0-r7, lr} GPRCS1
193	// push {r8-r12} GPRCS2
194	// vpush {r8-d15} DPRCS1
195	//
196	// SplitR11WindowsSEH:
197	// push {r0-r10, r12} GPRCS1
198	// vpush {r8-d15} DPRCS1
199	// push {r11, lr} GPRCS3
200	//
201	// SplitR11AAPCSSignRA:
202	// push {r0-r10, r12} GPRSC1
203	// push {r11, lr} GPRCS2
204	// vpush {r8-d15} DPRCS1
205
206	// If FPCXTNS is spilled (for CMSE secure entryfunctions), it is always at
207	// the top of the stack frame.
208	// The DPRCS2 region is used for ABIs which only guarantee 4-byte alignment
209	// of SP. If used, it will be below the other save areas, after the stack has
210	// been re-aligned.
211
212	switch (Reg) {
213	default:
214	dbgs() << "Don't know where to spill " << printReg(Reg, TRI: RegInfo) << "\n";
215	llvm_unreachable("Don't know where to spill this register");
216	break;
217
218	case ARM::FPCXTNS:
219	return SpillArea::FPCXT;
220
221	case ARM::FPSCR:
222	case ARM::FPEXC:
223	return SpillArea::FPStatus;
224
225	case ARM::R0:
226	case ARM::R1:
227	case ARM::R2:
228	case ARM::R3:
229	case ARM::R4:
230	case ARM::R5:
231	case ARM::R6:
232	case ARM::R7:
233	return SpillArea::GPRCS1;
234
235	case ARM::R8:
236	case ARM::R9:
237	case ARM::R10:
238	if (Variation == ARMSubtarget::SplitR7)
239	return SpillArea::GPRCS2;
240	else
241	return SpillArea::GPRCS1;
242
243	case ARM::R11:
244	if (Variation == ARMSubtarget::SplitR7 \|\|
245	Variation == ARMSubtarget::SplitR11AAPCSSignRA)
246	return SpillArea::GPRCS2;
247	if (Variation == ARMSubtarget::SplitR11WindowsSEH)
248	return SpillArea::GPRCS3;
249
250	return SpillArea::GPRCS1;
251
252	case ARM::R12:
253	if (Variation == ARMSubtarget::SplitR7)
254	return SpillArea::GPRCS2;
255	else
256	return SpillArea::GPRCS1;
257
258	case ARM::LR:
259	if (Variation == ARMSubtarget::SplitR11AAPCSSignRA)
260	return SpillArea::GPRCS2;
261	if (Variation == ARMSubtarget::SplitR11WindowsSEH)
262	return SpillArea::GPRCS3;
263
264	return SpillArea::GPRCS1;
265
266	case ARM::D0:
267	case ARM::D1:
268	case ARM::D2:
269	case ARM::D3:
270	case ARM::D4:
271	case ARM::D5:
272	case ARM::D6:
273	case ARM::D7:
274	return SpillArea::DPRCS1;
275
276	case ARM::D8:
277	case ARM::D9:
278	case ARM::D10:
279	case ARM::D11:
280	case ARM::D12:
281	case ARM::D13:
282	case ARM::D14:
283	case ARM::D15:
284	if (Reg >= ARM::D8 && Reg < ARM::D8 + NumAlignedDPRCS2Regs)
285	return SpillArea::DPRCS2;
286	else
287	return SpillArea::DPRCS1;
288
289	case ARM::D16:
290	case ARM::D17:
291	case ARM::D18:
292	case ARM::D19:
293	case ARM::D20:
294	case ARM::D21:
295	case ARM::D22:
296	case ARM::D23:
297	case ARM::D24:
298	case ARM::D25:
299	case ARM::D26:
300	case ARM::D27:
301	case ARM::D28:
302	case ARM::D29:
303	case ARM::D30:
304	case ARM::D31:
305	return SpillArea::DPRCS1;
306	}
307	}
308
309	ARMFrameLowering::ARMFrameLowering(const ARMSubtarget &sti)
310	: TargetFrameLowering (StackGrowsDown, sti.getStackAlignment(), `0`, Align (`4`)),
311	STI(sti) {}
312
313	bool ARMFrameLowering::keepFramePointer(const MachineFunction &MF) const {
314	// iOS always has a FP for backtracking, force other targets to keep their FP
315	// when doing FastISel. The emitted code is currently superior, and in cases
316	// like test-suite's lencod FastISel isn't quite correct when FP is eliminated.
317	return MF.getSubtarget<ARMSubtarget>().useFastISel();
318	}
319
320	/// Returns true if the target can safely skip saving callee-saved registers
321	/// for noreturn nounwind functions.
322	bool ARMFrameLowering::enableCalleeSaveSkip(const MachineFunction &MF) const {
323	assert(MF.getFunction().hasFnAttribute(Attribute::NoReturn) &&
324	MF.getFunction().hasFnAttribute(Attribute::NoUnwind) &&
325	!MF.getFunction().hasFnAttribute(Attribute::UWTable));
326
327	// Frame pointer and link register are not treated as normal CSR, thus we
328	// can always skip CSR saves for nonreturning functions.
329	return true;
330	}
331
332	/// hasFPImpl - Return true if the specified function should have a dedicated
333	/// frame pointer register. This is true if the function has variable sized
334	/// allocas or if frame pointer elimination is disabled.
335	bool ARMFrameLowering::hasFPImpl(const MachineFunction &MF) const {
336	const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
337	const MachineFrameInfo &MFI = MF.getFrameInfo();
338
339	// Check to see if the target want to forcibly keep frame pointer.
340	if (keepFramePointer(MF))
341	return true;
342
343	// ABI-required frame pointer.
344	if (MF.disableFramePointerElim())
345	return true;
346
347	// Frame pointer required for use within this function.
348	return (RegInfo->hasStackRealignment(MF) \|\| MFI.hasVarSizedObjects() \|\|
349	MFI.isFrameAddressTaken());
350	}
351
352	/// isFPReserved - Return true if the frame pointer register should be
353	/// considered a reserved register on the scope of the specified function.
354	bool ARMFrameLowering::isFPReserved(const MachineFunction &MF) const {
355	return hasFP(MF) \|\| MF.framePointerIsReserved();
356	}
357
358	/// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
359	/// not required, we reserve argument space for call sites in the function
360	/// immediately on entry to the current function. This eliminates the need for
361	/// add/sub sp brackets around call sites. Returns true if the call frame is
362	/// included as part of the stack frame.
363	bool ARMFrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
364	const MachineFrameInfo &MFI = MF.getFrameInfo();
365	unsigned CFSize = MFI.getMaxCallFrameSize();
366	// It's not always a good idea to include the call frame as part of the
367	// stack frame. ARM (especially Thumb) has small immediate offset to
368	// address the stack frame. So a large call frame can cause poor codegen
369	// and may even makes it impossible to scavenge a register.
370	if (CFSize >= ((`1` << `12`) - `1`) / `2`) // Half of imm12
371	return false;
372
373	return !MFI.hasVarSizedObjects();
374	}
375
376	/// canSimplifyCallFramePseudos - If there is a reserved call frame, the
377	/// call frame pseudos can be simplified. Unlike most targets, having a FP
378	/// is not sufficient here since we still may reference some objects via SP
379	/// even when FP is available in Thumb2 mode.
380	bool
381	ARMFrameLowering::canSimplifyCallFramePseudos(const MachineFunction &MF) const {
382	return hasReservedCallFrame(MF) \|\| MF.getFrameInfo().hasVarSizedObjects();
383	}
384
385	// Returns how much of the incoming argument stack area we should clean up in an
386	// epilogue. For the C calling convention this will be 0, for guaranteed tail
387	// call conventions it can be positive (a normal return or a tail call to a
388	// function that uses less stack space for arguments) or negative (for a tail
389	// call to a function that needs more stack space than us for arguments).
390	static int getArgumentStackToRestore(MachineFunction &MF,
391	MachineBasicBlock &MBB) {
392	MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
393	bool IsTailCallReturn = false;
394	if (MBB.end() != MBBI) {
395	unsigned RetOpcode = MBBI ->getOpcode();
396	IsTailCallReturn = RetOpcode == ARM::TCRETURNdi \|\|
397	RetOpcode == ARM::TCRETURNri \|\|
398	RetOpcode == ARM::TCRETURNrinotr12;
399	}
400	ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
401
402	int ArgumentPopSize = `0`;
403	if (IsTailCallReturn) {
404	MachineOperand &StackAdjust = MBBI ->getOperand(i: `1`);
405
406	// For a tail-call in a callee-pops-arguments environment, some or all of
407	// the stack may actually be in use for the call's arguments, this is
408	// calculated during LowerCall and consumed here...
409	ArgumentPopSize = StackAdjust.getImm();
410	} else {
411	// ... otherwise the amount to pop is all* of the argument space,*
412	// conveniently stored in the MachineFunctionInfo by
413	// LowerFormalArguments. This will, of course, be zero for the C calling
414	// convention.
415	ArgumentPopSize = AFI->getArgumentStackToRestore();
416	}
417
418	return ArgumentPopSize;
419	}
420
421	static bool needsWinCFI(const MachineFunction &MF) {
422	const Function &F = MF.getFunction();
423	return MF.getTarget().getMCAsmInfo().usesWindowsCFI() &&
424	F.needsUnwindTableEntry();
425	}
426
427	// Given a load or a store instruction, generate an appropriate unwinding SEH
428	// code on Windows.
429	static MachineBasicBlock::iterator insertSEH(MachineBasicBlock::iterator MBBI,
430	const TargetInstrInfo &TII,
431	unsigned Flags) {
432	unsigned Opc = MBBI ->getOpcode();
433	MachineBasicBlock *MBB = MBBI ->getParent();
434	MachineFunction &MF = *MBB->getParent();
435	DebugLoc DL = MBBI ->getDebugLoc();
436	MachineInstrBuilder MIB;
437	const ARMSubtarget &Subtarget = MF.getSubtarget<ARMSubtarget>();
438	const ARMBaseRegisterInfo *RegInfo = Subtarget.getRegisterInfo();
439
440	Flags \|= MachineInstr::NoMerge;
441
442	switch (Opc) {
443	default:
444	report_fatal_error(reason: "No SEH Opcode for instruction " + TII.getName(Opcode: Opc));
445	break;
446	case ARM::t2ADDri: // add.w r11, sp, #xx
447	case ARM::t2ADDri12: // add.w r11, sp, #xx
448	case ARM::t2MOVTi16: // movt r4, #xx
449	case ARM::tBL: // bl __chkstk
450	// These are harmless if used for just setting up a frame pointer,
451	// but that frame pointer can't be relied upon for unwinding, unless
452	// set up with SEH_SaveSP.
453	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: ARM::SEH_Nop))
454	.addImm(/Wide=/Val: `1`)
455	.setMIFlags(Flags);
456	break;
457
458	case ARM::t2MOVi16: { // mov(w) r4, #xx
459	bool Wide = MBBI ->getOperand(i: `1`).getImm() >= `256`;
460	if (!Wide) {
461	MachineInstrBuilder NewInstr =
462	BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: ARM::tMOVi8)).setMIFlags(MBBI ->getFlags());
463	NewInstr.add(MO: MBBI ->getOperand(i: `0`));
464	NewInstr.add(MO: t1CondCodeOp(/isDead=/true));
465	for (MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MBBI ->operands()))
466	NewInstr.add(MO);
467	MachineBasicBlock::iterator NewMBBI = MBB->insertAfter(I: MBBI, MI: NewInstr);
468	MBB->erase(I: MBBI);
469	MBBI = NewMBBI;
470	}
471	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: ARM::SEH_Nop)).addImm(Val: Wide).setMIFlags(Flags);
472	break;
473	}
474
475	case ARM::tBLXr: // blx r12 (__chkstk)
476	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: ARM::SEH_Nop))
477	.addImm(/Wide=/Val: `0`)
478	.setMIFlags(Flags);
479	break;
480
481	case ARM::t2MOVi32imm: // movw+movt
482	// This pseudo instruction expands into two mov instructions. If the
483	// second operand is a symbol reference, this will stay as two wide
484	// instructions, movw+movt. If they're immediates, the first one can
485	// end up as a narrow mov though.
486	// As two SEH instructions are appended here, they won't get interleaved
487	// between the two final movw/movt instructions, but it doesn't make any
488	// practical difference.
489	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: ARM::SEH_Nop))
490	.addImm(/Wide=/Val: `1`)
491	.setMIFlags(Flags);
492	MBB->insertAfter(I: MBBI, MI: MIB);
493	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: ARM::SEH_Nop))
494	.addImm(/Wide=/Val: `1`)
495	.setMIFlags(Flags);
496	break;
497
498	case ARM::t2STR_PRE:
499	if (MBBI ->getOperand(i: `0`).getReg() == ARM::SP &&
500	MBBI ->getOperand(i: `2`).getReg() == ARM::SP &&
501	MBBI ->getOperand(i: `3`).getImm() == -`4`) {
502	unsigned Reg = RegInfo->getSEHRegNum(i: MBBI ->getOperand(i: `1`).getReg());
503	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: ARM::SEH_SaveRegs))
504	.addImm(Val: `1ULL` << Reg)
505	.addImm(/Wide=/Val: `1`)
506	.setMIFlags(Flags);
507	} else {
508	report_fatal_error(reason: "No matching SEH Opcode for t2STR_PRE");
509	}
510	break;
511
512	case ARM::t2LDR_POST:
513	if (MBBI ->getOperand(i: `1`).getReg() == ARM::SP &&
514	MBBI ->getOperand(i: `2`).getReg() == ARM::SP &&
515	MBBI ->getOperand(i: `3`).getImm() == `4`) {
516	unsigned Reg = RegInfo->getSEHRegNum(i: MBBI ->getOperand(i: `0`).getReg());
517	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: ARM::SEH_SaveRegs))
518	.addImm(Val: `1ULL` << Reg)
519	.addImm(/Wide=/Val: `1`)
520	.setMIFlags(Flags);
521	} else {
522	report_fatal_error(reason: "No matching SEH Opcode for t2LDR_POST");
523	}
524	break;
525
526	case ARM::t2LDMIA_RET:
527	case ARM::t2LDMIA_UPD:
528	case ARM::t2STMDB_UPD: {
529	unsigned Mask = `0`;
530	bool Wide = false;
531	for (unsigned i = `4`, NumOps = MBBI ->getNumOperands(); i != NumOps; ++i) {
532	const MachineOperand &MO = MBBI ->getOperand(i);
533	if (!MO.isReg() \|\| MO.isImplicit())
534	continue;
535	unsigned Reg = RegInfo->getSEHRegNum(i: MO.getReg());
536	if (Reg == `15`)
537	Reg = `14`;
538	if (Reg >= `8` && Reg <= `13`)
539	Wide = true;
540	else if (Opc == ARM::t2LDMIA_UPD && Reg == `14`)
541	Wide = true;
542	Mask \|= `1` << Reg;
543	}
544	if (!Wide) {
545	unsigned NewOpc;
546	switch (Opc) {
547	case ARM::t2LDMIA_RET:
548	NewOpc = ARM::tPOP_RET;
549	break;
550	case ARM::t2LDMIA_UPD:
551	NewOpc = ARM::tPOP;
552	break;
553	case ARM::t2STMDB_UPD:
554	NewOpc = ARM::tPUSH;
555	break;
556	default:
557	llvm_unreachable("");
558	}
559	MachineInstrBuilder NewInstr =
560	BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: NewOpc)).setMIFlags(MBBI ->getFlags());
561	for (unsigned i = `2`, NumOps = MBBI ->getNumOperands(); i != NumOps; ++i)
562	NewInstr.add(MO: MBBI ->getOperand(i));
563	MachineBasicBlock::iterator NewMBBI = MBB->insertAfter(I: MBBI, MI: NewInstr);
564	MBB->erase(I: MBBI);
565	MBBI = NewMBBI;
566	}
567	unsigned SEHOpc =
568	(Opc == ARM::t2LDMIA_RET) ? ARM::SEH_SaveRegs_Ret : ARM::SEH_SaveRegs;
569	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: SEHOpc))
570	.addImm(Val: Mask)
571	.addImm(Val: Wide ? `1` : `0`)
572	.setMIFlags(Flags);
573	break;
574	}
575	case ARM::VSTMDDB_UPD:
576	case ARM::VLDMDIA_UPD: {
577	int First = -`1`, Last = `0`;
578	for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MBBI ->operands(), N: `4`)) {
579	unsigned Reg = RegInfo->getSEHRegNum(i: MO.getReg());
580	if (First == -`1`)
581	First = Reg;
582	Last = Reg;
583	}
584	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: ARM::SEH_SaveFRegs))
585	.addImm(Val: First)
586	.addImm(Val: Last)
587	.setMIFlags(Flags);
588	break;
589	}
590	case ARM::tSUBspi:
591	case ARM::tADDspi:
592	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: ARM::SEH_StackAlloc))
593	.addImm(Val: MBBI ->getOperand(i: `2`).getImm() * `4`)
594	.addImm(/Wide=/Val: `0`)
595	.setMIFlags(Flags);
596	break;
597	case ARM::t2SUBspImm:
598	case ARM::t2SUBspImm12:
599	case ARM::t2ADDspImm:
600	case ARM::t2ADDspImm12:
601	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: ARM::SEH_StackAlloc))
602	.addImm(Val: MBBI ->getOperand(i: `2`).getImm())
603	.addImm(/Wide=/Val: `1`)
604	.setMIFlags(Flags);
605	break;
606
607	case ARM::tMOVr:
608	if (MBBI ->getOperand(i: `1`).getReg() == ARM::SP &&
609	(Flags & MachineInstr::FrameSetup)) {
610	unsigned Reg = RegInfo->getSEHRegNum(i: MBBI ->getOperand(i: `0`).getReg());
611	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: ARM::SEH_SaveSP))
612	.addImm(Val: Reg)
613	.setMIFlags(Flags);
614	} else if (MBBI ->getOperand(i: `0`).getReg() == ARM::SP &&
615	(Flags & MachineInstr::FrameDestroy)) {
616	unsigned Reg = RegInfo->getSEHRegNum(i: MBBI ->getOperand(i: `1`).getReg());
617	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: ARM::SEH_SaveSP))
618	.addImm(Val: Reg)
619	.setMIFlags(Flags);
620	} else {
621	report_fatal_error(reason: "No SEH Opcode for MOV");
622	}
623	break;
624
625	case ARM::tBX_RET:
626	case ARM::t2BXAUT_RET:
627	case ARM::CLEANUPRET:
628	case ARM::CATCHRET:
629	case ARM::TCRETURNri:
630	case ARM::TCRETURNrinotr12:
631	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: ARM::SEH_Nop_Ret))
632	.addImm(/Wide=/Val: `0`)
633	.setMIFlags(Flags);
634	break;
635
636	case ARM::TCRETURNdi:
637	MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: ARM::SEH_Nop_Ret))
638	.addImm(/Wide=/Val: `1`)
639	.setMIFlags(Flags);
640	break;
641	}
642	return MBB->insertAfter(I: MBBI, MI: MIB);
643	}
644
645	static MachineBasicBlock::iterator
646	initMBBRange(MachineBasicBlock &MBB, const MachineBasicBlock::iterator &MBBI) {
647	if (MBBI == MBB.begin())
648	return MachineBasicBlock::iterator();
649	return std::prev(x: MBBI);
650	}
651
652	static void insertSEHRange(MachineBasicBlock &MBB,
653	MachineBasicBlock::iterator Start,
654	const MachineBasicBlock::iterator &End,
655	const ARMBaseInstrInfo &TII, unsigned MIFlags) {
656	if (Start.isValid())
657	Start = std::next(x: Start);
658	else
659	Start = MBB.begin();
660
661	for (auto MI = Start; MI != End;) {
662	auto Next = std::next(x: MI);
663	// Check if this instruction already has got a SEH opcode added. In that
664	// case, don't do this generic mapping.
665	if (Next != End && isSEHInstruction(MI: *Next)) {
666	MI = std::next(x: Next);
667	while (MI != End && isSEHInstruction(MI: *MI))
668	++MI;
669	continue;
670	}
671	insertSEH(MBBI: MI, TII, Flags: MIFlags);
672	MI = Next;
673	}
674	}
675
676	static void emitRegPlusImmediate(
677	bool isARM, MachineBasicBlock &MBB, MachineBasicBlock::iterator &MBBI,
678	const DebugLoc &dl, const ARMBaseInstrInfo &TII, unsigned DestReg,
679	unsigned SrcReg, int NumBytes, unsigned MIFlags = MachineInstr::NoFlags,
680	ARMCC::CondCodes Pred = ARMCC::AL, unsigned PredReg = `0`) {
681	if (isARM)
682	emitARMRegPlusImmediate(MBB, MBBI, dl, DestReg, BaseReg: SrcReg, NumBytes,
683	Pred, PredReg, TII, MIFlags);
684	else
685	emitT2RegPlusImmediate(MBB, MBBI, dl, DestReg, BaseReg: SrcReg, NumBytes,
686	Pred, PredReg, TII, MIFlags);
687	}
688
689	static void emitSPUpdate(bool isARM, MachineBasicBlock &MBB,
690	MachineBasicBlock::iterator &MBBI, const DebugLoc &dl,
691	const ARMBaseInstrInfo &TII, int NumBytes,
692	unsigned MIFlags = MachineInstr::NoFlags,
693	ARMCC::CondCodes Pred = ARMCC::AL,
694	unsigned PredReg = `0`) {
695	emitRegPlusImmediate(isARM, MBB, MBBI, dl, TII, DestReg: ARM::SP, SrcReg: ARM::SP, NumBytes,
696	MIFlags, Pred, PredReg);
697	}
698
699	static int sizeOfSPAdjustment(const MachineInstr &MI) {
700	int RegSize;
701	switch (MI.getOpcode()) {
702	case ARM::VSTMDDB_UPD:
703	RegSize = `8`;
704	break;
705	case ARM::STMDB_UPD:
706	case ARM::t2STMDB_UPD:
707	RegSize = `4`;
708	break;
709	case ARM::t2STR_PRE:
710	case ARM::STR_PRE_IMM:
711	return `4`;
712	default:
713	llvm_unreachable("Unknown push or pop like instruction");
714	}
715
716	int count = `0`;
717	// ARM and Thumb2 push/pop insts have explicit "sp, sp" operands (+
718	// pred) so the list starts at 4.
719	for (int i = MI.getNumOperands() - `1`; i >= `4`; --i)
720	count += RegSize;
721	return count;
722	}
723
724	static bool WindowsRequiresStackProbe(const MachineFunction &MF,
725	size_t StackSizeInBytes) {
726	const MachineFrameInfo &MFI = MF.getFrameInfo();
727	const Function &F = MF.getFunction();
728	unsigned StackProbeSize = (MFI.getStackProtectorIndex() > `0`) ? `4080` : `4096`;
729
730	StackProbeSize =
731	F.getFnAttributeAsParsedInteger(Kind: "stack-probe-size", Default: StackProbeSize);
732	return (StackSizeInBytes >= StackProbeSize) &&
733	!F.hasFnAttribute(Kind: "no-stack-arg-probe");
734	}
735
736	namespace {
737
738	struct StackAdjustingInsts {
739	struct InstInfo {
740	MachineBasicBlock::iterator I;
741	unsigned SPAdjust;
742	bool BeforeFPSet;
743
744	#if !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
745	void dump() {
746	dbgs() << " " << (BeforeFPSet ? "before-fp " : " ")
747	<< "sp-adjust=" << SPAdjust;
748	I->dump();
749	}
750	#endif
751	};
752
753	SmallVector<InstInfo, `4`> Insts;
754
755	void addInst(MachineBasicBlock::iterator I, unsigned SPAdjust,
756	bool BeforeFPSet = false) {
757	InstInfo Info = {.I: I, .SPAdjust: SPAdjust, .BeforeFPSet: BeforeFPSet};
758	Insts.push_back(Elt: Info);
759	}
760
761	void addExtraBytes(const MachineBasicBlock::iterator I, unsigned ExtraBytes) {
762	auto Info =
763	llvm::find_if(Range&: Insts, P: [&](InstInfo &Info) { return Info.I == I; });
764	assert(Info != Insts.end() && "invalid sp adjusting instruction");
765	Info->SPAdjust += ExtraBytes;
766	}
767
768	void emitDefCFAOffsets(MachineBasicBlock &MBB, bool HasFP) {
769	CFIInstBuilder CFIBuilder(MBB, MBB.end(), MachineInstr::FrameSetup);
770	unsigned CFAOffset = `0`;
771	for (auto &Info : Insts) {
772	if (HasFP && !Info.BeforeFPSet)
773	return;
774
775	CFAOffset += Info.SPAdjust;
776	CFIBuilder.setInsertPoint(std::next(x: Info.I));
777	CFIBuilder.buildDefCFAOffset(Offset: CFAOffset);
778	}
779	}
780
781	#if !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
782	void dump() {
783	dbgs() << "StackAdjustingInsts:\n";
784	for (auto &Info : Insts)
785	Info.dump();
786	}
787	#endif
788	};
789
790	} // end anonymous namespace
791
792	/// Emit an instruction sequence that will align the address in
793	/// register Reg by zero-ing out the lower bits. For versions of the
794	/// architecture that support Neon, this must be done in a single
795	/// instruction, since skipAlignedDPRCS2Spills assumes it is done in a
796	/// single instruction. That function only gets called when optimizing
797	/// spilling of D registers on a core with the Neon instruction set
798	/// present.
799	static void emitAligningInstructions(MachineFunction &MF, ARMFunctionInfo *AFI,
800	const TargetInstrInfo &TII,
801	MachineBasicBlock &MBB,
802	MachineBasicBlock::iterator MBBI,
803	const DebugLoc &DL, const unsigned Reg,
804	const Align Alignment,
805	const bool MustBeSingleInstruction) {
806	const ARMSubtarget &AST = MF.getSubtarget<ARMSubtarget>();
807	const bool CanUseBFC = AST.hasV6T2Ops() \|\| AST.hasV7Ops();
808	const unsigned AlignMask = Alignment.value() - `1U`;
809	const unsigned NrBitsToZero = Log2(A: Alignment);
810	assert(!AFI->isThumb1OnlyFunction() && "Thumb1 not supported");
811	if (!AFI->isThumbFunction()) {
812	// if the BFC instruction is available, use that to zero the lower
813	// bits:
814	// bfc Reg, #0, log2(Alignment)
815	// otherwise use BIC, if the mask to zero the required number of bits
816	// can be encoded in the bic immediate field
817	// bic Reg, Reg, Alignment-1
818	// otherwise, emit
819	// lsr Reg, Reg, log2(Alignment)
820	// lsl Reg, Reg, log2(Alignment)
821	if (CanUseBFC) {
822	BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: ARM::BFC), DestReg: Reg)
823	.addReg(RegNo: Reg, Flags: RegState::Kill)
824	.addImm(Val: ~AlignMask)
825	.add(MOs: predOps(Pred: ARMCC::AL));
826	} else if (AlignMask <= `255`) {
827	BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: ARM::BICri), DestReg: Reg)
828	.addReg(RegNo: Reg, Flags: RegState::Kill)
829	.addImm(Val: AlignMask)
830	.add(MOs: predOps(Pred: ARMCC::AL))
831	.add(MO: condCodeOp());
832	} else {
833	assert(!MustBeSingleInstruction &&
834	"Shouldn't call emitAligningInstructions demanding a single "
835	"instruction to be emitted for large stack alignment for a target "
836	"without BFC.");
837	BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: ARM::MOVsi), DestReg: Reg)
838	.addReg(RegNo: Reg, Flags: RegState::Kill)
839	.addImm(Val: ARM_AM::getSORegOpc(ShOp: ARM_AM::lsr, Imm: NrBitsToZero))
840	.add(MOs: predOps(Pred: ARMCC::AL))
841	.add(MO: condCodeOp());
842	BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: ARM::MOVsi), DestReg: Reg)
843	.addReg(RegNo: Reg, Flags: RegState::Kill)
844	.addImm(Val: ARM_AM::getSORegOpc(ShOp: ARM_AM::lsl, Imm: NrBitsToZero))
845	.add(MOs: predOps(Pred: ARMCC::AL))
846	.add(MO: condCodeOp());
847	}
848	} else {
849	// Since this is only reached for Thumb-2 targets, the BFC instruction
850	// should always be available.
851	assert(CanUseBFC);
852	BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: ARM::t2BFC), DestReg: Reg)
853	.addReg(RegNo: Reg, Flags: RegState::Kill)
854	.addImm(Val: ~AlignMask)
855	.add(MOs: predOps(Pred: ARMCC::AL));
856	}
857	}
858
859	/// We need the offset of the frame pointer relative to other MachineFrameInfo
860	/// offsets which are encoded relative to SP at function begin.
861	/// See also emitPrologue() for how the FP is set up.
862	/// Unfortunately we cannot determine this value in determineCalleeSaves() yet
863	/// as assignCalleeSavedSpillSlots() hasn't run at this point. Instead we use
864	/// this to produce a conservative estimate that we check in an assert() later.
865	static int getMaxFPOffset(const ARMSubtarget &STI, const ARMFunctionInfo &AFI,
866	const MachineFunction &MF) {
867	ARMSubtarget::PushPopSplitVariation PushPopSplit =
868	STI.getPushPopSplitVariation(MF);
869	// For Thumb1, push.w isn't available, so the first push will always push
870	// r7 and lr onto the stack first.
871	if (AFI.isThumb1OnlyFunction())
872	return -AFI.getArgRegsSaveSize() - (`2` * `4`);
873	// This is a conservative estimation: Assume the frame pointer being r7 and
874	// pc("r15") up to r8 getting spilled before (= 8 registers).
875	int MaxRegBytes = `8` * `4`;
876	if (PushPopSplit == ARMSubtarget::SplitR11AAPCSSignRA)
877	// Here, r11 can be stored below all of r4-r15.
878	MaxRegBytes = `11` * `4`;
879	if (PushPopSplit == ARMSubtarget::SplitR11WindowsSEH) {
880	// Here, r11 can be stored below all of r4-r15 plus d8-d15.
881	MaxRegBytes = `11` * `4` + `8` * `8`;
882	}
883	int FPCXTSaveSize =
884	(STI.hasV8_1MMainlineOps() && AFI.isCmseNSEntryFunction()) ? `4` : `0`;
885	return -FPCXTSaveSize - AFI.getArgRegsSaveSize() - MaxRegBytes;
886	}
887
888	void ARMFrameLowering::emitPrologue(MachineFunction &MF,
889	MachineBasicBlock &MBB) const {
890	MachineBasicBlock::iterator MBBI = MBB.begin();
891	MachineFrameInfo &MFI = MF.getFrameInfo();
892	ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
893	const TargetMachine &TM = MF.getTarget();
894	const ARMBaseRegisterInfo *RegInfo = STI.getRegisterInfo();
895	const ARMBaseInstrInfo &TII = *STI.getInstrInfo();
896	assert(!AFI->isThumb1OnlyFunction() &&
897	"This emitPrologue does not support Thumb1!");
898	bool isARM = !AFI->isThumbFunction();
899	Align Alignment = STI.getFrameLowering()->getStackAlign();
900	unsigned ArgRegsSaveSize = AFI->getArgRegsSaveSize();
901	unsigned NumBytes = MFI.getStackSize();
902	const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
903	int FPCXTSaveSize = `0`;
904	bool NeedsWinCFI = needsWinCFI(MF);
905	ARMSubtarget::PushPopSplitVariation PushPopSplit =
906	STI.getPushPopSplitVariation(MF);
907
908	LLVM_DEBUG(dbgs() << "Emitting prologue for " << MF.getName() << "\n");
909
910	// Debug location must be unknown since the first debug location is used
911	// to determine the end of the prologue.
912	DebugLoc dl;
913
914	Register FramePtr = RegInfo->getFrameRegister(MF);
915
916	// Determine the sizes of each callee-save spill areas and record which frame
917	// belongs to which callee-save spill areas.
918	unsigned GPRCS1Size = `0`, GPRCS2Size = `0`, FPStatusSize = `0`,
919	DPRCS1Size = `0`, GPRCS3Size = `0`, DPRCS2Size = `0`;
920	int FramePtrSpillFI = `0`;
921	int D8SpillFI = `0`;
922
923	// All calls are tail calls in GHC calling conv, and functions have no
924	// prologue/epilogue.
925	if (MF.getFunction().getCallingConv() == CallingConv::GHC)
926	return;
927
928	StackAdjustingInsts DefCFAOffsetCandidates;
929	bool HasFP = hasFP(MF);
930
931	if (!AFI->hasStackFrame() &&
932	(!STI.isTargetWindows() \|\| !WindowsRequiresStackProbe(MF, StackSizeInBytes: NumBytes))) {
933	if (NumBytes != `0`) {
934	emitSPUpdate(isARM, MBB, MBBI, dl, TII, NumBytes: -NumBytes,
935	MIFlags: MachineInstr::FrameSetup);
936	DefCFAOffsetCandidates.addInst(I: std::prev(x: MBBI), SPAdjust: NumBytes, BeforeFPSet: true);
937	}
938	if (!NeedsWinCFI)
939	DefCFAOffsetCandidates.emitDefCFAOffsets(MBB, HasFP);
940	if (NeedsWinCFI && MBBI != MBB.begin()) {
941	insertSEHRange(MBB, Start: {}, End: MBBI, TII, MIFlags: MachineInstr::FrameSetup);
942	BuildMI(BB&: MBB, I: MBBI, MIMD: dl, MCID: TII.get(Opcode: ARM::SEH_PrologEnd))
943	.setMIFlag(MachineInstr::FrameSetup);
944	MF.setHasWinCFI(true);
945	}
946	return;
947	}
948
949	// Determine spill area sizes, and some important frame indices.
950	SpillArea FramePtrSpillArea = SpillArea::GPRCS1;
951	bool BeforeFPPush = true;
952	for (const CalleeSavedInfo &I : CSI) {
953	MCRegister Reg = I.getReg();
954	int FI = I.getFrameIdx();
955
956	SpillArea Area = getSpillArea(Reg, Variation: PushPopSplit,
957	NumAlignedDPRCS2Regs: AFI->getNumAlignedDPRCS2Regs(), RegInfo);
958
959	if (Reg == FramePtr.asMCReg()) {
960	FramePtrSpillFI = FI;
961	FramePtrSpillArea = Area;
962	}
963	if (Reg == ARM::D8)
964	D8SpillFI = FI;
965
966	switch (Area) {
967	case SpillArea::FPCXT:
968	FPCXTSaveSize += `4`;
969	break;
970	case SpillArea::GPRCS1:
971	GPRCS1Size += `4`;
972	break;
973	case SpillArea::GPRCS2:
974	GPRCS2Size += `4`;
975	break;
976	case SpillArea::FPStatus:
977	FPStatusSize += `4`;
978	break;
979	case SpillArea::DPRCS1:
980	DPRCS1Size += `8`;
981	break;
982	case SpillArea::GPRCS3:
983	GPRCS3Size += `4`;
984	break;
985	case SpillArea::DPRCS2:
986	DPRCS2Size += `8`;
987	break;
988	}
989	}
990
991	MachineBasicBlock::iterator LastPush = MBB.end(), GPRCS1Push, GPRCS2Push,
992	DPRCS1Push, GPRCS3Push;
993
994	// Move past the PAC computation.
995	if (AFI->shouldSignReturnAddress())
996	LastPush = MBBI ++;
997
998	// Move past FPCXT area.
999	if (FPCXTSaveSize > `0`) {
1000	LastPush = MBBI ++;
1001	DefCFAOffsetCandidates.addInst(I: LastPush, SPAdjust: FPCXTSaveSize, BeforeFPSet: BeforeFPPush);
1002	}
1003
1004	// Allocate the vararg register save area.
1005	if (ArgRegsSaveSize) {
1006	emitSPUpdate(isARM, MBB, MBBI, dl, TII, NumBytes: -ArgRegsSaveSize,
1007	MIFlags: MachineInstr::FrameSetup);
1008	LastPush = std::prev(x: MBBI);
1009	DefCFAOffsetCandidates.addInst(I: LastPush, SPAdjust: ArgRegsSaveSize, BeforeFPSet: BeforeFPPush);
1010	}
1011
1012	// Move past area 1.
1013	if (GPRCS1Size > `0`) {
1014	GPRCS1Push = LastPush = MBBI ++;
1015	DefCFAOffsetCandidates.addInst(I: LastPush, SPAdjust: GPRCS1Size, BeforeFPSet: BeforeFPPush);
1016	if (FramePtrSpillArea == SpillArea::GPRCS1)
1017	BeforeFPPush = false;
1018	}
1019
1020	// Determine starting offsets of spill areas. These offsets are all positive
1021	// offsets from the bottom of the lowest-addressed callee-save area
1022	// (excluding DPRCS2, which is th the re-aligned stack region) to the bottom
1023	// of the spill area in question.
1024	unsigned FPCXTOffset = NumBytes - ArgRegsSaveSize - FPCXTSaveSize;
1025	unsigned GPRCS1Offset = FPCXTOffset - GPRCS1Size;
1026	unsigned GPRCS2Offset = GPRCS1Offset - GPRCS2Size;
1027	unsigned FPStatusOffset = GPRCS2Offset - FPStatusSize;
1028
1029	Align DPRAlign = DPRCS1Size ? std::min(a: Align (`8`), b: Alignment) : Align (`4`);
1030	unsigned DPRGapSize = (ArgRegsSaveSize + FPCXTSaveSize + GPRCS1Size +
1031	GPRCS2Size + FPStatusSize) %
1032	DPRAlign.value();
1033
1034	unsigned DPRCS1Offset = FPStatusOffset - DPRGapSize - DPRCS1Size;
1035
1036	if (HasFP) {
1037	// Offset from the CFA to the saved frame pointer, will be negative.
1038	[[maybe_unused]] int FPOffset = MFI.getObjectOffset(ObjectIdx: FramePtrSpillFI);
1039	LLVM_DEBUG(dbgs() << "FramePtrSpillFI: " << FramePtrSpillFI
1040	<< ", FPOffset: " << FPOffset << "\n");
1041	assert(getMaxFPOffset(STI, *AFI, MF) <= FPOffset &&
1042	"Max FP estimation is wrong");
1043	AFI->setFramePtrSpillOffset(MFI.getObjectOffset(ObjectIdx: FramePtrSpillFI) +
1044	NumBytes);
1045	}
1046	AFI->setGPRCalleeSavedArea1Offset(GPRCS1Offset);
1047	AFI->setGPRCalleeSavedArea2Offset(GPRCS2Offset);
1048	AFI->setDPRCalleeSavedArea1Offset(DPRCS1Offset);
1049
1050	// Move past area 2.
1051	if (GPRCS2Size > `0`) {
1052	assert(PushPopSplit != ARMSubtarget::SplitR11WindowsSEH);
1053	GPRCS2Push = LastPush = MBBI ++;
1054	DefCFAOffsetCandidates.addInst(I: LastPush, SPAdjust: GPRCS2Size, BeforeFPSet: BeforeFPPush);
1055	if (FramePtrSpillArea == SpillArea::GPRCS2)
1056	BeforeFPPush = false;
1057	}
1058
1059	// Move past FP status save area.
1060	if (FPStatusSize > `0`) {
1061	while (MBBI != MBB.end()) {
1062	unsigned Opc = MBBI ->getOpcode();
1063	if (Opc == ARM::VMRS \|\| Opc == ARM::VMRS_FPEXC)
1064	MBBI ++;
1065	else
1066	break;
1067	}
1068	LastPush = MBBI ++;
1069	DefCFAOffsetCandidates.addInst(I: LastPush, SPAdjust: FPStatusSize);
1070	}
1071
1072	// Prolog/epilog inserter assumes we correctly align DPRs on the stack, so our
1073	// .cfi_offset operations will reflect that.
1074	if (DPRGapSize) {
1075	assert(DPRGapSize == `4` && "unexpected alignment requirements for DPRs");
1076	if (LastPush != MBB.end() &&
1077	tryFoldSPUpdateIntoPushPop(Subtarget: STI, MF, MI: &*LastPush, NumBytes: DPRGapSize))
1078	DefCFAOffsetCandidates.addExtraBytes(I: LastPush, ExtraBytes: DPRGapSize);
1079	else {
1080	emitSPUpdate(isARM, MBB, MBBI, dl, TII, NumBytes: -DPRGapSize,
1081	MIFlags: MachineInstr::FrameSetup);
1082	DefCFAOffsetCandidates.addInst(I: std::prev(x: MBBI), SPAdjust: DPRGapSize, BeforeFPSet: BeforeFPPush);
1083	}
1084	}
1085
1086	// Move past DPRCS1Size.
1087	if (DPRCS1Size > `0`) {
1088	// Since vpush register list cannot have gaps, there may be multiple vpush
1089	// instructions in the prologue.
1090	while (MBBI != MBB.end() && MBBI ->getOpcode() == ARM::VSTMDDB_UPD) {
1091	DefCFAOffsetCandidates.addInst(I: MBBI, SPAdjust: sizeOfSPAdjustment(MI: *MBBI),
1092	BeforeFPSet: BeforeFPPush);
1093	DPRCS1Push = LastPush = MBBI ++;
1094	}
1095	}
1096
1097	// Move past the aligned DPRCS2 area.
1098	if (DPRCS2Size > `0`) {
1099	MBBI = skipAlignedDPRCS2Spills(MI: MBBI, NumAlignedDPRCS2Regs: AFI->getNumAlignedDPRCS2Regs());
1100	// The code inserted by emitAlignedDPRCS2Spills realigns the stack, and
1101	// leaves the stack pointer pointing to the DPRCS2 area.
1102	//
1103	// Adjust NumBytes to represent the stack slots below the DPRCS2 area.
1104	NumBytes += MFI.getObjectOffset(ObjectIdx: D8SpillFI);
1105	} else
1106	NumBytes = DPRCS1Offset;
1107
1108	// Move GPRCS3, if using using SplitR11WindowsSEH.
1109	if (GPRCS3Size > `0`) {
1110	assert(PushPopSplit == ARMSubtarget::SplitR11WindowsSEH);
1111	GPRCS3Push = LastPush = MBBI ++;
1112	DefCFAOffsetCandidates.addInst(I: LastPush, SPAdjust: GPRCS3Size, BeforeFPSet: BeforeFPPush);
1113	if (FramePtrSpillArea == SpillArea::GPRCS3)
1114	BeforeFPPush = false;
1115	NumBytes -= GPRCS3Size;
1116	}
1117
1118	bool NeedsWinCFIStackAlloc = NeedsWinCFI;
1119	if (PushPopSplit == ARMSubtarget::SplitR11WindowsSEH && HasFP)
1120	NeedsWinCFIStackAlloc = false;
1121
1122	if (STI.isTargetWindows() && WindowsRequiresStackProbe(MF, StackSizeInBytes: NumBytes)) {
1123	uint32_t NumWords = NumBytes >> `2`;
1124
1125	if (NumWords < `65536`) {
1126	BuildMI(BB&: MBB, I: MBBI, MIMD: dl, MCID: TII.get(Opcode: ARM::t2MOVi16), DestReg: ARM::R4)
1127	.addImm(Val: NumWords)
1128	.setMIFlags(MachineInstr::FrameSetup)
1129	.add(MOs: predOps(Pred: ARMCC::AL));
1130	} else {
1131	// Split into two instructions here, instead of using t2MOVi32imm,
1132	// to allow inserting accurate SEH instructions (including accurate
1133	// instruction size for each of them).
1134	BuildMI(BB&: MBB, I: MBBI, MIMD: dl, MCID: TII.get(Opcode: ARM::t2MOVi16), DestReg: ARM::R4)
1135	.addImm(Val: NumWords & `0xffff`)
1136	.setMIFlags(MachineInstr::FrameSetup)
1137	.add(MOs: predOps(Pred: ARMCC::AL));
1138	BuildMI(BB&: MBB, I: MBBI, MIMD: dl, MCID: TII.get(Opcode: ARM::t2MOVTi16), DestReg: ARM::R4)
1139	.addReg(RegNo: ARM::R4)
1140	.addImm(Val: NumWords >> `16`)
1141	.setMIFlags(MachineInstr::FrameSetup)
1142	.add(MOs: predOps(Pred: ARMCC::AL));
1143	}
1144
1145	const ARMTargetLowering *TLI = STI.getTargetLowering();
1146	RTLIB::LibcallImpl ChkStkLibcall = TLI->getLibcallImpl(Call: RTLIB::STACK_PROBE);
1147	if (ChkStkLibcall == RTLIB::Unsupported)
1148	reportFatalUsageError(reason: "no available implementation of __chkstk");
1149	const char *ChkStk = TLI->getLibcallImplName(Call: ChkStkLibcall).data();
1150
1151	switch (TM.getCodeModel()) {
1152	case CodeModel::Tiny:
1153	llvm_unreachable("Tiny code model not available on ARM.");
1154	case CodeModel::Small:
1155	case CodeModel::Medium:
1156	case CodeModel::Kernel:
1157	BuildMI(BB&: MBB, I: MBBI, MIMD: dl, MCID: TII.get(Opcode: ARM::tBL))
1158	.add(MOs: predOps(Pred: ARMCC::AL))
1159	.addExternalSymbol(FnName: ChkStk)
1160	.addReg(RegNo: ARM::R4, Flags: RegState::Implicit)
1161	.setMIFlags(MachineInstr::FrameSetup);
1162	break;
1163	case CodeModel::Large:
1164	BuildMI(BB&: MBB, I: MBBI, MIMD: dl, MCID: TII.get(Opcode: ARM::t2MOVi32imm), DestReg: ARM::R12)
1165	.addExternalSymbol(FnName: ChkStk)
1166	.setMIFlags(MachineInstr::FrameSetup);
1167
1168	BuildMI(BB&: MBB, I: MBBI, MIMD: dl, MCID: TII.get(Opcode: ARM::tBLXr))
1169	.add(MOs: predOps(Pred: ARMCC::AL))
1170	.addReg(RegNo: ARM::R12, Flags: RegState::Kill)
1171	.addReg(RegNo: ARM::R4, Flags: RegState::Implicit)
1172	.setMIFlags(MachineInstr::FrameSetup);
1173	break;
1174	}
1175
1176	MachineInstrBuilder Instr, SEH;
1177	Instr = BuildMI(BB&: MBB, I: MBBI, MIMD: dl, MCID: TII.get(Opcode: ARM::t2SUBrr), DestReg: ARM::SP)
1178	.addReg(RegNo: ARM::SP, Flags: RegState::Kill)
1179	.addReg(RegNo: ARM::R4, Flags: RegState::Kill)
1180	.setMIFlags(MachineInstr::FrameSetup)
1181	.add(MOs: predOps(Pred: ARMCC::AL))
1182	.add(MO: condCodeOp());
1183	if (NeedsWinCFIStackAlloc) {
1184	SEH = BuildMI(MF, MIMD: dl, MCID: TII.get(Opcode: ARM::SEH_StackAlloc))
1185	.addImm(Val: NumBytes)
1186	.addImm(/Wide=/Val: `1`)
1187	.setMIFlags(MachineInstr::FrameSetup);
1188	MBB.insertAfter(I: Instr, MI: SEH);
1189	}
1190	NumBytes = `0`;
1191	}
1192
1193	if (NumBytes) {
1194	// Adjust SP after all the callee-save spills.
1195	if (AFI->getNumAlignedDPRCS2Regs() == `0` &&
1196	tryFoldSPUpdateIntoPushPop(Subtarget: STI, MF, MI: &*LastPush, NumBytes))
1197	DefCFAOffsetCandidates.addExtraBytes(I: LastPush, ExtraBytes: NumBytes);
1198	else {
1199	emitSPUpdate(isARM, MBB, MBBI, dl, TII, NumBytes: -NumBytes,
1200	MIFlags: MachineInstr::FrameSetup);
1201	DefCFAOffsetCandidates.addInst(I: std::prev(x: MBBI), SPAdjust: NumBytes);
1202	}
1203
1204	if (HasFP && isARM)
1205	// Restore from fp only in ARM mode: e.g. sub sp, r7, #24
1206	// Note it's not safe to do this in Thumb2 mode because it would have
1207	// taken two instructions:
1208	// mov sp, r7
1209	// sub sp, #24
1210	// If an interrupt is taken between the two instructions, then sp is in
1211	// an inconsistent state (pointing to the middle of callee-saved area).
1212	// The interrupt handler can end up clobbering the registers.
1213	AFI->setShouldRestoreSPFromFP(true);
1214	}
1215
1216	// Set FP to point to the stack slot that contains the previous FP.
1217	// For iOS, FP is R7, which has now been stored in spill area 1.
1218	// Otherwise, if this is not iOS, all the callee-saved registers go
1219	// into spill area 1, including the FP in R11. In either case, it
1220	// is in area one and the adjustment needs to take place just after
1221	// that push.
1222	MachineBasicBlock::iterator AfterPush;
1223	if (HasFP) {
1224	MachineBasicBlock::iterator FPPushInst;
1225	// Offset from SP immediately after the push which saved the FP to the FP
1226	// save slot.
1227	int64_t FPOffsetAfterPush;
1228	switch (FramePtrSpillArea) {
1229	case SpillArea::GPRCS1:
1230	FPPushInst = GPRCS1Push;
1231	FPOffsetAfterPush = MFI.getObjectOffset(ObjectIdx: FramePtrSpillFI) +
1232	ArgRegsSaveSize + FPCXTSaveSize +
1233	sizeOfSPAdjustment(MI: *FPPushInst);
1234	LLVM_DEBUG(dbgs() << "Frame pointer in GPRCS1, offset "
1235	<< FPOffsetAfterPush << " after that push\n");
1236	break;
1237	case SpillArea::GPRCS2:
1238	FPPushInst = GPRCS2Push;
1239	FPOffsetAfterPush = MFI.getObjectOffset(ObjectIdx: FramePtrSpillFI) +
1240	ArgRegsSaveSize + FPCXTSaveSize + GPRCS1Size +
1241	sizeOfSPAdjustment(MI: *FPPushInst);
1242	LLVM_DEBUG(dbgs() << "Frame pointer in GPRCS2, offset "
1243	<< FPOffsetAfterPush << " after that push\n");
1244	break;
1245	case SpillArea::GPRCS3:
1246	FPPushInst = GPRCS3Push;
1247	FPOffsetAfterPush = MFI.getObjectOffset(ObjectIdx: FramePtrSpillFI) +
1248	ArgRegsSaveSize + FPCXTSaveSize + GPRCS1Size +
1249	FPStatusSize + GPRCS2Size + DPRCS1Size + DPRGapSize +
1250	sizeOfSPAdjustment(MI: *FPPushInst);
1251	LLVM_DEBUG(dbgs() << "Frame pointer in GPRCS3, offset "
1252	<< FPOffsetAfterPush << " after that push\n");
1253	break;
1254	default:
1255	llvm_unreachable("frame pointer in unknown spill area");
1256	break;
1257	}
1258	AfterPush = std::next(x: FPPushInst);
1259	if (PushPopSplit == ARMSubtarget::SplitR11WindowsSEH)
1260	assert(FPOffsetAfterPush == `0`);
1261
1262	// Emit the MOV or ADD to set up the frame pointer register.
1263	emitRegPlusImmediate(isARM: !AFI->isThumbFunction(), MBB, MBBI&: AfterPush, dl, TII,
1264	DestReg: FramePtr, SrcReg: ARM::SP, NumBytes: FPOffsetAfterPush,
1265	MIFlags: MachineInstr::FrameSetup);
1266
1267	if (!NeedsWinCFI) {
1268	// Emit DWARF info to find the CFA using the frame pointer from this
1269	// point onward.
1270	CFIInstBuilder CFIBuilder(MBB, AfterPush, MachineInstr::FrameSetup);
1271	if (FPOffsetAfterPush != `0`)
1272	CFIBuilder.buildDefCFA(Reg: FramePtr, Offset: -MFI.getObjectOffset(ObjectIdx: FramePtrSpillFI));
1273	else
1274	CFIBuilder.buildDefCFARegister(Reg: FramePtr);
1275	}
1276	}
1277
1278	// Emit a SEH opcode indicating the prologue end. The rest of the prologue
1279	// instructions below don't need to be replayed to unwind the stack.
1280	if (NeedsWinCFI && MBBI != MBB.begin()) {
1281	MachineBasicBlock::iterator End = MBBI;
1282	if (HasFP && PushPopSplit == ARMSubtarget::SplitR11WindowsSEH)
1283	End = AfterPush;
1284	insertSEHRange(MBB, Start: {}, End, TII, MIFlags: MachineInstr::FrameSetup);
1285	BuildMI(BB&: MBB, I: End, MIMD: dl, MCID: TII.get(Opcode: ARM::SEH_PrologEnd))
1286	.setMIFlag(MachineInstr::FrameSetup);
1287	MF.setHasWinCFI(true);
1288	}
1289
1290	// Now that the prologue's actual instructions are finalised, we can insert
1291	// the necessary DWARF cf instructions to describe the situation. Start by
1292	// recording where each register ended up:
1293	if (!NeedsWinCFI) {
1294	for (const auto &Entry : reverse(C: CSI)) {
1295	MCRegister Reg = Entry.getReg();
1296	int FI = Entry.getFrameIdx();
1297	MachineBasicBlock::iterator CFIPos;
1298	switch (getSpillArea(Reg, Variation: PushPopSplit, NumAlignedDPRCS2Regs: AFI->getNumAlignedDPRCS2Regs(),
1299	RegInfo)) {
1300	case SpillArea::GPRCS1:
1301	CFIPos = std::next(x: GPRCS1Push);
1302	break;
1303	case SpillArea::GPRCS2:
1304	CFIPos = std::next(x: GPRCS2Push);
1305	break;
1306	case SpillArea::DPRCS1:
1307	CFIPos = std::next(x: DPRCS1Push);
1308	break;
1309	case SpillArea::GPRCS3:
1310	CFIPos = std::next(x: GPRCS3Push);
1311	break;
1312	case SpillArea::FPStatus:
1313	case SpillArea::FPCXT:
1314	case SpillArea::DPRCS2:
1315	// FPCXT and DPRCS2 are not represented in the DWARF info.
1316	break;
1317	}
1318
1319	if (CFIPos.isValid()) {
1320	CFIInstBuilder (MBB, CFIPos, MachineInstr::FrameSetup)
1321	.buildOffset(Reg: Reg == ARM::R12 ? ARM::RA_AUTH_CODE : Reg,
1322	Offset: MFI.getObjectOffset(ObjectIdx: FI));
1323	}
1324	}
1325	}
1326
1327	// Now we can emit descriptions of where the canonical frame address was
1328	// throughout the process. If we have a frame pointer, it takes over the job
1329	// half-way through, so only the first few .cfi_def_cfa_offset instructions
1330	// actually get emitted.
1331	if (!NeedsWinCFI) {
1332	LLVM_DEBUG(DefCFAOffsetCandidates.dump());
1333	DefCFAOffsetCandidates.emitDefCFAOffsets(MBB, HasFP);
1334	}
1335
1336	if (STI.isTargetELF() && hasFP(MF))
1337	MFI.setOffsetAdjustment(MFI.getOffsetAdjustment() -
1338	AFI->getFramePtrSpillOffset());
1339
1340	AFI->setFPCXTSaveAreaSize(FPCXTSaveSize);
1341	AFI->setGPRCalleeSavedArea1Size(GPRCS1Size);
1342	AFI->setGPRCalleeSavedArea2Size(GPRCS2Size);
1343	AFI->setFPStatusSavesSize(FPStatusSize);
1344	AFI->setDPRCalleeSavedGapSize(DPRGapSize);
1345	AFI->setDPRCalleeSavedArea1Size(DPRCS1Size);
1346	AFI->setGPRCalleeSavedArea3Size(GPRCS3Size);
1347
1348	// If we need dynamic stack realignment, do it here. Be paranoid and make
1349	// sure if we also have VLAs, we have a base pointer for frame access.
1350	// If aligned NEON registers were spilled, the stack has already been
1351	// realigned.
1352	if (!AFI->getNumAlignedDPRCS2Regs() && RegInfo->hasStackRealignment(MF)) {
1353	Align MaxAlign = MFI.getMaxAlign();
1354	assert(!AFI->isThumb1OnlyFunction());
1355	if (!AFI->isThumbFunction()) {
1356	emitAligningInstructions(MF, AFI, TII, MBB, MBBI, DL: dl, Reg: ARM::SP, Alignment: MaxAlign,
1357	MustBeSingleInstruction: false);
1358	} else {
1359	// We cannot use sp as source/dest register here, thus we're using r4 to
1360	// perform the calculations. We're emitting the following sequence:
1361	// mov r4, sp
1362	// -- use emitAligningInstructions to produce best sequence to zero
1363	// -- out lower bits in r4
1364	// mov sp, r4
1365	// FIXME: It will be better just to find spare register here.
1366	BuildMI(BB&: MBB, I: MBBI, MIMD: dl, MCID: TII.get(Opcode: ARM::tMOVr), DestReg: ARM::R4)
1367	.addReg(RegNo: ARM::SP, Flags: RegState::Kill)
1368	.add(MOs: predOps(Pred: ARMCC::AL));
1369	emitAligningInstructions(MF, AFI, TII, MBB, MBBI, DL: dl, Reg: ARM::R4, Alignment: MaxAlign,
1370	MustBeSingleInstruction: false);
1371	BuildMI(BB&: MBB, I: MBBI, MIMD: dl, MCID: TII.get(Opcode: ARM::tMOVr), DestReg: ARM::SP)
1372	.addReg(RegNo: ARM::R4, Flags: RegState::Kill)
1373	.add(MOs: predOps(Pred: ARMCC::AL));
1374	}
1375
1376	AFI->setShouldRestoreSPFromFP(true);
1377	}
1378
1379	// If we need a base pointer, set it up here. It's whatever the value
1380	// of the stack pointer is at this point. Any variable size objects
1381	// will be allocated after this, so we can still use the base pointer
1382	// to reference locals.
1383	// FIXME: Clarify FrameSetup flags here.
1384	if (RegInfo->hasBasePointer(MF) && !MBB.isEHFuncletEntry()) {
1385	if (isARM)
1386	BuildMI(BB&: MBB, I: MBBI, MIMD: dl, MCID: TII.get(Opcode: ARM::MOVr), DestReg: RegInfo->getBaseRegister())
1387	.addReg(RegNo: ARM::SP)
1388	.add(MOs: predOps(Pred: ARMCC::AL))
1389	.add(MO: condCodeOp());
1390	else
1391	BuildMI(BB&: MBB, I: MBBI, MIMD: dl, MCID: TII.get(Opcode: ARM::tMOVr), DestReg: RegInfo->getBaseRegister())
1392	.addReg(RegNo: ARM::SP)
1393	.add(MOs: predOps(Pred: ARMCC::AL));
1394	}
1395
1396	// If the frame has variable sized objects then the epilogue must restore
1397	// the sp from fp. We can assume there's an FP here since hasFP already
1398	// checks for hasVarSizedObjects.
1399	if (MFI.hasVarSizedObjects())
1400	AFI->setShouldRestoreSPFromFP(true);
1401	}
1402
1403	void ARMFrameLowering::emitEpilogue(MachineFunction &MF,
1404	MachineBasicBlock &MBB) const {
1405	MachineFrameInfo &MFI = MF.getFrameInfo();
1406	ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
1407	const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
1408	const ARMBaseInstrInfo &TII =
1409	*static_cast<const ARMBaseInstrInfo *>(MF.getSubtarget().getInstrInfo());
1410	assert(!AFI->isThumb1OnlyFunction() &&
1411	"This emitEpilogue does not support Thumb1!");
1412	bool isARM = !AFI->isThumbFunction();
1413	ARMSubtarget::PushPopSplitVariation PushPopSplit =
1414	STI.getPushPopSplitVariation(MF);
1415
1416	LLVM_DEBUG(dbgs() << "Emitting epilogue for " << MF.getName() << "\n");
1417
1418	// Amount of stack space we reserved next to incoming args for either
1419	// varargs registers or stack arguments in tail calls made by this function.
1420	unsigned ReservedArgStack = AFI->getArgRegsSaveSize();
1421
1422	// How much of the stack used by incoming arguments this function is expected
1423	// to restore in this particular epilogue.
1424	int IncomingArgStackToRestore = getArgumentStackToRestore(MF, MBB);
1425	int NumBytes = (int)MFI.getStackSize();
1426	Register FramePtr = RegInfo->getFrameRegister(MF);
1427
1428	// All calls are tail calls in GHC calling conv, and functions have no
1429	// prologue/epilogue.
1430	if (MF.getFunction().getCallingConv() == CallingConv::GHC)
1431	return;
1432
1433	// First put ourselves on the first (from top) terminator instructions.
1434	MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
1435	DebugLoc dl = MBBI != MBB.end() ? MBBI ->getDebugLoc() : DebugLoc ();
1436
1437	MachineBasicBlock::iterator RangeStart;
1438	if (!AFI->hasStackFrame()) {
1439	if (MF.hasWinCFI()) {
1440	BuildMI(BB&: MBB, I: MBBI, MIMD: dl, MCID: TII.get(Opcode: ARM::SEH_EpilogStart))
1441	.setMIFlag(MachineInstr::FrameDestroy);
1442	RangeStart = initMBBRange(MBB, MBBI);
1443	}
1444
1445	if (NumBytes + IncomingArgStackToRestore != `0`)
1446	emitSPUpdate(isARM, MBB, MBBI, dl, TII,
1447	NumBytes: NumBytes + IncomingArgStackToRestore,
1448	MIFlags: MachineInstr::FrameDestroy);
1449	} else {
1450	// Unwind MBBI to point to first LDR / VLDRD.
1451	if (MBBI != MBB.begin()) {
1452	do {
1453	--MBBI;
1454	} while (MBBI != MBB.begin() &&
1455	MBBI ->getFlag(Flag: MachineInstr::FrameDestroy));
1456	if (!MBBI ->getFlag(Flag: MachineInstr::FrameDestroy))
1457	++MBBI;
1458	}
1459
1460	if (MF.hasWinCFI()) {
1461	BuildMI(BB&: MBB, I: MBBI, MIMD: dl, MCID: TII.get(Opcode: ARM::SEH_EpilogStart))
1462	.setMIFlag(MachineInstr::FrameDestroy);
1463	RangeStart = initMBBRange(MBB, MBBI);
1464	}
1465
1466	// Move SP to start of FP callee save spill area.
1467	NumBytes -=
1468	(ReservedArgStack + AFI->getFPCXTSaveAreaSize() +
1469	AFI->getGPRCalleeSavedArea1Size() + AFI->getGPRCalleeSavedArea2Size() +
1470	AFI->getFPStatusSavesSize() + AFI->getDPRCalleeSavedGapSize() +
1471	AFI->getDPRCalleeSavedArea1Size() + AFI->getGPRCalleeSavedArea3Size());
1472
1473	// Reset SP based on frame pointer only if the stack frame extends beyond
1474	// frame pointer stack slot or target is ELF and the function has FP.
1475	if (AFI->shouldRestoreSPFromFP()) {
1476	NumBytes = AFI->getFramePtrSpillOffset() - NumBytes;
1477	if (NumBytes) {
1478	if (isARM)
1479	emitARMRegPlusImmediate(MBB, MBBI, dl, DestReg: ARM::SP, BaseReg: FramePtr, NumBytes: -NumBytes,
1480	Pred: ARMCC::AL, PredReg: `0`, TII,
1481	MIFlags: MachineInstr::FrameDestroy);
1482	else {
1483	// It's not possible to restore SP from FP in a single instruction.
1484	// For iOS, this looks like:
1485	// mov sp, r7
1486	// sub sp, #24
1487	// This is bad, if an interrupt is taken after the mov, sp is in an
1488	// inconsistent state.
1489	// Use the first callee-saved register as a scratch register.
1490	assert(!MFI.getPristineRegs(MF).test(ARM::R4) &&
1491	"No scratch register to restore SP from FP!");
1492	emitT2RegPlusImmediate(MBB, MBBI, dl, DestReg: ARM::R4, BaseReg: FramePtr, NumBytes: -NumBytes,
1493	Pred: ARMCC::AL, PredReg: `0`, TII, MIFlags: MachineInstr::FrameDestroy);
1494	BuildMI(BB&: MBB, I: MBBI, MIMD: dl, MCID: TII.get(Opcode: ARM::tMOVr), DestReg: ARM::SP)
1495	.addReg(RegNo: ARM::R4)
1496	.add(MOs: predOps(Pred: ARMCC::AL))
1497	.setMIFlag(MachineInstr::FrameDestroy);
1498	}
1499	} else {
1500	// Thumb2 or ARM.
1501	if (isARM)
1502	BuildMI(BB&: MBB, I: MBBI, MIMD: dl, MCID: TII.get(Opcode: ARM::MOVr), DestReg: ARM::SP)
1503	.addReg(RegNo: FramePtr)
1504	.add(MOs: predOps(Pred: ARMCC::AL))
1505	.add(MO: condCodeOp())
1506	.setMIFlag(MachineInstr::FrameDestroy);
1507	else
1508	BuildMI(BB&: MBB, I: MBBI, MIMD: dl, MCID: TII.get(Opcode: ARM::tMOVr), DestReg: ARM::SP)
1509	.addReg(RegNo: FramePtr)
1510	.add(MOs: predOps(Pred: ARMCC::AL))
1511	.setMIFlag(MachineInstr::FrameDestroy);
1512	}
1513	} else if (NumBytes &&
1514	!tryFoldSPUpdateIntoPushPop(Subtarget: STI, MF, MI: &*MBBI, NumBytes))
1515	emitSPUpdate(isARM, MBB, MBBI, dl, TII, NumBytes,
1516	MIFlags: MachineInstr::FrameDestroy);
1517
1518	// Increment past our save areas.
1519	if (AFI->getGPRCalleeSavedArea3Size()) {
1520	assert(PushPopSplit == ARMSubtarget::SplitR11WindowsSEH);
1521	(void)PushPopSplit;
1522	MBBI ++;
1523	}
1524
1525	if (MBBI != MBB.end() && AFI->getDPRCalleeSavedArea1Size()) {
1526	MBBI ++;
1527	// Since vpop register list cannot have gaps, there may be multiple vpop
1528	// instructions in the epilogue.
1529	while (MBBI != MBB.end() && MBBI ->getOpcode() == ARM::VLDMDIA_UPD)
1530	MBBI ++;
1531	}
1532	if (AFI->getDPRCalleeSavedGapSize()) {
1533	assert(AFI->getDPRCalleeSavedGapSize() == `4` &&
1534	"unexpected DPR alignment gap");
1535	emitSPUpdate(isARM, MBB, MBBI, dl, TII, NumBytes: AFI->getDPRCalleeSavedGapSize(),
1536	MIFlags: MachineInstr::FrameDestroy);
1537	}
1538
1539	if (AFI->getGPRCalleeSavedArea2Size()) {
1540	assert(PushPopSplit != ARMSubtarget::SplitR11WindowsSEH);
1541	(void)PushPopSplit;
1542	MBBI ++;
1543	}
1544	if (AFI->getGPRCalleeSavedArea1Size()) MBBI ++;
1545
1546	if (ReservedArgStack \|\| IncomingArgStackToRestore) {
1547	assert((int)ReservedArgStack + IncomingArgStackToRestore >= `0` &&
1548	"attempting to restore negative stack amount");
1549	emitSPUpdate(isARM, MBB, MBBI, dl, TII,
1550	NumBytes: ReservedArgStack + IncomingArgStackToRestore,
1551	MIFlags: MachineInstr::FrameDestroy);
1552	}
1553
1554	// Validate PAC, It should have been already popped into R12. For CMSE entry
1555	// function, the validation instruction is emitted during expansion of the
1556	// tBXNS_RET, since the validation must use the value of SP at function
1557	// entry, before saving, resp. after restoring, FPCXTNS.
1558	if (AFI->shouldSignReturnAddress() && !AFI->isCmseNSEntryFunction()) {
1559	bool CanUseBXAut =
1560	STI.isThumb() && STI.hasV8_1MMainlineOps() && STI.hasPACBTI();
1561	auto TMBBI = MBB.getFirstTerminator();
1562	bool IsBXReturn =
1563	TMBBI != MBB.end() && TMBBI ->getOpcode() == ARM::tBX_RET;
1564	if (IsBXReturn && CanUseBXAut)
1565	TMBBI ->setDesc(STI.getInstrInfo()->get(Opcode: ARM::t2BXAUT_RET));
1566	else
1567	BuildMI(BB&: MBB, I: MBBI, MIMD: DebugLoc (), MCID: STI.getInstrInfo()->get(Opcode: ARM::t2AUT));
1568	}
1569	}
1570
1571	if (MF.hasWinCFI()) {
1572	insertSEHRange(MBB, Start: RangeStart, End: MBB.end(), TII, MIFlags: MachineInstr::FrameDestroy);
1573	BuildMI(BB&: MBB, I: MBB.end(), MIMD: dl, MCID: TII.get(Opcode: ARM::SEH_EpilogEnd))
1574	.setMIFlag(MachineInstr::FrameDestroy);
1575	}
1576	}
1577
1578	/// getFrameIndexReference - Provide a base+offset reference to an FI slot for
1579	/// debug info. It's the same as what we use for resolving the code-gen
1580	/// references for now. FIXME: This can go wrong when references are
1581	/// SP-relative and simple call frames aren't used.
1582	StackOffset ARMFrameLowering::getFrameIndexReference(const MachineFunction &MF,
1583	int FI,
1584	Register &FrameReg) const {
1585	return StackOffset::getFixed(Fixed: ResolveFrameIndexReference(MF, FI, FrameReg, SPAdj: `0`));
1586	}
1587
1588	StackOffset
1589	ARMFrameLowering::getNonLocalFrameIndexReference(const MachineFunction &MF,
1590	int FI) const {
1591	const MachineFrameInfo &MFI = MF.getFrameInfo();
1592	int Offset = MFI.getObjectOffset(ObjectIdx: FI) + MFI.getStackSize();
1593	return StackOffset::getFixed(Fixed: Offset);
1594	}
1595
1596	int ARMFrameLowering::ResolveFrameIndexReference(const MachineFunction &MF,
1597	int FI, Register &FrameReg,
1598	int SPAdj) const {
1599	const MachineFrameInfo &MFI = MF.getFrameInfo();
1600	const ARMBaseRegisterInfo RegInfo = static_cast<const* ARMBaseRegisterInfo *>(
1601	MF.getSubtarget().getRegisterInfo());
1602	const ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
1603	int Offset = MFI.getObjectOffset(ObjectIdx: FI) + MFI.getStackSize();
1604	int FPOffset = Offset - AFI->getFramePtrSpillOffset();
1605	bool isFixed = MFI.isFixedObjectIndex(ObjectIdx: FI);
1606
1607	FrameReg = ARM::SP;
1608	Offset += SPAdj;
1609
1610	// SP can move around if there are allocas. We may also lose track of SP
1611	// when emergency spilling inside a non-reserved call frame setup.
1612	bool hasMovingSP = !hasReservedCallFrame(MF);
1613
1614	// When dynamically realigning the stack, use the frame pointer for
1615	// parameters, and the stack/base pointer for locals.
1616	if (RegInfo->hasStackRealignment(MF)) {
1617	assert(hasFP(MF) && "dynamic stack realignment without a FP!");
1618	if (isFixed) {
1619	FrameReg = RegInfo->getFrameRegister(MF);
1620	Offset = FPOffset;
1621	} else if (hasMovingSP) {
1622	assert(RegInfo->hasBasePointer(MF) &&
1623	"VLAs and dynamic stack alignment, but missing base pointer!");
1624	FrameReg = RegInfo->getBaseRegister();
1625	Offset -= SPAdj;
1626	}
1627	return Offset;
1628	}
1629
1630	// If there is a frame pointer, use it when we can.
1631	if (hasFP(MF) && AFI->hasStackFrame()) {
1632	// Use frame pointer to reference fixed objects. Use it for locals if
1633	// there are VLAs (and thus the SP isn't reliable as a base).
1634	if (isFixed \|\| (hasMovingSP && !RegInfo->hasBasePointer(MF))) {
1635	FrameReg = RegInfo->getFrameRegister(MF);
1636	return FPOffset;
1637	} else if (hasMovingSP) {
1638	assert(RegInfo->hasBasePointer(MF) && "missing base pointer!");
1639	if (AFI->isThumb2Function()) {
1640	// Try to use the frame pointer if we can, else use the base pointer
1641	// since it's available. This is handy for the emergency spill slot, in
1642	// particular.
1643	if (FPOffset >= -`255` && FPOffset < `0`) {
1644	FrameReg = RegInfo->getFrameRegister(MF);
1645	return FPOffset;
1646	}
1647	}
1648	} else if (AFI->isThumbFunction()) {
1649	// Prefer SP to base pointer, if the offset is suitably aligned and in
1650	// range as the effective range of the immediate offset is bigger when
1651	// basing off SP.
1652	// Use add <rd>, sp, #<imm8>
1653	// ldr <rd>, [sp, #<imm8>]
1654	if (Offset >= `0` && (Offset & `3`) == `0` && Offset <= `1020`)
1655	return Offset;
1656	// In Thumb2 mode, the negative offset is very limited. Try to avoid
1657	// out of range references. ldr <rt>,[<rn>, #-<imm8>]
1658	if (AFI->isThumb2Function() && FPOffset >= -`255` && FPOffset < `0`) {
1659	FrameReg = RegInfo->getFrameRegister(MF);
1660	return FPOffset;
1661	}
1662	} else if (Offset > (FPOffset < `0` ? -FPOffset : FPOffset)) {
1663	// Otherwise, use SP or FP, whichever is closer to the stack slot.
1664	FrameReg = RegInfo->getFrameRegister(MF);
1665	return FPOffset;
1666	}
1667	}
1668	// Use the base pointer if we have one.
1669	// FIXME: Maybe prefer sp on Thumb1 if it's legal and the offset is cheaper?
1670	// That can happen if we forced a base pointer for a large call frame.
1671	if (RegInfo->hasBasePointer(MF)) {
1672	FrameReg = RegInfo->getBaseRegister();
1673	Offset -= SPAdj;
1674	}
1675	return Offset;
1676	}
1677
1678	void ARMFrameLowering::emitPushInst(MachineBasicBlock &MBB,
1679	MachineBasicBlock::iterator MI,
1680	ArrayRef<CalleeSavedInfo> CSI,
1681	unsigned StmOpc, unsigned StrOpc,
1682	bool NoGap,
1683	function_ref<bool(unsigned)> Func) const {
1684	MachineFunction &MF = *MBB.getParent();
1685	const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1686	const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
1687
1688	DebugLoc DL;
1689
1690	using RegAndKill = std::pair<unsigned, bool>;
1691
1692	SmallVector<RegAndKill, `4`> Regs;
1693	unsigned i = CSI.size();
1694	while (i != `0`) {
1695	unsigned LastReg = `0`;
1696	for (; i != `0`; --i) {
1697	MCRegister Reg = CSI [i-`1`].getReg();
1698	if (!Func (Reg))
1699	continue;
1700
1701	const MachineRegisterInfo &MRI = MF.getRegInfo();
1702	bool isLiveIn = MRI.isLiveIn(Reg);
1703	if (!isLiveIn && !MRI.isReserved(PhysReg: Reg))
1704	MBB.addLiveIn(PhysReg: Reg);
1705	// If NoGap is true, push consecutive registers and then leave the rest
1706	// for other instructions. e.g.
1707	// vpush {d8, d10, d11} -> vpush {d8}, vpush {d10, d11}
1708	if (NoGap && LastReg && LastReg != Reg-`1`)
1709	break;
1710	LastReg = Reg;
1711	// Do not set a kill flag on values that are also marked as live-in. This
1712	// happens with the @llvm-returnaddress intrinsic and with arguments
1713	// passed in callee saved registers.
1714	// Omitting the kill flags is conservatively correct even if the live-in
1715	// is not used after all.
1716	Regs.push_back(Elt: std::make_pair(x&: Reg, /isKill=/y: !isLiveIn));
1717	}
1718
1719	if (Regs.empty())
1720	continue;
1721
1722	llvm::sort(C&: Regs, Comp: [&](const RegAndKill &LHS, const RegAndKill &RHS) {
1723	return TRI.getEncodingValue(Reg: LHS.first) < TRI.getEncodingValue(Reg: RHS.first);
1724	});
1725
1726	if (Regs.size() > `1` \|\| StrOpc== `0`) {
1727	MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: StmOpc), DestReg: ARM::SP)
1728	.addReg(RegNo: ARM::SP)
1729	.setMIFlags(MachineInstr::FrameSetup)
1730	.add(MOs: predOps(Pred: ARMCC::AL));
1731	for (const auto &[Reg, Kill] : Regs)
1732	MIB.addReg(RegNo: Reg, Flags: getKillRegState(B: Kill));
1733	} else if (Regs.size() == `1`) {
1734	BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: StrOpc), DestReg: ARM::SP)
1735	.addReg(RegNo: Regs [`0`].first, Flags: getKillRegState(B: Regs [`0`].second))
1736	.addReg(RegNo: ARM::SP)
1737	.setMIFlags(MachineInstr::FrameSetup)
1738	.addImm(Val: -`4`)
1739	.add(MOs: predOps(Pred: ARMCC::AL));
1740	}
1741	Regs.clear();
1742
1743	// Put any subsequent vpush instructions before this one: they will refer to
1744	// higher register numbers so need to be pushed first in order to preserve
1745	// monotonicity.
1746	if (MI != MBB.begin())
1747	--MI;
1748	}
1749	}
1750
1751	void ARMFrameLowering::emitPopInst(MachineBasicBlock &MBB,
1752	MachineBasicBlock::iterator MI,
1753	MutableArrayRef<CalleeSavedInfo> CSI,
1754	unsigned LdmOpc, unsigned LdrOpc,
1755	bool isVarArg, bool NoGap,
1756	function_ref<bool(unsigned)> Func) const {
1757	MachineFunction &MF = *MBB.getParent();
1758	const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1759	const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
1760	ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
1761	bool hasPAC = AFI->shouldSignReturnAddress();
1762	DebugLoc DL;
1763	bool isTailCall = false;
1764	bool isInterrupt = false;
1765	bool isTrap = false;
1766	bool isCmseEntry = false;
1767	ARMSubtarget::PushPopSplitVariation PushPopSplit =
1768	STI.getPushPopSplitVariation(MF);
1769	if (MBB.end() != MI) {
1770	DL = MI ->getDebugLoc();
1771	unsigned RetOpcode = MI ->getOpcode();
1772	isTailCall =
1773	(RetOpcode == ARM::TCRETURNdi \|\| RetOpcode == ARM::TCRETURNri \|\|
1774	RetOpcode == ARM::TCRETURNrinotr12);
1775	isInterrupt =
1776	RetOpcode == ARM::SUBS_PC_LR \|\| RetOpcode == ARM::t2SUBS_PC_LR;
1777	isTrap = RetOpcode == ARM::TRAP \|\| RetOpcode == ARM::tTRAP;
1778	isCmseEntry = (RetOpcode == ARM::tBXNS \|\| RetOpcode == ARM::tBXNS_RET);
1779	}
1780
1781	SmallVector<unsigned, `4`> Regs;
1782	unsigned i = CSI.size();
1783	while (i != `0`) {
1784	unsigned LastReg = `0`;
1785	bool DeleteRet = false;
1786	for (; i != `0`; --i) {
1787	CalleeSavedInfo &Info = CSI [i-`1`];
1788	MCRegister Reg = Info.getReg();
1789	if (!Func (Reg))
1790	continue;
1791
1792	if (Reg == ARM::LR && !isTailCall && !isVarArg && !isInterrupt &&
1793	!isCmseEntry && !isTrap && AFI->getArgumentStackToRestore() == `0` &&
1794	STI.hasV5TOps() && MBB.succ_empty() && !hasPAC &&
1795	(PushPopSplit != ARMSubtarget::SplitR11WindowsSEH &&
1796	PushPopSplit != ARMSubtarget::SplitR11AAPCSSignRA)) {
1797	Reg = ARM::PC;
1798	// Fold the return instruction into the LDM.
1799	DeleteRet = true;
1800	LdmOpc = AFI->isThumbFunction() ? ARM::t2LDMIA_RET : ARM::LDMIA_RET;
1801	}
1802
1803	// If NoGap is true, pop consecutive registers and then leave the rest
1804	// for other instructions. e.g.
1805	// vpop {d8, d10, d11} -> vpop {d8}, vpop {d10, d11}
1806	if (NoGap && LastReg && LastReg != Reg-`1`)
1807	break;
1808
1809	LastReg = Reg;
1810	Regs.push_back(Elt: Reg);
1811	}
1812
1813	if (Regs.empty())
1814	continue;
1815
1816	llvm::sort(C&: Regs, Comp: [&](unsigned LHS, unsigned RHS) {
1817	return TRI.getEncodingValue(Reg: LHS) < TRI.getEncodingValue(Reg: RHS);
1818	});
1819
1820	if (Regs.size() > `1` \|\| LdrOpc == `0`) {
1821	MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: LdmOpc), DestReg: ARM::SP)
1822	.addReg(RegNo: ARM::SP)
1823	.add(MOs: predOps(Pred: ARMCC::AL))
1824	.setMIFlags(MachineInstr::FrameDestroy);
1825	for (unsigned Reg : Regs)
1826	MIB.addReg(RegNo: Reg, Flags: getDefRegState(B: true));
1827	if (DeleteRet) {
1828	if (MI != MBB.end()) {
1829	MIB.copyImplicitOps(OtherMI: *MI);
1830	MI ->eraseFromParent();
1831	}
1832	}
1833	MI = MIB;
1834	} else if (Regs.size() == `1`) {
1835	// If we adjusted the reg to PC from LR above, switch it back here. We
1836	// only do that for LDM.
1837	if (Regs [`0`] == ARM::PC)
1838	Regs [`0`] = ARM::LR;
1839	MachineInstrBuilder MIB =
1840	BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: LdrOpc), DestReg: Regs [`0`])
1841	.addReg(RegNo: ARM::SP, Flags: RegState::Define)
1842	.addReg(RegNo: ARM::SP)
1843	.setMIFlags(MachineInstr::FrameDestroy);
1844	// ARM mode needs an extra reg0 here due to addrmode2. Will go away once
1845	// that refactoring is complete (eventually).
1846	if (LdrOpc == ARM::LDR_POST_REG \|\| LdrOpc == ARM::LDR_POST_IMM) {
1847	MIB.addReg(RegNo: `0`);
1848	MIB.addImm(Val: ARM_AM::getAM2Opc(Opc: ARM_AM::add, Imm12: `4`, SO: ARM_AM::no_shift));
1849	} else
1850	MIB.addImm(Val: `4`);
1851	MIB.add(MOs: predOps(Pred: ARMCC::AL));
1852	}
1853	Regs.clear();
1854
1855	// Put any subsequent vpop instructions after this one: they will refer to
1856	// higher register numbers so need to be popped afterwards.
1857	if (MI != MBB.end())
1858	++MI;
1859	}
1860	}
1861
1862	void ARMFrameLowering::emitFPStatusSaves(MachineBasicBlock &MBB,
1863	MachineBasicBlock::iterator MI,
1864	ArrayRef<CalleeSavedInfo> CSI,
1865	unsigned PushOpc) const {
1866	MachineFunction &MF = *MBB.getParent();
1867	const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1868
1869	SmallVector<MCRegister> Regs;
1870	auto RegPresent = [&CSI](MCRegister Reg) {
1871	return llvm::any_of(Range&: CSI, P: [Reg](const CalleeSavedInfo &C) {
1872	return C.getReg() == Reg;
1873	});
1874	};
1875
1876	// If we need to save FPSCR, then we must move FPSCR into R4 with the VMRS
1877	// instruction.
1878	if (RegPresent (ARM::FPSCR)) {
1879	BuildMI(BB&: MBB, I: MI, MIMD: DebugLoc (), MCID: TII.get(Opcode: ARM::VMRS), DestReg: ARM::R4)
1880	.add(MOs: predOps(Pred: ARMCC::AL))
1881	.setMIFlags(MachineInstr::FrameSetup);
1882
1883	Regs.push_back(Elt: ARM::R4);
1884	}
1885
1886	// If we need to save FPEXC, then we must move FPEXC into R5 with the
1887	// VMRS_FPEXC instruction.
1888	if (RegPresent (ARM::FPEXC)) {
1889	BuildMI(BB&: MBB, I: MI, MIMD: DebugLoc (), MCID: TII.get(Opcode: ARM::VMRS_FPEXC), DestReg: ARM::R5)
1890	.add(MOs: predOps(Pred: ARMCC::AL))
1891	.setMIFlags(MachineInstr::FrameSetup);
1892
1893	Regs.push_back(Elt: ARM::R5);
1894	}
1895
1896	// If neither FPSCR and FPEXC are present, then do nothing.
1897	if (Regs.size() == `0`)
1898	return;
1899
1900	// Push both R4 and R5 onto the stack, if present.
1901	MachineInstrBuilder MIB =
1902	BuildMI(BB&: MBB, I: MI, MIMD: DebugLoc (), MCID: TII.get(Opcode: PushOpc), DestReg: ARM::SP)
1903	.addReg(RegNo: ARM::SP)
1904	.add(MOs: predOps(Pred: ARMCC::AL))
1905	.setMIFlags(MachineInstr::FrameSetup);
1906
1907	for (Register Reg : Regs) {
1908	MIB.addReg(RegNo: Reg);
1909	}
1910	}
1911
1912	void ARMFrameLowering::emitFPStatusRestores(
1913	MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
1914	MutableArrayRef<CalleeSavedInfo> CSI, unsigned LdmOpc) const {
1915	MachineFunction &MF = *MBB.getParent();
1916	const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1917
1918	auto RegPresent = [&CSI](MCRegister Reg) {
1919	return llvm::any_of(Range&: CSI, P: [Reg](const CalleeSavedInfo &C) {
1920	return C.getReg() == Reg;
1921	});
1922	};
1923
1924	// Do nothing if we don't need to restore any FP status registers.
1925	if (!RegPresent (ARM::FPSCR) && !RegPresent (ARM::FPEXC))
1926	return;
1927
1928	// Pop registers off of the stack.
1929	MachineInstrBuilder MIB =
1930	BuildMI(BB&: MBB, I: MI, MIMD: DebugLoc (), MCID: TII.get(Opcode: LdmOpc), DestReg: ARM::SP)
1931	.addReg(RegNo: ARM::SP)
1932	.add(MOs: predOps(Pred: ARMCC::AL))
1933	.setMIFlags(MachineInstr::FrameDestroy);
1934
1935	// If FPSCR was saved, it will be popped into R4.
1936	if (RegPresent (ARM::FPSCR)) {
1937	MIB.addReg(RegNo: ARM::R4, Flags: RegState::Define);
1938	}
1939
1940	// If FPEXC was saved, it will be popped into R5.
1941	if (RegPresent (ARM::FPEXC)) {
1942	MIB.addReg(RegNo: ARM::R5, Flags: RegState::Define);
1943	}
1944
1945	// Move the FPSCR value back into the register with the VMSR instruction.
1946	if (RegPresent (ARM::FPSCR)) {
1947	BuildMI(BB&: MBB, I: MI, MIMD: DebugLoc (), MCID: STI.getInstrInfo()->get(Opcode: ARM::VMSR))
1948	.addReg(RegNo: ARM::R4)
1949	.add(MOs: predOps(Pred: ARMCC::AL))
1950	.setMIFlags(MachineInstr::FrameDestroy);
1951	}
1952
1953	// Move the FPEXC value back into the register with the VMSR_FPEXC
1954	// instruction.
1955	if (RegPresent (ARM::FPEXC)) {
1956	BuildMI(BB&: MBB, I: MI, MIMD: DebugLoc (), MCID: STI.getInstrInfo()->get(Opcode: ARM::VMSR_FPEXC))
1957	.addReg(RegNo: ARM::R5)
1958	.add(MOs: predOps(Pred: ARMCC::AL))
1959	.setMIFlags(MachineInstr::FrameDestroy);
1960	}
1961	}
1962
1963	/// Emit aligned spill instructions for NumAlignedDPRCS2Regs D-registers
1964	/// starting from d8. Also insert stack realignment code and leave the stack
1965	/// pointer pointing to the d8 spill slot.
1966	static void emitAlignedDPRCS2Spills(MachineBasicBlock &MBB,
1967	MachineBasicBlock::iterator MI,
1968	unsigned NumAlignedDPRCS2Regs,
1969	ArrayRef<CalleeSavedInfo> CSI,
1970	const TargetRegisterInfo *TRI) {
1971	MachineFunction &MF = *MBB.getParent();
1972	ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
1973	DebugLoc DL = MI != MBB.end() ? MI ->getDebugLoc() : DebugLoc ();
1974	const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1975	MachineFrameInfo &MFI = MF.getFrameInfo();
1976
1977	// Mark the D-register spill slots as properly aligned. Since MFI computes
1978	// stack slot layout backwards, this can actually mean that the d-reg stack
1979	// slot offsets can be wrong. The offset for d8 will always be correct.
1980	for (const CalleeSavedInfo &I : CSI) {
1981	unsigned DNum = I.getReg() - ARM::D8;
1982	if (DNum > NumAlignedDPRCS2Regs - `1`)
1983	continue;
1984	int FI = I.getFrameIdx();
1985	// The even-numbered registers will be 16-byte aligned, the odd-numbered
1986	// registers will be 8-byte aligned.
1987	MFI.setObjectAlignment(ObjectIdx: FI, Alignment: DNum % `2` ? Align (`8`) : Align (`16`));
1988
1989	// The stack slot for D8 needs to be maximally aligned because this is
1990	// actually the point where we align the stack pointer. MachineFrameInfo
1991	// computes all offsets relative to the incoming stack pointer which is a
1992	// bit weird when realigning the stack. Any extra padding for this
1993	// over-alignment is not realized because the code inserted below adjusts
1994	// the stack pointer by numregs 8 before aligning the stack pointer.*
1995	if (DNum == `0`)
1996	MFI.setObjectAlignment(ObjectIdx: FI, Alignment: MFI.getMaxAlign());
1997	}
1998
1999	// Move the stack pointer to the d8 spill slot, and align it at the same
2000	// time. Leave the stack slot address in the scratch register r4.
2001	//
2002	// sub r4, sp, #numregs 8*
2003	// bic r4, r4, #align - 1
2004	// mov sp, r4
2005	//
2006	bool isThumb = AFI->isThumbFunction();
2007	assert(!AFI->isThumb1OnlyFunction() && "Can't realign stack for thumb1");
2008	AFI->setShouldRestoreSPFromFP(true);
2009
2010	// sub r4, sp, #numregs 8*
2011	// The immediate is <= 64, so it doesn't need any special encoding.
2012	unsigned Opc = isThumb ? ARM::t2SUBri : ARM::SUBri;
2013	BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: ARM::R4)
2014	.addReg(RegNo: ARM::SP)
2015	.addImm(Val: `8` * NumAlignedDPRCS2Regs)
2016	.add(MOs: predOps(Pred: ARMCC::AL))
2017	.add(MO: condCodeOp());
2018
2019	Align MaxAlign = MF.getFrameInfo().getMaxAlign();
2020	// We must set parameter MustBeSingleInstruction to true, since
2021	// skipAlignedDPRCS2Spills expects exactly 3 instructions to perform
2022	// stack alignment. Luckily, this can always be done since all ARM
2023	// architecture versions that support Neon also support the BFC
2024	// instruction.
2025	emitAligningInstructions(MF, AFI, TII, MBB, MBBI: MI, DL, Reg: ARM::R4, Alignment: MaxAlign, MustBeSingleInstruction: true);
2026
2027	// mov sp, r4
2028	// The stack pointer must be adjusted before spilling anything, otherwise
2029	// the stack slots could be clobbered by an interrupt handler.
2030	// Leave r4 live, it is used below.
2031	Opc = isThumb ? ARM::tMOVr : ARM::MOVr;
2032	MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: ARM::SP)
2033	.addReg(RegNo: ARM::R4)
2034	.add(MOs: predOps(Pred: ARMCC::AL));
2035	if (!isThumb)
2036	MIB.add(MO: condCodeOp());
2037
2038	// Now spill NumAlignedDPRCS2Regs registers starting from d8.
2039	// r4 holds the stack slot address.
2040	unsigned NextReg = ARM::D8;
2041
2042	// 16-byte aligned vst1.64 with 4 d-regs and address writeback.
2043	// The writeback is only needed when emitting two vst1.64 instructions.
2044	if (NumAlignedDPRCS2Regs >= `6`) {
2045	MCRegister SupReg =
2046	TRI->getMatchingSuperReg(Reg: NextReg, SubIdx: ARM::dsub_0, RC: &ARM::QQPRRegClass);
2047	MBB.addLiveIn(PhysReg: SupReg);
2048	BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: ARM::VST1d64Qwb_fixed), DestReg: ARM::R4)
2049	.addReg(RegNo: ARM::R4, Flags: RegState::Kill)
2050	.addImm(Val: `16`)
2051	.addReg(RegNo: NextReg)
2052	.addReg(RegNo: SupReg, Flags: RegState::ImplicitKill)
2053	.add(MOs: predOps(Pred: ARMCC::AL));
2054	NextReg += `4`;
2055	NumAlignedDPRCS2Regs -= `4`;
2056	}
2057
2058	// We won't modify r4 beyond this point. It currently points to the next
2059	// register to be spilled.
2060	unsigned R4BaseReg = NextReg;
2061
2062	// 16-byte aligned vst1.64 with 4 d-regs, no writeback.
2063	if (NumAlignedDPRCS2Regs >= `4`) {
2064	MCRegister SupReg =
2065	TRI->getMatchingSuperReg(Reg: NextReg, SubIdx: ARM::dsub_0, RC: &ARM::QQPRRegClass);
2066	MBB.addLiveIn(PhysReg: SupReg);
2067	BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: ARM::VST1d64Q))
2068	.addReg(RegNo: ARM::R4)
2069	.addImm(Val: `16`)
2070	.addReg(RegNo: NextReg)
2071	.addReg(RegNo: SupReg, Flags: RegState::ImplicitKill)
2072	.add(MOs: predOps(Pred: ARMCC::AL));
2073	NextReg += `4`;
2074	NumAlignedDPRCS2Regs -= `4`;
2075	}
2076
2077	// 16-byte aligned vst1.64 with 2 d-regs.
2078	if (NumAlignedDPRCS2Regs >= `2`) {
2079	MCRegister SupReg =
2080	TRI->getMatchingSuperReg(Reg: NextReg, SubIdx: ARM::dsub_0, RC: &ARM::QPRRegClass);
2081	MBB.addLiveIn(PhysReg: SupReg);
2082	BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: ARM::VST1q64))
2083	.addReg(RegNo: ARM::R4)
2084	.addImm(Val: `16`)
2085	.addReg(RegNo: SupReg)
2086	.add(MOs: predOps(Pred: ARMCC::AL));
2087	NextReg += `2`;
2088	NumAlignedDPRCS2Regs -= `2`;
2089	}
2090
2091	// Finally, use a vanilla vstr.64 for the odd last register.
2092	if (NumAlignedDPRCS2Regs) {
2093	MBB.addLiveIn(PhysReg: NextReg);
2094	// vstr.64 uses addrmode5 which has an offset scale of 4.
2095	BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: ARM::VSTRD))
2096	.addReg(RegNo: NextReg)
2097	.addReg(RegNo: ARM::R4)
2098	.addImm(Val: (NextReg - R4BaseReg) * `2`)
2099	.add(MOs: predOps(Pred: ARMCC::AL));
2100	}
2101
2102	// The last spill instruction inserted should kill the scratch register r4.
2103	std::prev(x: MI)->addRegisterKilled(IncomingReg: ARM::R4, RegInfo: TRI);
2104	}
2105
2106	/// Skip past the code inserted by emitAlignedDPRCS2Spills, and return an
2107	/// iterator to the following instruction.
2108	static MachineBasicBlock::iterator
2109	skipAlignedDPRCS2Spills(MachineBasicBlock::iterator MI,
2110	unsigned NumAlignedDPRCS2Regs) {
2111	// sub r4, sp, #numregs 8*
2112	// bic r4, r4, #align - 1
2113	// mov sp, r4
2114	++MI; ++MI; ++MI;
2115	assert(MI->mayStore() && "Expecting spill instruction");
2116
2117	// These switches all fall through.
2118	switch(NumAlignedDPRCS2Regs) {
2119	case `7`:
2120	++MI;
2121	assert(MI->mayStore() && "Expecting spill instruction");
2122	[[fallthrough]];
2123	default:
2124	++MI;
2125	assert(MI->mayStore() && "Expecting spill instruction");
2126	[[fallthrough]];
2127	case `1`:
2128	case `2`:
2129	case `4`:
2130	assert(MI->killsRegister(ARM::R4, /TRI=/nullptr) && "Missed kill flag");
2131	++MI;
2132	}
2133	return MI;
2134	}
2135
2136	/// Emit aligned reload instructions for NumAlignedDPRCS2Regs D-registers
2137	/// starting from d8. These instructions are assumed to execute while the
2138	/// stack is still aligned, unlike the code inserted by emitPopInst.
2139	static void emitAlignedDPRCS2Restores(MachineBasicBlock &MBB,
2140	MachineBasicBlock::iterator MI,
2141	unsigned NumAlignedDPRCS2Regs,
2142	ArrayRef<CalleeSavedInfo> CSI,
2143	const TargetRegisterInfo *TRI) {
2144	MachineFunction &MF = *MBB.getParent();
2145	ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2146	DebugLoc DL = MI != MBB.end() ? MI ->getDebugLoc() : DebugLoc ();
2147	const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
2148
2149	// Find the frame index assigned to d8.
2150	int D8SpillFI = `0`;
2151	for (const CalleeSavedInfo &I : CSI)
2152	if (I.getReg() == ARM::D8) {
2153	D8SpillFI = I.getFrameIdx();
2154	break;
2155	}
2156
2157	// Materialize the address of the d8 spill slot into the scratch register r4.
2158	// This can be fairly complicated if the stack frame is large, so just use
2159	// the normal frame index elimination mechanism to do it. This code runs as
2160	// the initial part of the epilog where the stack and base pointers haven't
2161	// been changed yet.
2162	bool isThumb = AFI->isThumbFunction();
2163	assert(!AFI->isThumb1OnlyFunction() && "Can't realign stack for thumb1");
2164
2165	unsigned Opc = isThumb ? ARM::t2ADDri : ARM::ADDri;
2166	BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: ARM::R4)
2167	.addFrameIndex(Idx: D8SpillFI)
2168	.addImm(Val: `0`)
2169	.add(MOs: predOps(Pred: ARMCC::AL))
2170	.add(MO: condCodeOp());
2171
2172	// Now restore NumAlignedDPRCS2Regs registers starting from d8.
2173	unsigned NextReg = ARM::D8;
2174
2175	// 16-byte aligned vld1.64 with 4 d-regs and writeback.
2176	if (NumAlignedDPRCS2Regs >= `6`) {
2177	MCRegister SupReg =
2178	TRI->getMatchingSuperReg(Reg: NextReg, SubIdx: ARM::dsub_0, RC: &ARM::QQPRRegClass);
2179	BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: ARM::VLD1d64Qwb_fixed), DestReg: NextReg)
2180	.addReg(RegNo: ARM::R4, Flags: RegState::Define)
2181	.addReg(RegNo: ARM::R4, Flags: RegState::Kill)
2182	.addImm(Val: `16`)
2183	.addReg(RegNo: SupReg, Flags: RegState::ImplicitDefine)
2184	.add(MOs: predOps(Pred: ARMCC::AL));
2185	NextReg += `4`;
2186	NumAlignedDPRCS2Regs -= `4`;
2187	}
2188
2189	// We won't modify r4 beyond this point. It currently points to the next
2190	// register to be spilled.
2191	unsigned R4BaseReg = NextReg;
2192
2193	// 16-byte aligned vld1.64 with 4 d-regs, no writeback.
2194	if (NumAlignedDPRCS2Regs >= `4`) {
2195	MCRegister SupReg =
2196	TRI->getMatchingSuperReg(Reg: NextReg, SubIdx: ARM::dsub_0, RC: &ARM::QQPRRegClass);
2197	BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: ARM::VLD1d64Q), DestReg: NextReg)
2198	.addReg(RegNo: ARM::R4)
2199	.addImm(Val: `16`)
2200	.addReg(RegNo: SupReg, Flags: RegState::ImplicitDefine)
2201	.add(MOs: predOps(Pred: ARMCC::AL));
2202	NextReg += `4`;
2203	NumAlignedDPRCS2Regs -= `4`;
2204	}
2205
2206	// 16-byte aligned vld1.64 with 2 d-regs.
2207	if (NumAlignedDPRCS2Regs >= `2`) {
2208	MCRegister SupReg =
2209	TRI->getMatchingSuperReg(Reg: NextReg, SubIdx: ARM::dsub_0, RC: &ARM::QPRRegClass);
2210	BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: ARM::VLD1q64), DestReg: SupReg)
2211	.addReg(RegNo: ARM::R4)
2212	.addImm(Val: `16`)
2213	.add(MOs: predOps(Pred: ARMCC::AL));
2214	NextReg += `2`;
2215	NumAlignedDPRCS2Regs -= `2`;
2216	}
2217
2218	// Finally, use a vanilla vldr.64 for the remaining odd register.
2219	if (NumAlignedDPRCS2Regs)
2220	BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: ARM::VLDRD), DestReg: NextReg)
2221	.addReg(RegNo: ARM::R4)
2222	.addImm(Val: `2` * (NextReg - R4BaseReg))
2223	.add(MOs: predOps(Pred: ARMCC::AL));
2224
2225	// Last store kills r4.
2226	std::prev(x: MI)->addRegisterKilled(IncomingReg: ARM::R4, RegInfo: TRI);
2227	}
2228
2229	bool ARMFrameLowering::spillCalleeSavedRegisters(
2230	MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
2231	ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo TRI) const* {
2232	if (CSI.empty())
2233	return false;
2234
2235	MachineFunction &MF = *MBB.getParent();
2236	ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2237	ARMSubtarget::PushPopSplitVariation PushPopSplit =
2238	STI.getPushPopSplitVariation(MF);
2239	const ARMBaseRegisterInfo *RegInfo = STI.getRegisterInfo();
2240
2241	unsigned PushOpc = AFI->isThumbFunction() ? ARM::t2STMDB_UPD : ARM::STMDB_UPD;
2242	unsigned PushOneOpc = AFI->isThumbFunction() ?
2243	ARM::t2STR_PRE : ARM::STR_PRE_IMM;
2244	unsigned FltOpc = ARM::VSTMDDB_UPD;
2245	unsigned NumAlignedDPRCS2Regs = AFI->getNumAlignedDPRCS2Regs();
2246	// Compute PAC in R12.
2247	if (AFI->shouldSignReturnAddress()) {
2248	BuildMI(BB&: MBB, I: MI, MIMD: DebugLoc (), MCID: STI.getInstrInfo()->get(Opcode: ARM::t2PAC))
2249	.setMIFlags(MachineInstr::FrameSetup);
2250	}
2251	// Save the non-secure floating point context.
2252	if (llvm::any_of(Range&: CSI, P: [](const CalleeSavedInfo &C) {
2253	return C.getReg() == ARM::FPCXTNS;
2254	})) {
2255	BuildMI(BB&: MBB, I: MI, MIMD: DebugLoc (), MCID: STI.getInstrInfo()->get(Opcode: ARM::VSTR_FPCXTNS_pre),
2256	DestReg: ARM::SP)
2257	.addReg(RegNo: ARM::SP)
2258	.addImm(Val: -`4`)
2259	.add(MOs: predOps(Pred: ARMCC::AL));
2260	}
2261
2262	auto CheckRegArea = [PushPopSplit, NumAlignedDPRCS2Regs,
2263	RegInfo](unsigned Reg, SpillArea TestArea) {
2264	return getSpillArea(Reg, Variation: PushPopSplit, NumAlignedDPRCS2Regs, RegInfo) ==
2265	TestArea;
2266	};
2267	auto IsGPRCS1 = [&CheckRegArea](unsigned Reg) {
2268	return CheckRegArea (Reg, SpillArea::GPRCS1);
2269	};
2270	auto IsGPRCS2 = [&CheckRegArea](unsigned Reg) {
2271	return CheckRegArea (Reg, SpillArea::GPRCS2);
2272	};
2273	auto IsDPRCS1 = [&CheckRegArea](unsigned Reg) {
2274	return CheckRegArea (Reg, SpillArea::DPRCS1);
2275	};
2276	auto IsGPRCS3 = [&CheckRegArea](unsigned Reg) {
2277	return CheckRegArea (Reg, SpillArea::GPRCS3);
2278	};
2279
2280	emitPushInst(MBB, MI, CSI, StmOpc: PushOpc, StrOpc: PushOneOpc, NoGap: false, Func: IsGPRCS1);
2281	emitPushInst(MBB, MI, CSI, StmOpc: PushOpc, StrOpc: PushOneOpc, NoGap: false, Func: IsGPRCS2);
2282	emitFPStatusSaves(MBB, MI, CSI, PushOpc);
2283	emitPushInst(MBB, MI, CSI, StmOpc: FltOpc, StrOpc: `0`, NoGap: true, Func: IsDPRCS1);
2284	emitPushInst(MBB, MI, CSI, StmOpc: PushOpc, StrOpc: PushOneOpc, NoGap: false, Func: IsGPRCS3);
2285
2286	// The code above does not insert spill code for the aligned DPRCS2 registers.
2287	// The stack realignment code will be inserted between the push instructions
2288	// and these spills.
2289	if (NumAlignedDPRCS2Regs)
2290	emitAlignedDPRCS2Spills(MBB, MI, NumAlignedDPRCS2Regs, CSI, TRI);
2291
2292	return true;
2293	}
2294
2295	bool ARMFrameLowering::restoreCalleeSavedRegisters(
2296	MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
2297	MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo TRI) const* {
2298	if (CSI.empty())
2299	return false;
2300
2301	MachineFunction &MF = *MBB.getParent();
2302	ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2303	const ARMBaseRegisterInfo *RegInfo = STI.getRegisterInfo();
2304
2305	bool isVarArg = AFI->getArgRegsSaveSize() > `0`;
2306	unsigned NumAlignedDPRCS2Regs = AFI->getNumAlignedDPRCS2Regs();
2307	ARMSubtarget::PushPopSplitVariation PushPopSplit =
2308	STI.getPushPopSplitVariation(MF);
2309
2310	// The emitPopInst calls below do not insert reloads for the aligned DPRCS2
2311	// registers. Do that here instead.
2312	if (NumAlignedDPRCS2Regs)
2313	emitAlignedDPRCS2Restores(MBB, MI, NumAlignedDPRCS2Regs, CSI, TRI);
2314
2315	unsigned PopOpc = AFI->isThumbFunction() ? ARM::t2LDMIA_UPD : ARM::LDMIA_UPD;
2316	unsigned LdrOpc =
2317	AFI->isThumbFunction() ? ARM::t2LDR_POST : ARM::LDR_POST_IMM;
2318	unsigned FltOpc = ARM::VLDMDIA_UPD;
2319
2320	auto CheckRegArea = [PushPopSplit, NumAlignedDPRCS2Regs,
2321	RegInfo](unsigned Reg, SpillArea TestArea) {
2322	return getSpillArea(Reg, Variation: PushPopSplit, NumAlignedDPRCS2Regs, RegInfo) ==
2323	TestArea;
2324	};
2325	auto IsGPRCS1 = [&CheckRegArea](unsigned Reg) {
2326	return CheckRegArea (Reg, SpillArea::GPRCS1);
2327	};
2328	auto IsGPRCS2 = [&CheckRegArea](unsigned Reg) {
2329	return CheckRegArea (Reg, SpillArea::GPRCS2);
2330	};
2331	auto IsDPRCS1 = [&CheckRegArea](unsigned Reg) {
2332	return CheckRegArea (Reg, SpillArea::DPRCS1);
2333	};
2334	auto IsGPRCS3 = [&CheckRegArea](unsigned Reg) {
2335	return CheckRegArea (Reg, SpillArea::GPRCS3);
2336	};
2337
2338	emitPopInst(MBB, MI, CSI, LdmOpc: PopOpc, LdrOpc, isVarArg, NoGap: false, Func: IsGPRCS3);
2339	emitPopInst(MBB, MI, CSI, LdmOpc: FltOpc, LdrOpc: `0`, isVarArg, NoGap: true, Func: IsDPRCS1);
2340	emitFPStatusRestores(MBB, MI, CSI, LdmOpc: PopOpc);
2341	emitPopInst(MBB, MI, CSI, LdmOpc: PopOpc, LdrOpc, isVarArg, NoGap: false, Func: IsGPRCS2);
2342	emitPopInst(MBB, MI, CSI, LdmOpc: PopOpc, LdrOpc, isVarArg, NoGap: false, Func: IsGPRCS1);
2343
2344	return true;
2345	}
2346
2347	// FIXME: Make generic?
2348	static unsigned EstimateFunctionSizeInBytes(const MachineFunction &MF,
2349	const ARMBaseInstrInfo &TII,
2350	const ARMSubtarget &STI,
2351	const ARMBaseRegisterInfo *RegInfo,
2352	BitVector &SavedRegs,
2353	bool BigFrameOffsets) {
2354	unsigned FnSize = `0`;
2355
2356	bool KCFI = MF.getFunction().getParent()->getModuleFlag(Key: "kcfi");
2357
2358	if (MF.shouldSplitStack()) {
2359	// Split stack prologue saves r4,r5; makes a copy of sp and loads
2360	// a literal; compares the two, and if sp < literal, pushes
2361	// further registers and calls __morestack.
2362	FnSize += `0x24`;
2363	}
2364
2365	// Count the number of saved high registers, which take more effort
2366	// to push and pop in the prologue and epilogue.
2367	unsigned SavedHighRegs = `0`;
2368	for (auto Reg : {ARM::R8, ARM::R9, ARM::R10, ARM::R11, ARM::R12})
2369	if (SavedRegs.test(Idx: Reg))
2370	++SavedHighRegs;
2371
2372	// Size of a particularly large Thumb1 stack setup prologue:
2373	// update sp for variadic functions (2 bytes)
2374	// + push registers (2 bytes for PUSH {low regs} + 4 bytes per high register
2375	// that needs to be copied into a low reg and then pushed)
2376	// + frame pointer (might use r11, requiring pushing it first, 6 bytes)
2377	// + stack update (up to 12 bytes if a constant load is needed and a branch
2378	// required later to skip the constant)
2379	// + stack realignment (8, if needed)
2380	// + make base pointer (2).
2381	unsigned PrologueSize = `2` + `4` * SavedHighRegs + `6` + `12` + `2`;
2382	if (RegInfo->hasStackRealignment(MF))
2383	PrologueSize += `8`;
2384	FnSize += PrologueSize;
2385
2386	// Size of a large epilogue:
2387	// restore sp from frame pointer (6 bytes if it's in r11)
2388	// + pop registers (2 bytes + 4 per high register, as above)
2389	// + pop r11 if it was saved to make frame pointer (4 bytes)
2390	// + pop return address into a low reg (2 bytes)
2391	// + update sp to undo variadic function setup (2 bytes)
2392	// + BX to where you popped the return address (2 bytes)
2393	unsigned EpilogueSize = `6` + `2` + `4` * SavedHighRegs + `4` + `2` + `2`;
2394
2395	bool FirstBlock = true;
2396	for (auto &MBB : MF) {
2397	if (!FirstBlock) {
2398	// We might have to insert padding to align the start of this basic
2399	// block.
2400	unsigned Alignment = MBB.getMaxBytesForAlignment();
2401	FnSize += Alignment;
2402	}
2403
2404	unsigned SizeBeforeThisBB = FnSize;
2405
2406	bool seenBranch = false, seenConstantLoad = false;
2407	for (auto &MI : MBB) {
2408	unsigned InstSize;
2409	switch (MI.getOpcode()) {
2410	case ARM::tADDframe:
2411	if (BigFrameOffsets)
2412	// We might need two ADD instructions, or even a constant
2413	// load. In the latter case we must count the constant as
2414	// well as the load instruction and the addition, for 8
2415	// bytes total.
2416	InstSize = `8`;
2417	else
2418	InstSize = `2`;
2419	break;
2420	case ARM::tLDRspi:
2421	case ARM::tSTRspi:
2422	if (BigFrameOffsets)
2423	// In a really nasty case, accessing a stack slot might
2424	// require saving and restoring a scratch register (4 bytes)
2425	// to make space to load (2 bytes) a constant (4 bytes) to
2426	// add to SP or FP (2 bytes) and then do the load/store to
2427	// the resulting register (2 bytes).
2428	InstSize = `14`;
2429	else
2430	InstSize = `2`;
2431	break;
2432	case TargetOpcode::COPY:
2433	// In some situations, COPY has to go via a high register, to
2434	// avoid corrupting the PSR flags: Thumb moves between low and
2435	// high registers don't write the PSR, whereas low/low moves
2436	// do.
2437	InstSize = `4`; // may have to go via a high reg
2438	break;
2439	case ARM::MEMCPY:
2440	InstSize = `4`; // becomes one LDMIA_UPD + STMIA_UPD pair
2441	break;
2442
2443	case ARM::Int_eh_sjlj_dispatchsetup:
2444	// Worst case is 6 bytes, loading a constant from a literal pool.
2445	InstSize = `6`;
2446	break;
2447
2448	case ARM::tLDRpci_pic:
2449	InstSize = `4`; // ordinary LDRpci + add to pc
2450	break;
2451
2452	case ARM::ADJCALLSTACKDOWN:
2453	case ARM::ADJCALLSTACKUP:
2454	InstSize = `2`;
2455	break;
2456
2457	case ARM::tBLXNS_CALL:
2458	// A call across a CMSE trust boundary involves a lot of work. We must
2459	// clear all the registers that might contain our own secrets, which
2460	// means pushing them first. So first push low registers, then move
2461	// four high regs into them and push those too.
2462	InstSize = `2` + `2` * `4` + `2`;
2463	// Then we must clear the low bit of the target register, because
2464	// instead of indicating Arm/Thumb it indicates the CMSE security
2465	// status. That takes two instructions.
2466	InstSize += `4`;
2467	// Now actually clear all the registers, via a MOV for each one. This
2468	// includes argument registers as well as saved regs, so it can be
2469	// r1-r12 inclusive.
2470	InstSize += `2` * `12`;
2471	// Clear the PSR flags (a wide instruction even in Armv8-M Baseline).
2472	InstSize += `4`;
2473	// Perform the BLXNS call itself.
2474	InstSize += `2`;
2475	// Now pop all the saved registers, which takes the same amount of
2476	// effort as pushing them did.
2477	InstSize += `2` + `2` * `4` + `2`;
2478	break;
2479
2480	case ARM::tBXNS_RET:
2481	// When returning across a CMSE boundary, we must potentially clear all
2482	// the registers that we haven't restored to the caller's value: r0-r3
2483	// and r12. We also clear the PSR flags, and finally return via BXNS.
2484	InstSize = `5` * `2` + `4` + `2`;
2485	break;
2486
2487	case TargetOpcode::LOAD_STACK_GUARD:
2488	if (STI.genExecuteOnly())
2489	// In execute-only code generation, it costs seven 2-byte
2490	// instructions (MOV + 3 ADD + 3 LSL) to load an arbitrary
2491	// 32-bit constant, plus two 4-byte MSRs to save/restore the
2492	// flags those instructions clobber. Then we load from the
2493	// resulting address with one more 2-byte instruction.
2494	InstSize = `7` * `2` + `2` * `4` + `8`;
2495	else
2496	// If we're not generating execute-only code, the constant
2497	// just costs an LDR and a literal, and then another LDR is
2498	// needed to load from that address.
2499	InstSize = `2` * `2` + `4`;
2500	break;
2501
2502	default:
2503	InstSize = TII.getInstSizeInBytes(MI);
2504	break;
2505	}
2506
2507	FnSize += InstSize;
2508
2509	// If the instruction loads a constant, score the size of the
2510	// constant, in case it can't be shared with other basic blocks.
2511	for (MachineMemOperand *MO : MI.memoperands()) {
2512	const PseudoSourceValue *PSV =
2513	dyn_cast_if_present<const PseudoSourceValue *>(
2514	Val: MO->getPointerInfo().V);
2515	if (PSV && PSV->kind() == PseudoSourceValue::ConstantPool) {
2516	unsigned ConstSize = MO->getType().getSizeInBytes();
2517	FnSize += ConstSize;
2518	seenConstantLoad = true;
2519	}
2520	}
2521
2522	// If the instruction is a return or a tailcall, count the size
2523	// of an epilogue. (We do this for each return, in case the
2524	// epilogue must be duplicated.)
2525	if (MI.isReturn() \|\| TII.isTailCall(Inst: MI)) {
2526	FnSize += EpilogueSize;
2527	}
2528
2529	// If the instruction is a call, and KCFI is enabled, then count the
2530	// cost of a KCFI_CHECK_Thumb1 pseudo.
2531	if (KCFI && MI.isCall() && MI.getCFIType()) {
2532	const MCInstrDesc &MCID = TII.get(Opcode: ARM::KCFI_CHECK_Thumb1);
2533	FnSize += MCID.getSize();
2534	}
2535
2536	if (MI.isUnconditionalBranch())
2537	seenBranch = true;
2538	}
2539
2540	// If there's no branch instruction in the block and we saw a constant,
2541	// count a branch + realignment to 4 bytes, in case we have to branch round
2542	// it.
2543	if (seenConstantLoad && !seenBranch) {
2544	FnSize += `4`;
2545	}
2546
2547	// Also, if the block is really, really big, then count an extra 4 bytes
2548	// (again branch + realignment) for potentially splitting it in order to
2549	// put constants in the middle, avoiding the problem of an LDR not being
2550	// able to reach all the way to the end. The LDR offset limit is 1024
2551	// bytes; the splitting itself adds some cost, but since any function this
2552	// large is likely to have already gone over the "must stack LR" limit, we
2553	// can keep things simple by assuming we split at half that rate.
2554	unsigned BBSize = FnSize - SizeBeforeThisBB;
2555	FnSize += `4` * BBSize / `512`;
2556	}
2557	if (MF.getJumpTableInfo()) {
2558	for (auto &Table : MF.getJumpTableInfo()->getJumpTables()) {
2559	unsigned TableLen = Table.MBBs.size();
2560	unsigned TableSizeBytes = TableLen * `4`;
2561	FnSize += TableSizeBytes;
2562	}
2563	}
2564	LLVM_DEBUG(dbgs() << "Estimated function size for " << MF.getName() << " = "
2565	<< FnSize << " bytes\n");
2566	return FnSize;
2567	}
2568
2569	/// estimateRSStackSizeLimit - Look at each instruction that references stack
2570	/// frames and return the stack size limit beyond which some of these
2571	/// instructions will require a scratch register during their expansion later.
2572	// FIXME: Move to TII?
2573	static unsigned estimateRSStackSizeLimit(MachineFunction &MF,
2574	const TargetFrameLowering *TFI,
2575	bool &HasNonSPFrameIndex) {
2576	const ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2577	const ARMBaseInstrInfo &TII =
2578	*static_cast<const ARMBaseInstrInfo *>(MF.getSubtarget().getInstrInfo());
2579	unsigned Limit = (`1` << `12`) - `1`;
2580	for (auto &MBB : MF) {
2581	for (auto &MI : MBB) {
2582	if (MI.isDebugInstr())
2583	continue;
2584	if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE)
2585	continue;
2586	for (unsigned i = `0`, e = MI.getNumOperands(); i != e; ++i) {
2587	if (!MI.getOperand(i).isFI())
2588	continue;
2589
2590	// When using ADDri to get the address of a stack object, 255 is the
2591	// largest offset guaranteed to fit in the immediate offset.
2592	if (MI.getOpcode() == ARM::ADDri) {
2593	Limit = std::min(a: Limit, b: (`1U` << `8`) - `1`);
2594	break;
2595	}
2596	// t2ADDri will not require an extra register, it can reuse the
2597	// destination.
2598	if (MI.getOpcode() == ARM::t2ADDri \|\| MI.getOpcode() == ARM::t2ADDri12)
2599	break;
2600
2601	const MCInstrDesc &MCID = MI.getDesc();
2602	const TargetRegisterClass *RegClass = TII.getRegClass(MCID, OpNum: i);
2603	if (RegClass && !RegClass->contains(Reg: ARM::SP))
2604	HasNonSPFrameIndex = true;
2605
2606	// Otherwise check the addressing mode.
2607	switch (MI.getDesc().TSFlags & ARMII::AddrModeMask) {
2608	case ARMII::AddrMode_i12:
2609	case ARMII::AddrMode2:
2610	// Default 12 bit limit.
2611	break;
2612	case ARMII::AddrMode3:
2613	case ARMII::AddrModeT2_i8neg:
2614	Limit = std::min(a: Limit, b: (`1U` << `8`) - `1`);
2615	break;
2616	case ARMII::AddrMode5FP16:
2617	Limit = std::min(a: Limit, b: ((`1U` << `8`) - `1`) * `2`);
2618	break;
2619	case ARMII::AddrMode5:
2620	case ARMII::AddrModeT2_i8s4:
2621	case ARMII::AddrModeT2_ldrex:
2622	Limit = std::min(a: Limit, b: ((`1U` << `8`) - `1`) * `4`);
2623	break;
2624	case ARMII::AddrModeT2_i12:
2625	// i12 supports only positive offset so these will be converted to
2626	// i8 opcodes. See llvm::rewriteT2FrameIndex.
2627	if (TFI->hasFP(MF) && AFI->hasStackFrame())
2628	Limit = std::min(a: Limit, b: (`1U` << `8`) - `1`);
2629	break;
2630	case ARMII::AddrMode4:
2631	case ARMII::AddrMode6:
2632	// Addressing modes 4 & 6 (load/store) instructions can't encode an
2633	// immediate offset for stack references.
2634	return `0`;
2635	case ARMII::AddrModeT2_i7:
2636	Limit = std::min(a: Limit, b: ((`1U` << `7`) - `1`) * `1`);
2637	break;
2638	case ARMII::AddrModeT2_i7s2:
2639	Limit = std::min(a: Limit, b: ((`1U` << `7`) - `1`) * `2`);
2640	break;
2641	case ARMII::AddrModeT2_i7s4:
2642	Limit = std::min(a: Limit, b: ((`1U` << `7`) - `1`) * `4`);
2643	break;
2644	default:
2645	llvm_unreachable("Unhandled addressing mode in stack size limit calculation");
2646	}
2647	break; // At most one FI per instruction
2648	}
2649	}
2650	}
2651
2652	return Limit;
2653	}
2654
2655	// In functions that realign the stack, it can be an advantage to spill the
2656	// callee-saved vector registers after realigning the stack. The vst1 and vld1
2657	// instructions take alignment hints that can improve performance.
2658	static void
2659	checkNumAlignedDPRCS2Regs(MachineFunction &MF, BitVector &SavedRegs) {
2660	MF.getInfo<ARMFunctionInfo>()->setNumAlignedDPRCS2Regs(`0`);
2661	if (!SpillAlignedNEONRegs)
2662	return;
2663
2664	// Naked functions don't spill callee-saved registers.
2665	if (MF.getFunction().hasFnAttribute(Kind: Attribute::Naked))
2666	return;
2667
2668	// We are planning to use NEON instructions vst1 / vld1.
2669	if (!MF.getSubtarget<ARMSubtarget>().hasNEON())
2670	return;
2671
2672	// Don't bother if the default stack alignment is sufficiently high.
2673	if (MF.getSubtarget().getFrameLowering()->getStackAlign() >= Align (`8`))
2674	return;
2675
2676	// Aligned spills require stack realignment.
2677	if (!static_cast<const ARMBaseRegisterInfo *>(
2678	MF.getSubtarget().getRegisterInfo())->canRealignStack(MF))
2679	return;
2680
2681	// We always spill contiguous d-registers starting from d8. Count how many
2682	// needs spilling. The register allocator will almost always use the
2683	// callee-saved registers in order, but it can happen that there are holes in
2684	// the range. Registers above the hole will be spilled to the standard DPRCS
2685	// area.
2686	unsigned NumSpills = `0`;
2687	for (; NumSpills < `8`; ++NumSpills)
2688	if (!SavedRegs.test(Idx: ARM::D8 + NumSpills))
2689	break;
2690
2691	// Don't do this for just one d-register. It's not worth it.
2692	if (NumSpills < `2`)
2693	return;
2694
2695	// Spill the first NumSpills D-registers after realigning the stack.
2696	MF.getInfo<ARMFunctionInfo>()->setNumAlignedDPRCS2Regs(NumSpills);
2697
2698	// A scratch register is required for the vst1 / vld1 instructions.
2699	SavedRegs.set(ARM::R4);
2700	}
2701
2702	bool ARMFrameLowering::enableShrinkWrapping(const MachineFunction &MF) const {
2703	// For CMSE entry functions, we want to save the FPCXT_NS immediately
2704	// upon function entry (resp. restore it immediately before return)
2705	if (STI.hasV8_1MMainlineOps() &&
2706	MF.getInfo<ARMFunctionInfo>()->isCmseNSEntryFunction())
2707	return false;
2708
2709	// We are disabling shrinkwrapping for now when PAC is enabled, as
2710	// shrinkwrapping can cause clobbering of r12 when the PAC code is
2711	// generated. A follow-up patch will fix this in a more performant manner.
2712	if (MF.getInfo<ARMFunctionInfo>()->shouldSignReturnAddress(
2713	SpillsLR: true / SpillsLR /))
2714	return false;
2715
2716	return true;
2717	}
2718
2719	bool ARMFrameLowering::requiresAAPCSFrameRecord(
2720	const MachineFunction &MF) const {
2721	const auto &Subtarget = MF.getSubtarget<ARMSubtarget>();
2722	return Subtarget.createAAPCSFrameChain() && hasFP(MF);
2723	}
2724
2725	// Thumb1 may require a spill when storing to a frame index through FP (or any
2726	// access with execute-only), for cases where FP is a high register (R11). This
2727	// scans the function for cases where this may happen.
2728	static bool canSpillOnFrameIndexAccess(const MachineFunction &MF,
2729	const TargetFrameLowering &TFI) {
2730	const ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2731	if (!AFI->isThumb1OnlyFunction())
2732	return false;
2733
2734	const ARMSubtarget &STI = MF.getSubtarget<ARMSubtarget>();
2735	for (const auto &MBB : MF)
2736	for (const auto &MI : MBB)
2737	if (MI.getOpcode() == ARM::tSTRspi \|\| MI.getOpcode() == ARM::tSTRi \|\|
2738	STI.genExecuteOnly())
2739	for (const auto &Op : MI.operands())
2740	if (Op.isFI()) {
2741	Register Reg;
2742	TFI.getFrameIndexReference(MF, FI: Op.getIndex(), FrameReg&: Reg);
2743	if (ARM::hGPRRegClass.contains(Reg) && Reg != ARM::SP)
2744	return true;
2745	}
2746	return false;
2747	}
2748
2749	void ARMFrameLowering::determineCalleeSaves(MachineFunction &MF,
2750	BitVector &SavedRegs,
2751	RegScavenger RS) const* {
2752	TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
2753	// This tells PEI to spill the FP as if it is any other callee-save register
2754	// to take advantage the eliminateFrameIndex machinery. This also ensures it
2755	// is spilled in the order specified by getCalleeSavedRegs() to make it easier
2756	// to combine multiple loads / stores.
2757	bool CanEliminateFrame = !(requiresAAPCSFrameRecord(MF) && hasFP(MF)) &&
2758	!MF.disableFramePointerElim();
2759	bool CS1Spilled = false;
2760	bool LRSpilled = false;
2761	unsigned NumGPRSpills = `0`;
2762	unsigned NumFPRSpills = `0`;
2763	SmallVector<unsigned, `4`> UnspilledCS1GPRs;
2764	SmallVector<unsigned, `4`> UnspilledCS2GPRs;
2765	const Function &F = MF.getFunction();
2766	const ARMBaseRegisterInfo RegInfo = static_cast<const* ARMBaseRegisterInfo *>(
2767	MF.getSubtarget().getRegisterInfo());
2768	const ARMBaseInstrInfo &TII =
2769	*static_cast<const ARMBaseInstrInfo *>(MF.getSubtarget().getInstrInfo());
2770	ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2771	MachineFrameInfo &MFI = MF.getFrameInfo();
2772	MachineRegisterInfo &MRI = MF.getRegInfo();
2773	const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
2774	(void)TRI; // Silence unused warning in non-assert builds.
2775	Register FramePtr = STI.getFramePointerReg();
2776	ARMSubtarget::PushPopSplitVariation PushPopSplit =
2777	STI.getPushPopSplitVariation(MF);
2778
2779	// For a floating point interrupt, save these registers always, since LLVM
2780	// currently doesn't model reads/writes to these registers.
2781	if (F.hasFnAttribute(Kind: "interrupt") && F.hasFnAttribute(Kind: "save-fp")) {
2782	SavedRegs.set(ARM::FPSCR);
2783	SavedRegs.set(ARM::R4);
2784
2785	// This register will only be present on non-MClass registers.
2786	if (STI.isMClass()) {
2787	SavedRegs.reset(Idx: ARM::FPEXC);
2788	} else {
2789	SavedRegs.set(ARM::FPEXC);
2790	SavedRegs.set(ARM::R5);
2791	}
2792	}
2793
2794	// Spill R4 if Thumb2 function requires stack realignment - it will be used as
2795	// scratch register. Also spill R4 if Thumb2 function has varsized objects,
2796	// since it's not always possible to restore sp from fp in a single
2797	// instruction.
2798	// FIXME: It will be better just to find spare register here.
2799	if (AFI->isThumb2Function() &&
2800	(MFI.hasVarSizedObjects() \|\| RegInfo->hasStackRealignment(MF)))
2801	SavedRegs.set(ARM::R4);
2802
2803	// If a stack probe will be emitted, spill R4 and LR, since they are
2804	// clobbered by the stack probe call.
2805	// This estimate should be a safe, conservative estimate. The actual
2806	// stack probe is enabled based on the size of the local objects;
2807	// this estimate also includes the varargs store size.
2808	if (STI.isTargetWindows() &&
2809	WindowsRequiresStackProbe(MF, StackSizeInBytes: MFI.estimateStackSize(MF))) {
2810	SavedRegs.set(ARM::R4);
2811	SavedRegs.set(ARM::LR);
2812	}
2813
2814	if (AFI->isThumb1OnlyFunction()) {
2815	// Spill LR if Thumb1 function uses variable length argument lists.
2816	if (AFI->getArgRegsSaveSize() > `0`)
2817	SavedRegs.set(ARM::LR);
2818
2819	// Spill R4 if Thumb1 epilogue has to restore SP from FP or the function
2820	// requires stack alignment. We don't know for sure what the stack size
2821	// will be, but for this, an estimate is good enough. If there anything
2822	// changes it, it'll be a spill, which implies we've used all the registers
2823	// and so R4 is already used, so not marking it here will be OK.
2824	// FIXME: It will be better just to find spare register here.
2825	if (MFI.hasVarSizedObjects() \|\| RegInfo->hasStackRealignment(MF) \|\|
2826	MFI.estimateStackSize(MF) > `508`)
2827	SavedRegs.set(ARM::R4);
2828	}
2829
2830	// See if we can spill vector registers to aligned stack.
2831	checkNumAlignedDPRCS2Regs(MF, SavedRegs);
2832
2833	// Spill the BasePtr if it's used.
2834	if (RegInfo->hasBasePointer(MF))
2835	SavedRegs.set(RegInfo->getBaseRegister());
2836
2837	// On v8.1-M.Main CMSE entry functions save/restore FPCXT.
2838	if (STI.hasV8_1MMainlineOps() && AFI->isCmseNSEntryFunction())
2839	CanEliminateFrame = false;
2840
2841	// When return address signing is enabled R12 is treated as callee-saved.
2842	if (AFI->shouldSignReturnAddress())
2843	CanEliminateFrame = false;
2844
2845	// Don't spill FP if the frame can be eliminated. This is determined
2846	// by scanning the callee-save registers to see if any is modified.
2847	const MCPhysReg *CSRegs = RegInfo->getCalleeSavedRegs(MF: &MF);
2848	for (unsigned i = `0`; CSRegs[i]; ++i) {
2849	unsigned Reg = CSRegs[i];
2850	bool Spilled = false;
2851	if (SavedRegs.test(Idx: Reg)) {
2852	Spilled = true;
2853	CanEliminateFrame = false;
2854	}
2855
2856	if (!ARM::GPRRegClass.contains(Reg)) {
2857	if (Spilled) {
2858	if (ARM::SPRRegClass.contains(Reg))
2859	NumFPRSpills++;
2860	else if (ARM::DPRRegClass.contains(Reg))
2861	NumFPRSpills += `2`;
2862	else if (ARM::QPRRegClass.contains(Reg))
2863	NumFPRSpills += `4`;
2864	}
2865	continue;
2866	}
2867
2868	if (Spilled) {
2869	NumGPRSpills++;
2870
2871	if (PushPopSplit != ARMSubtarget::SplitR7) {
2872	if (Reg == ARM::LR)
2873	LRSpilled = true;
2874	CS1Spilled = true;
2875	continue;
2876	}
2877
2878	// Keep track if LR and any of R4, R5, R6, and R7 is spilled.
2879	switch (Reg) {
2880	case ARM::LR:
2881	LRSpilled = true;
2882	[[fallthrough]];
2883	case ARM::R0: case ARM::R1:
2884	case ARM::R2: case ARM::R3:
2885	case ARM::R4: case ARM::R5:
2886	case ARM::R6: case ARM::R7:
2887	CS1Spilled = true;
2888	break;
2889	default:
2890	break;
2891	}
2892	} else {
2893	if (PushPopSplit != ARMSubtarget::SplitR7) {
2894	UnspilledCS1GPRs.push_back(Elt: Reg);
2895	continue;
2896	}
2897
2898	switch (Reg) {
2899	case ARM::R0: case ARM::R1:
2900	case ARM::R2: case ARM::R3:
2901	case ARM::R4: case ARM::R5:
2902	case ARM::R6: case ARM::R7:
2903	case ARM::LR:
2904	UnspilledCS1GPRs.push_back(Elt: Reg);
2905	break;
2906	default:
2907	UnspilledCS2GPRs.push_back(Elt: Reg);
2908	break;
2909	}
2910	}
2911	}
2912
2913	bool ForceLRSpill = false;
2914
2915	// If any of the stack slot references may be out of range of an immediate
2916	// offset, make sure a register (or a spill slot) is available for the
2917	// register scavenger. Note that if we're indexing off the frame pointer, the
2918	// effective stack size is 4 bytes larger since the FP points to the stack
2919	// slot of the previous FP. Also, if we have variable sized objects in the
2920	// function, stack slot references will often be negative, and some of
2921	// our instructions are positive-offset only, so conservatively consider
2922	// that case to want a spill slot (or register) as well. Similarly, if
2923	// the function adjusts the stack pointer during execution and the
2924	// adjustments aren't already part of our stack size estimate, our offset
2925	// calculations may be off, so be conservative.
2926	// FIXME: We could add logic to be more precise about negative offsets
2927	// and which instructions will need a scratch register for them. Is it
2928	// worth the effort and added fragility?
2929	unsigned EstimatedStackSize =
2930	MFI.estimateStackSize(MF) + `4` * (NumGPRSpills + NumFPRSpills);
2931
2932	// Determine biggest (positive) SP offset in MachineFrameInfo.
2933	int MaxFixedOffset = `0`;
2934	for (int I = MFI.getObjectIndexBegin(); I < `0`; ++I) {
2935	int MaxObjectOffset = MFI.getObjectOffset(ObjectIdx: I) + MFI.getObjectSize(ObjectIdx: I);
2936	MaxFixedOffset = std::max(a: MaxFixedOffset, b: MaxObjectOffset);
2937	}
2938
2939	bool HasFP = hasFP(MF);
2940	if (HasFP) {
2941	if (AFI->hasStackFrame())
2942	EstimatedStackSize += `4`;
2943	} else {
2944	// If FP is not used, SP will be used to access arguments, so count the
2945	// size of arguments into the estimation.
2946	EstimatedStackSize += MaxFixedOffset;
2947	}
2948	EstimatedStackSize += `16`; // For possible paddings.
2949
2950	unsigned EstimatedRSStackSizeLimit, EstimatedRSFixedSizeLimit;
2951	bool HasNonSPFrameIndex = false;
2952	if (AFI->isThumb1OnlyFunction()) {
2953	// For Thumb1, don't bother to iterate over the function. The only
2954	// instruction that requires an emergency spill slot is a store to a
2955	// frame index.
2956	//
2957	// tSTRspi, which is used for sp-relative accesses, has an 8-bit unsigned
2958	// immediate. tSTRi, which is used for bp- and fp-relative accesses, has
2959	// a 5-bit unsigned immediate.
2960	//
2961	// We could try to check if the function actually contains a tSTRspi
2962	// that might need the spill slot, but it's not really important.
2963	// Functions with VLAs or extremely large call frames are rare, and
2964	// if a function is allocating more than 1KB of stack, an extra 4-byte
2965	// slot probably isn't relevant.
2966	//
2967	// A special case is the scenario where r11 is used as FP, where accesses
2968	// to a frame index will require its value to be moved into a low reg.
2969	// This is handled later on, once we are able to determine if we have any
2970	// fp-relative accesses.
2971	if (RegInfo->hasBasePointer(MF))
2972	EstimatedRSStackSizeLimit = (`1U` << `5`) * `4`;
2973	else
2974	EstimatedRSStackSizeLimit = (`1U` << `8`) * `4`;
2975	EstimatedRSFixedSizeLimit = (`1U` << `5`) * `4`;
2976	} else {
2977	EstimatedRSStackSizeLimit =
2978	estimateRSStackSizeLimit(MF, TFI: this, HasNonSPFrameIndex);
2979	EstimatedRSFixedSizeLimit = EstimatedRSStackSizeLimit;
2980	}
2981	// Final estimate of whether sp or bp-relative accesses might require
2982	// scavenging.
2983	bool HasLargeStack = EstimatedStackSize > EstimatedRSStackSizeLimit;
2984
2985	// If the stack pointer moves and we don't have a base pointer, the
2986	// estimate logic doesn't work. The actual offsets might be larger when
2987	// we're constructing a call frame, or we might need to use negative
2988	// offsets from fp.
2989	bool HasMovingSP = MFI.hasVarSizedObjects() \|\|
2990	(MFI.adjustsStack() && !canSimplifyCallFramePseudos(MF));
2991	bool HasBPOrFixedSP = RegInfo->hasBasePointer(MF) \|\| !HasMovingSP;
2992
2993	// If we have a frame pointer, we assume arguments will be accessed
2994	// relative to the frame pointer. Check whether fp-relative accesses to
2995	// arguments require scavenging.
2996	//
2997	// We could do slightly better on Thumb1; in some cases, an sp-relative
2998	// offset would be legal even though an fp-relative offset is not.
2999	int MaxFPOffset = getMaxFPOffset(STI, AFI: *AFI, MF);
3000	bool HasLargeArgumentList =
3001	HasFP && (MaxFixedOffset - MaxFPOffset) > (int)EstimatedRSFixedSizeLimit;
3002
3003	bool BigFrameOffsets = HasLargeStack \|\| !HasBPOrFixedSP \|\|
3004	HasLargeArgumentList \|\| HasNonSPFrameIndex;
3005	LLVM_DEBUG(dbgs() << "EstimatedLimit: " << EstimatedRSStackSizeLimit
3006	<< "; EstimatedStack: " << EstimatedStackSize
3007	<< "; EstimatedFPStack: " << MaxFixedOffset - MaxFPOffset
3008	<< "; BigFrameOffsets: " << BigFrameOffsets << "\n");
3009
3010	if (!LRSpilled && AFI->isThumb1OnlyFunction()) {
3011	unsigned FnSize = EstimateFunctionSizeInBytes(MF, TII, STI, RegInfo,
3012	SavedRegs, BigFrameOffsets);
3013
3014	if (FnSize >= (`1` << `11`)) {
3015	// Force LR to be spilled if the Thumb function size is > 2048. This
3016	// enables use of BL to implement far jump.
3017	CanEliminateFrame = false;
3018	ForceLRSpill = true;
3019	}
3020	}
3021
3022	if (BigFrameOffsets \|\|
3023	!CanEliminateFrame \|\| RegInfo->cannotEliminateFrame(MF)) {
3024	AFI->setHasStackFrame(true);
3025
3026	// Save the FP if:
3027	// 1. We currently need it (HasFP), OR
3028	// 2. We might need it later due to stack realignment from aligned DPRCS2
3029	// saves (which will make hasFP() become true in emitPrologue).
3030	if (HasFP \|\| (isFPReserved(MF) && AFI->getNumAlignedDPRCS2Regs() > `0`)) {
3031	SavedRegs.set(FramePtr);
3032	// If the frame pointer is required by the ABI, also spill LR so that we
3033	// emit a complete frame record.
3034	if ((requiresAAPCSFrameRecord(MF) \|\| MF.disableFramePointerElim()) &&
3035	!LRSpilled) {
3036	SavedRegs.set(ARM::LR);
3037	LRSpilled = true;
3038	NumGPRSpills++;
3039	auto LRPos = llvm::find(Range&: UnspilledCS1GPRs, Val: ARM::LR);
3040	if (LRPos != UnspilledCS1GPRs.end())
3041	UnspilledCS1GPRs.erase(CI: LRPos);
3042	}
3043	auto FPPos = llvm::find(Range&: UnspilledCS1GPRs, Val: FramePtr);
3044	if (FPPos != UnspilledCS1GPRs.end())
3045	UnspilledCS1GPRs.erase(CI: FPPos);
3046	NumGPRSpills++;
3047	if (FramePtr == ARM::R7)
3048	CS1Spilled = true;
3049	}
3050
3051	// This is the number of extra spills inserted for callee-save GPRs which
3052	// would not otherwise be used by the function. When greater than zero it
3053	// guaranteees that it is possible to scavenge a register to hold the
3054	// address of a stack slot. On Thumb1, the register must be a valid operand
3055	// to tSTRi, i.e. r4-r7. For other subtargets, this is any GPR, i.e. r4-r11
3056	// or lr.
3057	//
3058	// If we don't insert a spill, we instead allocate an emergency spill
3059	// slot, which can be used by scavenging to spill an arbitrary register.
3060	//
3061	// We currently don't try to figure out whether any specific instruction
3062	// requires scavening an additional register.
3063	unsigned NumExtraCSSpill = `0`;
3064
3065	if (AFI->isThumb1OnlyFunction()) {
3066	// For Thumb1-only targets, we need some low registers when we save and
3067	// restore the high registers (which aren't allocatable, but could be
3068	// used by inline assembly) because the push/pop instructions can not
3069	// access high registers. If necessary, we might need to push more low
3070	// registers to ensure that there is at least one free that can be used
3071	// for the saving & restoring, and preferably we should ensure that as
3072	// many as are needed are available so that fewer push/pop instructions
3073	// are required.
3074
3075	// Low registers which are not currently pushed, but could be (r4-r7).
3076	SmallVector<unsigned, `4`> AvailableRegs;
3077
3078	// Unused argument registers (r0-r3) can be clobbered in the prologue for
3079	// free.
3080	int EntryRegDeficit = `0`;
3081	for (unsigned Reg : {ARM::R0, ARM::R1, ARM::R2, ARM::R3}) {
3082	if (!MF.getRegInfo().isLiveIn(Reg)) {
3083	--EntryRegDeficit;
3084	LLVM_DEBUG(dbgs()
3085	<< printReg(Reg, TRI)
3086	<< " is unused argument register, EntryRegDeficit = "
3087	<< EntryRegDeficit << "\n");
3088	}
3089	}
3090
3091	// Unused return registers can be clobbered in the epilogue for free.
3092	int ExitRegDeficit = AFI->getReturnRegsCount() - `4`;
3093	LLVM_DEBUG(dbgs() << AFI->getReturnRegsCount()
3094	<< " return regs used, ExitRegDeficit = "
3095	<< ExitRegDeficit << "\n");
3096
3097	int RegDeficit = std::max(a: EntryRegDeficit, b: ExitRegDeficit);
3098	LLVM_DEBUG(dbgs() << "RegDeficit = " << RegDeficit << "\n");
3099
3100	// r4-r6 can be used in the prologue if they are pushed by the first push
3101	// instruction.
3102	for (unsigned Reg : {ARM::R4, ARM::R5, ARM::R6}) {
3103	if (SavedRegs.test(Idx: Reg)) {
3104	--RegDeficit;
3105	LLVM_DEBUG(dbgs() << printReg(Reg, TRI)
3106	<< " is saved low register, RegDeficit = "
3107	<< RegDeficit << "\n");
3108	} else {
3109	AvailableRegs.push_back(Elt: Reg);
3110	LLVM_DEBUG(
3111	dbgs()
3112	<< printReg(Reg, TRI)
3113	<< " is non-saved low register, adding to AvailableRegs\n");
3114	}
3115	}
3116
3117	// r7 can be used if it is not being used as the frame pointer.
3118	if (!HasFP \|\| FramePtr != ARM::R7) {
3119	if (SavedRegs.test(Idx: ARM::R7)) {
3120	--RegDeficit;
3121	LLVM_DEBUG(dbgs() << "%r7 is saved low register, RegDeficit = "
3122	<< RegDeficit << "\n");
3123	} else {
3124	AvailableRegs.push_back(Elt: ARM::R7);
3125	LLVM_DEBUG(
3126	dbgs()
3127	<< "%r7 is non-saved low register, adding to AvailableRegs\n");
3128	}
3129	}
3130
3131	// Each of r8-r11 needs to be copied to a low register, then pushed.
3132	for (unsigned Reg : {ARM::R8, ARM::R9, ARM::R10, ARM::R11}) {
3133	if (SavedRegs.test(Idx: Reg)) {
3134	++RegDeficit;
3135	LLVM_DEBUG(dbgs() << printReg(Reg, TRI)
3136	<< " is saved high register, RegDeficit = "
3137	<< RegDeficit << "\n");
3138	}
3139	}
3140
3141	// LR can only be used by PUSH, not POP, and can't be used at all if the
3142	// llvm.returnaddress intrinsic is used. This is only worth doing if we
3143	// are more limited at function entry than exit.
3144	if ((EntryRegDeficit > ExitRegDeficit) &&
3145	!(MF.getRegInfo().isLiveIn(Reg: ARM::LR) &&
3146	MF.getFrameInfo().isReturnAddressTaken())) {
3147	if (SavedRegs.test(Idx: ARM::LR)) {
3148	--RegDeficit;
3149	LLVM_DEBUG(dbgs() << "%lr is saved register, RegDeficit = "
3150	<< RegDeficit << "\n");
3151	} else {
3152	AvailableRegs.push_back(Elt: ARM::LR);
3153	LLVM_DEBUG(dbgs() << "%lr is not saved, adding to AvailableRegs\n");
3154	}
3155	}
3156
3157	// If there are more high registers that need pushing than low registers
3158	// available, push some more low registers so that we can use fewer push
3159	// instructions. This might not reduce RegDeficit all the way to zero,
3160	// because we can only guarantee that r4-r6 are available, but r8-r11 may
3161	// need saving.
3162	LLVM_DEBUG(dbgs() << "Final RegDeficit = " << RegDeficit << "\n");
3163	for (; RegDeficit > `0` && !AvailableRegs.empty(); --RegDeficit) {
3164	unsigned Reg = AvailableRegs.pop_back_val();
3165	LLVM_DEBUG(dbgs() << "Spilling " << printReg(Reg, TRI)
3166	<< " to make up reg deficit\n");
3167	SavedRegs.set(Reg);
3168	NumGPRSpills++;
3169	CS1Spilled = true;
3170	assert(!MRI.isReserved(Reg) && "Should not be reserved");
3171	if (Reg != ARM::LR && !MRI.isPhysRegUsed(PhysReg: Reg))
3172	NumExtraCSSpill++;
3173	UnspilledCS1GPRs.erase(CI: llvm::find(Range&: UnspilledCS1GPRs, Val: Reg));
3174	if (Reg == ARM::LR)
3175	LRSpilled = true;
3176	}
3177	LLVM_DEBUG(dbgs() << "After adding spills, RegDeficit = " << RegDeficit
3178	<< "\n");
3179	}
3180
3181	// Avoid spilling LR in Thumb1 if there's a tail call: it's expensive to
3182	// restore LR in that case.
3183	bool ExpensiveLRRestore = AFI->isThumb1OnlyFunction() && MFI.hasTailCall();
3184
3185	// If LR is not spilled, but at least one of R4, R5, R6, and R7 is spilled.
3186	// Spill LR as well so we can fold BX_RET to the registers restore (LDM).
3187	if (!LRSpilled && CS1Spilled && !ExpensiveLRRestore) {
3188	SavedRegs.set(ARM::LR);
3189	NumGPRSpills++;
3190	SmallVectorImpl<unsigned>::iterator LRPos;
3191	LRPos = llvm::find(Range&: UnspilledCS1GPRs, Val: (unsigned)ARM::LR);
3192	if (LRPos != UnspilledCS1GPRs.end())
3193	UnspilledCS1GPRs.erase(CI: LRPos);
3194
3195	ForceLRSpill = false;
3196	if (!MRI.isReserved(PhysReg: ARM::LR) && !MRI.isPhysRegUsed(PhysReg: ARM::LR) &&
3197	!AFI->isThumb1OnlyFunction())
3198	NumExtraCSSpill++;
3199	}
3200
3201	// If stack and double are 8-byte aligned and we are spilling an odd number
3202	// of GPRs, spill one extra callee save GPR so we won't have to pad between
3203	// the integer and double callee save areas.
3204	LLVM_DEBUG(dbgs() << "NumGPRSpills = " << NumGPRSpills << "\n");
3205	const Align TargetAlign = getStackAlign();
3206	if (TargetAlign >= Align (`8`) && (NumGPRSpills & `1`)) {
3207	if (CS1Spilled && !UnspilledCS1GPRs.empty()) {
3208	for (unsigned Reg : UnspilledCS1GPRs) {
3209	// Don't spill high register if the function is thumb. In the case of
3210	// Windows on ARM, accept R11 (frame pointer)
3211	if (!AFI->isThumbFunction() \|\|
3212	(STI.isTargetWindows() && Reg == ARM::R11) \|\|
3213	isARMLowRegister(Reg) \|\|
3214	(Reg == ARM::LR && !ExpensiveLRRestore)) {
3215	SavedRegs.set(Reg);
3216	LLVM_DEBUG(dbgs() << "Spilling " << printReg(Reg, TRI)
3217	<< " to make up alignment\n");
3218	if (!MRI.isReserved(PhysReg: Reg) && !MRI.isPhysRegUsed(PhysReg: Reg) &&
3219	!(Reg == ARM::LR && AFI->isThumb1OnlyFunction()))
3220	NumExtraCSSpill++;
3221	break;
3222	}
3223	}
3224	} else if (!UnspilledCS2GPRs.empty() && !AFI->isThumb1OnlyFunction()) {
3225	unsigned Reg = UnspilledCS2GPRs.front();
3226	SavedRegs.set(Reg);
3227	LLVM_DEBUG(dbgs() << "Spilling " << printReg(Reg, TRI)
3228	<< " to make up alignment\n");
3229	if (!MRI.isReserved(PhysReg: Reg) && !MRI.isPhysRegUsed(PhysReg: Reg))
3230	NumExtraCSSpill++;
3231	}
3232	}
3233
3234	// Estimate if we might need to scavenge registers at some point in order
3235	// to materialize a stack offset. If so, either spill one additional
3236	// callee-saved register or reserve a special spill slot to facilitate
3237	// register scavenging. Thumb1 needs a spill slot for stack pointer
3238	// adjustments and for frame index accesses when FP is high register,
3239	// even when the frame itself is small.
3240	unsigned RegsNeeded = `0`;
3241	if (BigFrameOffsets \|\| canSpillOnFrameIndexAccess(MF, TFI: *this)) {
3242	RegsNeeded++;
3243	// With thumb1 execute-only we may need an additional register for saving
3244	// and restoring the CPSR.
3245	if (AFI->isThumb1OnlyFunction() && STI.genExecuteOnly() && !STI.useMovt())
3246	RegsNeeded++;
3247	}
3248
3249	if (RegsNeeded > NumExtraCSSpill) {
3250	// If any non-reserved CS register isn't spilled, just spill one or two
3251	// extra. That should take care of it!
3252	unsigned NumExtras = TargetAlign.value() / `4`;
3253	SmallVector<unsigned, `2`> Extras;
3254	while (NumExtras && !UnspilledCS1GPRs.empty()) {
3255	unsigned Reg = UnspilledCS1GPRs.pop_back_val();
3256	if (!MRI.isReserved(PhysReg: Reg) &&
3257	(!AFI->isThumb1OnlyFunction() \|\| isARMLowRegister(Reg))) {
3258	Extras.push_back(Elt: Reg);
3259	NumExtras--;
3260	}
3261	}
3262	// For non-Thumb1 functions, also check for hi-reg CS registers
3263	if (!AFI->isThumb1OnlyFunction()) {
3264	while (NumExtras && !UnspilledCS2GPRs.empty()) {
3265	unsigned Reg = UnspilledCS2GPRs.pop_back_val();
3266	if (!MRI.isReserved(PhysReg: Reg)) {
3267	Extras.push_back(Elt: Reg);
3268	NumExtras--;
3269	}
3270	}
3271	}
3272	if (NumExtras == `0`) {
3273	for (unsigned Reg : Extras) {
3274	SavedRegs.set(Reg);
3275	if (!MRI.isPhysRegUsed(PhysReg: Reg))
3276	NumExtraCSSpill++;
3277	}
3278	}
3279	while ((RegsNeeded > NumExtraCSSpill) && RS) {
3280	// Reserve a slot closest to SP or frame pointer.
3281	LLVM_DEBUG(dbgs() << "Reserving emergency spill slot\n");
3282	const TargetRegisterClass &RC = ARM::GPRRegClass;
3283	unsigned Size = TRI->getSpillSize(RC);
3284	Align Alignment = TRI->getSpillAlign(RC);
3285	RS->addScavengingFrameIndex(
3286	FI: MFI.CreateSpillStackObject(Size, Alignment));
3287	--RegsNeeded;
3288	}
3289	}
3290	}
3291
3292	if (ForceLRSpill)
3293	SavedRegs.set(ARM::LR);
3294	AFI->setLRIsSpilled(SavedRegs.test(Idx: ARM::LR));
3295	}
3296
3297	void ARMFrameLowering::updateLRRestored(MachineFunction &MF) {
3298	MachineFrameInfo &MFI = MF.getFrameInfo();
3299	if (!MFI.isCalleeSavedInfoValid())
3300	return;
3301
3302	// Check if all terminators do not implicitly use LR. Then we can 'restore' LR
3303	// into PC so it is not live out of the return block: Clear the Restored bit
3304	// in that case.
3305	for (CalleeSavedInfo &Info : MFI.getCalleeSavedInfo()) {
3306	if (Info.getReg() != ARM::LR)
3307	continue;
3308	if (all_of(Range&: MF, P: [](const MachineBasicBlock &MBB) {
3309	return all_of(Range: MBB.terminators(), P: [](const MachineInstr &Term) {
3310	return !Term.isReturn() \|\| Term.getOpcode() == ARM::LDMIA_RET \|\|
3311	Term.getOpcode() == ARM::t2LDMIA_RET \|\|
3312	Term.getOpcode() == ARM::tPOP_RET;
3313	});
3314	})) {
3315	Info.setRestored(false);
3316	break;
3317	}
3318	}
3319	}
3320
3321	void ARMFrameLowering::processFunctionBeforeFrameFinalized(
3322	MachineFunction &MF, RegScavenger RS) const* {
3323	TargetFrameLowering::processFunctionBeforeFrameFinalized(MF, RS);
3324	updateLRRestored(MF);
3325	}
3326
3327	void ARMFrameLowering::getCalleeSaves(const MachineFunction &MF,
3328	BitVector &SavedRegs) const {
3329	TargetFrameLowering::getCalleeSaves(MF, SavedRegs);
3330
3331	// If we have the "returned" parameter attribute which guarantees that we
3332	// return the value which was passed in r0 unmodified (e.g. C++ 'structors),
3333	// record that fact for IPRA.
3334	const ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3335	if (AFI->getPreservesR0())
3336	SavedRegs.set(ARM::R0);
3337	}
3338
3339	bool ARMFrameLowering::assignCalleeSavedSpillSlots(
3340	MachineFunction &MF, const TargetRegisterInfo *TRI,
3341	std::vector<CalleeSavedInfo> &CSI) const {
3342	// For CMSE entry functions, handle floating-point context as if it was a
3343	// callee-saved register.
3344	if (STI.hasV8_1MMainlineOps() &&
3345	MF.getInfo<ARMFunctionInfo>()->isCmseNSEntryFunction()) {
3346	CSI.emplace_back(args: ARM::FPCXTNS);
3347	CSI.back().setRestored(false);
3348	}
3349
3350	// For functions, which sign their return address, upon function entry, the
3351	// return address PAC is computed in R12. Treat R12 as a callee-saved register
3352	// in this case.
3353	const auto &AFI = *MF.getInfo<ARMFunctionInfo>();
3354	if (AFI.shouldSignReturnAddress()) {
3355	// The order of register must match the order we push them, because the
3356	// PEI assigns frame indices in that order. That order depends on the
3357	// PushPopSplitVariation, there are only two cases which we use with return
3358	// address signing:
3359	switch (STI.getPushPopSplitVariation(MF)) {
3360	case ARMSubtarget::SplitR7:
3361	// LR, R7, R6, R5, R4, <R12>, R11, R10, R9, R8, D15-D8
3362	CSI.insert(position: find_if(Range&: CSI,
3363	P: [=](const auto &CS) {
3364	MCRegister Reg = CS.getReg();
3365	return Reg == ARM::R10 \|\| Reg == ARM::R11 \|\|
3366	Reg == ARM::R8 \|\| Reg == ARM::R9 \|\|
3367	ARM::DPRRegClass.contains(Reg);
3368	}),
3369	x: CalleeSavedInfo (ARM::R12));
3370	break;
3371	case ARMSubtarget::SplitR11AAPCSSignRA:
3372	// With SplitR11AAPCSSignRA, R12 will always be the highest-addressed CSR
3373	// on the stack.
3374	CSI.insert(position: CSI.begin(), x: CalleeSavedInfo (ARM::R12));
3375	break;
3376	case ARMSubtarget::NoSplit:
3377	assert(!MF.disableFramePointerElim() &&
3378	"ABI-required frame pointers need a CSR split when signing return "
3379	"address.");
3380	CSI.insert(position: find_if(Range&: CSI,
3381	P: [=](const auto &CS) {
3382	MCRegister Reg = CS.getReg();
3383	return Reg != ARM::LR;
3384	}),
3385	x: CalleeSavedInfo (ARM::R12));
3386	break;
3387	default:
3388	llvm_unreachable("Unexpected CSR split with return address signing");
3389	}
3390	}
3391
3392	return false;
3393	}
3394
3395	const TargetFrameLowering::SpillSlot *
3396	ARMFrameLowering::getCalleeSavedSpillSlots(unsigned &NumEntries) const {
3397	static const SpillSlot FixedSpillOffsets[] = {{.Reg: ARM::FPCXTNS, .Offset: -`4`}};
3398	NumEntries = std::size(FixedSpillOffsets);
3399	return FixedSpillOffsets;
3400	}
3401
3402	MachineBasicBlock::iterator ARMFrameLowering::eliminateCallFramePseudoInstr(
3403	MachineFunction &MF, MachineBasicBlock &MBB,
3404	MachineBasicBlock::iterator I) const {
3405	const ARMBaseInstrInfo &TII =
3406	*static_cast<const ARMBaseInstrInfo *>(MF.getSubtarget().getInstrInfo());
3407	ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3408	bool isARM = !AFI->isThumbFunction();
3409	DebugLoc dl = I ->getDebugLoc();
3410	unsigned Opc = I ->getOpcode();
3411	bool IsDestroy = Opc == TII.getCallFrameDestroyOpcode();
3412	unsigned CalleePopAmount = IsDestroy ? I ->getOperand(i: `1`).getImm() : `0`;
3413
3414	assert(!AFI->isThumb1OnlyFunction() &&
3415	"This eliminateCallFramePseudoInstr does not support Thumb1!");
3416
3417	int PIdx = I ->findFirstPredOperandIdx();
3418	ARMCC::CondCodes Pred = (PIdx == -`1`)
3419	? ARMCC::AL
3420	: (ARMCC::CondCodes)I ->getOperand(i: PIdx).getImm();
3421	unsigned PredReg = TII.getFramePred(MI: *I);
3422
3423	if (!hasReservedCallFrame(MF)) {
3424	// Bail early if the callee is expected to do the adjustment.
3425	if (IsDestroy && CalleePopAmount != -`1U`)
3426	return MBB.erase(I);
3427
3428	// If we have alloca, convert as follows:
3429	// ADJCALLSTACKDOWN -> sub, sp, sp, amount
3430	// ADJCALLSTACKUP -> add, sp, sp, amount
3431	unsigned Amount = TII.getFrameSize(I: *I);
3432	if (Amount != `0`) {
3433	// We need to keep the stack aligned properly. To do this, we round the
3434	// amount of space needed for the outgoing arguments up to the next
3435	// alignment boundary.
3436	Amount = alignSPAdjust(SPAdj: Amount);
3437
3438	if (Opc == ARM::ADJCALLSTACKDOWN \|\| Opc == ARM::tADJCALLSTACKDOWN) {
3439	emitSPUpdate(isARM, MBB, MBBI&: I, dl, TII, NumBytes: -Amount, MIFlags: MachineInstr::NoFlags,
3440	Pred, PredReg);
3441	} else {
3442	assert(Opc == ARM::ADJCALLSTACKUP \|\| Opc == ARM::tADJCALLSTACKUP);
3443	emitSPUpdate(isARM, MBB, MBBI&: I, dl, TII, NumBytes: Amount, MIFlags: MachineInstr::NoFlags,
3444	Pred, PredReg);
3445	}
3446	}
3447	} else if (CalleePopAmount != -`1U`) {
3448	// If the calling convention demands that the callee pops arguments from the
3449	// stack, we want to add it back if we have a reserved call frame.
3450	emitSPUpdate(isARM, MBB, MBBI&: I, dl, TII, NumBytes: -CalleePopAmount,
3451	MIFlags: MachineInstr::NoFlags, Pred, PredReg);
3452	}
3453	return MBB.erase(I);
3454	}
3455
3456	/// Get the minimum constant for ARM that is greater than or equal to the
3457	/// argument. In ARM, constants can have any value that can be produced by
3458	/// rotating an 8-bit value to the right by an even number of bits within a
3459	/// 32-bit word.
3460	static uint32_t alignToARMConstant(uint32_t Value) {
3461	unsigned Shifted = `0`;
3462
3463	if (Value == `0`)
3464	return `0`;
3465
3466	while (!(Value & `0xC0000000`)) {
3467	Value = Value << `2`;
3468	Shifted += `2`;
3469	}
3470
3471	bool Carry = (Value & `0x00FFFFFF`);
3472	Value = ((Value & `0xFF000000`) >> `24`) + Carry;
3473
3474	if (Value & `0x0000100`)
3475	Value = Value & `0x000001FC`;
3476
3477	if (Shifted > `24`)
3478	Value = Value >> (Shifted - `24`);
3479	else
3480	Value = Value << (`24` - Shifted);
3481
3482	return Value;
3483	}
3484
3485	// The stack limit in the TCB is set to this many bytes above the actual
3486	// stack limit.
3487	static const uint64_t kSplitStackAvailable = `256`;
3488
3489	// Adjust the function prologue to enable split stacks. This currently only
3490	// supports android and linux.
3491	//
3492	// The ABI of the segmented stack prologue is a little arbitrarily chosen, but
3493	// must be well defined in order to allow for consistent implementations of the
3494	// __morestack helper function. The ABI is also not a normal ABI in that it
3495	// doesn't follow the normal calling conventions because this allows the
3496	// prologue of each function to be optimized further.
3497	//
3498	// Currently, the ABI looks like (when calling __morestack)
3499	//
3500	// r4 holds the minimum stack size requested for this function call*
3501	// r5 holds the stack size of the arguments to the function*
3502	// the beginning of the function is 3 instructions after the call to*
3503	// __morestack
3504	//
3505	// Implementations of __morestack should use r4 to allocate a new stack, r5 to
3506	// place the arguments on to the new stack, and the 3-instruction knowledge to
3507	// jump directly to the body of the function when working on the new stack.
3508	//
3509	// An old (and possibly no longer compatible) implementation of __morestack for
3510	// ARM can be found at [1].
3511	//
3512	// [1] - https://github.com/mozilla/rust/blob/86efd9/src/rt/arch/arm/morestack.S
3513	void ARMFrameLowering::adjustForSegmentedStacks(
3514	MachineFunction &MF, MachineBasicBlock &PrologueMBB) const {
3515	unsigned Opcode;
3516	const ARMSubtarget *ST = &MF.getSubtarget<ARMSubtarget>();
3517	bool Thumb = ST->isThumb();
3518	bool Thumb2 = ST->isThumb2();
3519
3520	// Sadly, this currently doesn't support varargs, platforms other than
3521	// android/linux. Note that thumb1/thumb2 are support for android/linux.
3522	if (MF.getFunction().isVarArg())
3523	report_fatal_error(reason: "Segmented stacks do not support vararg functions.");
3524	if (!ST->isTargetAndroid() && !ST->isTargetLinux())
3525	report_fatal_error(reason: "Segmented stacks not supported on this platform.");
3526
3527	MachineFrameInfo &MFI = MF.getFrameInfo();
3528	const ARMBaseInstrInfo &TII =
3529	*static_cast<const ARMBaseInstrInfo *>(MF.getSubtarget().getInstrInfo());
3530	ARMFunctionInfo *ARMFI = MF.getInfo<ARMFunctionInfo>();
3531	DebugLoc DL;
3532
3533	if (!MFI.needsSplitStackProlog())
3534	return;
3535
3536	uint64_t StackSize = MFI.getStackSize();
3537
3538	// Use R4 and R5 as scratch registers.
3539	// We save R4 and R5 before use and restore them before leaving the function.
3540	unsigned ScratchReg0 = ARM::R4;
3541	unsigned ScratchReg1 = ARM::R5;
3542	unsigned MovOp = ST->useMovt() ? ARM::t2MOVi32imm : ARM::tMOVi32imm;
3543	uint64_t AlignedStackSize;
3544
3545	MachineBasicBlock *PrevStackMBB = MF.CreateMachineBasicBlock();
3546	MachineBasicBlock *PostStackMBB = MF.CreateMachineBasicBlock();
3547	MachineBasicBlock *AllocMBB = MF.CreateMachineBasicBlock();
3548	MachineBasicBlock *GetMBB = MF.CreateMachineBasicBlock();
3549	MachineBasicBlock *McrMBB = MF.CreateMachineBasicBlock();
3550
3551	// Grab everything that reaches PrologueMBB to update there liveness as well.
3552	SmallPtrSet<MachineBasicBlock *, `8`> BeforePrologueRegion;
3553	SmallVector<MachineBasicBlock *, `2`> WalkList;
3554	WalkList.push_back(Elt: &PrologueMBB);
3555
3556	do {
3557	MachineBasicBlock *CurMBB = WalkList.pop_back_val();
3558	for (MachineBasicBlock *PredBB : CurMBB->predecessors()) {
3559	if (BeforePrologueRegion.insert(Ptr: PredBB).second)
3560	WalkList.push_back(Elt: PredBB);
3561	}
3562	} while (!WalkList.empty());
3563
3564	// The order in that list is important.
3565	// The blocks will all be inserted before PrologueMBB using that order.
3566	// Therefore the block that should appear first in the CFG should appear
3567	// first in the list.
3568	MachineBasicBlock *AddedBlocks[] = {PrevStackMBB, McrMBB, GetMBB, AllocMBB,
3569	PostStackMBB};
3570
3571	BeforePrologueRegion.insert_range(R&: AddedBlocks);
3572
3573	for (const auto &LI : PrologueMBB.liveins()) {
3574	for (MachineBasicBlock *PredBB : BeforePrologueRegion)
3575	PredBB->addLiveIn(RegMaskPair: LI);
3576	}
3577
3578	// Remove the newly added blocks from the list, since we know
3579	// we do not have to do the following updates for them.
3580	for (MachineBasicBlock *B : AddedBlocks) {
3581	BeforePrologueRegion.erase(Ptr: B);
3582	MF.insert(MBBI: PrologueMBB.getIterator(), MBB: B);
3583	}
3584
3585	for (MachineBasicBlock *MBB : BeforePrologueRegion) {
3586	// Make sure the LiveIns are still sorted and unique.
3587	MBB->sortUniqueLiveIns();
3588	// Replace the edges to PrologueMBB by edges to the sequences
3589	// we are about to add, but only update for immediate predecessors.
3590	if (MBB->isSuccessor(MBB: &PrologueMBB))
3591	MBB->ReplaceUsesOfBlockWith(Old: &PrologueMBB, New: AddedBlocks[`0`]);
3592	}
3593
3594	// The required stack size that is aligned to ARM constant criterion.
3595	AlignedStackSize = alignToARMConstant(Value: StackSize);
3596
3597	// When the frame size is less than 256 we just compare the stack
3598	// boundary directly to the value of the stack pointer, per gcc.
3599	bool CompareStackPointer = AlignedStackSize < kSplitStackAvailable;
3600
3601	// We will use two of the callee save registers as scratch registers so we
3602	// need to save those registers onto the stack.
3603	// We will use SR0 to hold stack limit and SR1 to hold the stack size
3604	// requested and arguments for __morestack().
3605	// SR0: Scratch Register #0
3606	// SR1: Scratch Register #1
3607	// push {SR0, SR1}
3608	if (Thumb) {
3609	BuildMI(BB: PrevStackMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::tPUSH))
3610	.add(MOs: predOps(Pred: ARMCC::AL))
3611	.addReg(RegNo: ScratchReg0)
3612	.addReg(RegNo: ScratchReg1);
3613	} else {
3614	BuildMI(BB: PrevStackMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::STMDB_UPD))
3615	.addReg(RegNo: ARM::SP, Flags: RegState::Define)
3616	.addReg(RegNo: ARM::SP)
3617	.add(MOs: predOps(Pred: ARMCC::AL))
3618	.addReg(RegNo: ScratchReg0)
3619	.addReg(RegNo: ScratchReg1);
3620	}
3621
3622	// Emit the relevant DWARF information about the change in stack pointer as
3623	// well as where to find both r4 and r5 (the callee-save registers)
3624	if (!MF.getTarget().getMCAsmInfo().usesWindowsCFI()) {
3625	CFIInstBuilder CFIBuilder(PrevStackMBB, MachineInstr::NoFlags);
3626	CFIBuilder.buildDefCFAOffset(Offset: `8`);
3627	CFIBuilder.buildOffset(Reg: ScratchReg1, Offset: -`4`);
3628	CFIBuilder.buildOffset(Reg: ScratchReg0, Offset: -`8`);
3629	}
3630
3631	// mov SR1, sp
3632	if (Thumb) {
3633	BuildMI(BB: McrMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::tMOVr), DestReg: ScratchReg1)
3634	.addReg(RegNo: ARM::SP)
3635	.add(MOs: predOps(Pred: ARMCC::AL));
3636	} else if (CompareStackPointer) {
3637	BuildMI(BB: McrMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::MOVr), DestReg: ScratchReg1)
3638	.addReg(RegNo: ARM::SP)
3639	.add(MOs: predOps(Pred: ARMCC::AL))
3640	.add(MO: condCodeOp());
3641	}
3642
3643	// sub SR1, sp, #StackSize
3644	if (!CompareStackPointer && Thumb) {
3645	if (AlignedStackSize < `256`) {
3646	BuildMI(BB: McrMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::tSUBi8), DestReg: ScratchReg1)
3647	.add(MO: condCodeOp())
3648	.addReg(RegNo: ScratchReg1)
3649	.addImm(Val: AlignedStackSize)
3650	.add(MOs: predOps(Pred: ARMCC::AL));
3651	} else {
3652	if (Thumb2 \|\| ST->genExecuteOnly()) {
3653	BuildMI(BB: McrMBB, MIMD: DL, MCID: TII.get(Opcode: MovOp), DestReg: ScratchReg0)
3654	.addImm(Val: AlignedStackSize);
3655	} else {
3656	auto MBBI = McrMBB->end();
3657	auto RegInfo = STI.getRegisterInfo();
3658	RegInfo->emitLoadConstPool(MBB&: *McrMBB, MBBI, dl: DL, DestReg: ScratchReg0, SubIdx: `0`,
3659	Val: AlignedStackSize);
3660	}
3661	BuildMI(BB: McrMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::tSUBrr), DestReg: ScratchReg1)
3662	.add(MO: condCodeOp())
3663	.addReg(RegNo: ScratchReg1)
3664	.addReg(RegNo: ScratchReg0)
3665	.add(MOs: predOps(Pred: ARMCC::AL));
3666	}
3667	} else if (!CompareStackPointer) {
3668	if (AlignedStackSize < `256`) {
3669	BuildMI(BB: McrMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::SUBri), DestReg: ScratchReg1)
3670	.addReg(RegNo: ARM::SP)
3671	.addImm(Val: AlignedStackSize)
3672	.add(MOs: predOps(Pred: ARMCC::AL))
3673	.add(MO: condCodeOp());
3674	} else {
3675	auto MBBI = McrMBB->end();
3676	auto RegInfo = STI.getRegisterInfo();
3677	RegInfo->emitLoadConstPool(MBB&: *McrMBB, MBBI, dl: DL, DestReg: ScratchReg0, SubIdx: `0`,
3678	Val: AlignedStackSize);
3679	BuildMI(BB: McrMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::SUBrr), DestReg: ScratchReg1)
3680	.addReg(RegNo: ARM::SP)
3681	.addReg(RegNo: ScratchReg0)
3682	.add(MOs: predOps(Pred: ARMCC::AL))
3683	.add(MO: condCodeOp());
3684	}
3685	}
3686
3687	if (Thumb && ST->isThumb1Only()) {
3688	if (ST->genExecuteOnly()) {
3689	BuildMI(BB: GetMBB, MIMD: DL, MCID: TII.get(Opcode: MovOp), DestReg: ScratchReg0)
3690	.addExternalSymbol(FnName: "__STACK_LIMIT");
3691	} else {
3692	unsigned PCLabelId = ARMFI->createPICLabelUId();
3693	ARMConstantPoolValue *NewCPV = ARMConstantPoolSymbol::Create(
3694	C&: MF.getFunction().getContext(), s: "__STACK_LIMIT", ID: PCLabelId, PCAdj: `0`);
3695	MachineConstantPool *MCP = MF.getConstantPool();
3696	unsigned CPI = MCP->getConstantPoolIndex(V: NewCPV, Alignment: Align (`4`));
3697
3698	// ldr SR0, [pc, offset(STACK_LIMIT)]
3699	BuildMI(BB: GetMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::tLDRpci), DestReg: ScratchReg0)
3700	.addConstantPoolIndex(Idx: CPI)
3701	.add(MOs: predOps(Pred: ARMCC::AL));
3702	}
3703
3704	// ldr SR0, [SR0]
3705	BuildMI(BB: GetMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::tLDRi), DestReg: ScratchReg0)
3706	.addReg(RegNo: ScratchReg0)
3707	.addImm(Val: `0`)
3708	.add(MOs: predOps(Pred: ARMCC::AL));
3709	} else {
3710	// Get TLS base address from the coprocessor
3711	// mrc p15, #0, SR0, c13, c0, #3
3712	BuildMI(BB: McrMBB, MIMD: DL, MCID: TII.get(Opcode: Thumb ? ARM::t2MRC : ARM::MRC),
3713	DestReg: ScratchReg0)
3714	.addImm(Val: `15`)
3715	.addImm(Val: `0`)
3716	.addImm(Val: `13`)
3717	.addImm(Val: `0`)
3718	.addImm(Val: `3`)
3719	.add(MOs: predOps(Pred: ARMCC::AL));
3720
3721	// Use the last tls slot on android and a private field of the TCP on linux.
3722	assert(ST->isTargetAndroid() \|\| ST->isTargetLinux());
3723	unsigned TlsOffset = ST->isTargetAndroid() ? `63` : `1`;
3724
3725	// Get the stack limit from the right offset
3726	// ldr SR0, [sr0, #4 TlsOffset]*
3727	BuildMI(BB: GetMBB, MIMD: DL, MCID: TII.get(Opcode: Thumb ? ARM::t2LDRi12 : ARM::LDRi12),
3728	DestReg: ScratchReg0)
3729	.addReg(RegNo: ScratchReg0)
3730	.addImm(Val: `4` * TlsOffset)
3731	.add(MOs: predOps(Pred: ARMCC::AL));
3732	}
3733
3734	// Compare stack limit with stack size requested.
3735	// cmp SR0, SR1
3736	Opcode = Thumb ? ARM::tCMPr : ARM::CMPrr;
3737	BuildMI(BB: GetMBB, MIMD: DL, MCID: TII.get(Opcode))
3738	.addReg(RegNo: ScratchReg0)
3739	.addReg(RegNo: ScratchReg1)
3740	.add(MOs: predOps(Pred: ARMCC::AL));
3741
3742	// This jump is taken if StackLimit <= SP - stack required.
3743	Opcode = Thumb ? ARM::tBcc : ARM::Bcc;
3744	BuildMI(BB: GetMBB, MIMD: DL, MCID: TII.get(Opcode))
3745	.addMBB(MBB: PostStackMBB)
3746	.addImm(Val: ARMCC::LS)
3747	.addReg(RegNo: ARM::CPSR);
3748
3749	// Calling __morestack(StackSize, Size of stack arguments).
3750	// __morestack knows that the stack size requested is in SR0(r4)
3751	// and amount size of stack arguments is in SR1(r5).
3752
3753	// Pass first argument for the __morestack by Scratch Register #0.
3754	// The amount size of stack required
3755	if (Thumb) {
3756	if (AlignedStackSize < `256`) {
3757	BuildMI(BB: AllocMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::tMOVi8), DestReg: ScratchReg0)
3758	.add(MO: condCodeOp())
3759	.addImm(Val: AlignedStackSize)
3760	.add(MOs: predOps(Pred: ARMCC::AL));
3761	} else {
3762	if (Thumb2 \|\| ST->genExecuteOnly()) {
3763	BuildMI(BB: AllocMBB, MIMD: DL, MCID: TII.get(Opcode: MovOp), DestReg: ScratchReg0)
3764	.addImm(Val: AlignedStackSize);
3765	} else {
3766	auto MBBI = AllocMBB->end();
3767	auto RegInfo = STI.getRegisterInfo();
3768	RegInfo->emitLoadConstPool(MBB&: *AllocMBB, MBBI, dl: DL, DestReg: ScratchReg0, SubIdx: `0`,
3769	Val: AlignedStackSize);
3770	}
3771	}
3772	} else {
3773	if (AlignedStackSize < `256`) {
3774	BuildMI(BB: AllocMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::MOVi), DestReg: ScratchReg0)
3775	.addImm(Val: AlignedStackSize)
3776	.add(MOs: predOps(Pred: ARMCC::AL))
3777	.add(MO: condCodeOp());
3778	} else {
3779	auto MBBI = AllocMBB->end();
3780	auto RegInfo = STI.getRegisterInfo();
3781	RegInfo->emitLoadConstPool(MBB&: *AllocMBB, MBBI, dl: DL, DestReg: ScratchReg0, SubIdx: `0`,
3782	Val: AlignedStackSize);
3783	}
3784	}
3785
3786	// Pass second argument for the __morestack by Scratch Register #1.
3787	// The amount size of stack consumed to save function arguments.
3788	if (Thumb) {
3789	if (ARMFI->getArgumentStackSize() < `256`) {
3790	BuildMI(BB: AllocMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::tMOVi8), DestReg: ScratchReg1)
3791	.add(MO: condCodeOp())
3792	.addImm(Val: alignToARMConstant(Value: ARMFI->getArgumentStackSize()))
3793	.add(MOs: predOps(Pred: ARMCC::AL));
3794	} else {
3795	if (Thumb2 \|\| ST->genExecuteOnly()) {
3796	BuildMI(BB: AllocMBB, MIMD: DL, MCID: TII.get(Opcode: MovOp), DestReg: ScratchReg1)
3797	.addImm(Val: alignToARMConstant(Value: ARMFI->getArgumentStackSize()));
3798	} else {
3799	auto MBBI = AllocMBB->end();
3800	auto RegInfo = STI.getRegisterInfo();
3801	RegInfo->emitLoadConstPool(
3802	MBB&: *AllocMBB, MBBI, dl: DL, DestReg: ScratchReg1, SubIdx: `0`,
3803	Val: alignToARMConstant(Value: ARMFI->getArgumentStackSize()));
3804	}
3805	}
3806	} else {
3807	if (alignToARMConstant(Value: ARMFI->getArgumentStackSize()) < `256`) {
3808	BuildMI(BB: AllocMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::MOVi), DestReg: ScratchReg1)
3809	.addImm(Val: alignToARMConstant(Value: ARMFI->getArgumentStackSize()))
3810	.add(MOs: predOps(Pred: ARMCC::AL))
3811	.add(MO: condCodeOp());
3812	} else {
3813	auto MBBI = AllocMBB->end();
3814	auto RegInfo = STI.getRegisterInfo();
3815	RegInfo->emitLoadConstPool(
3816	MBB&: *AllocMBB, MBBI, dl: DL, DestReg: ScratchReg1, SubIdx: `0`,
3817	Val: alignToARMConstant(Value: ARMFI->getArgumentStackSize()));
3818	}
3819	}
3820
3821	// push {lr} - Save return address of this function.
3822	if (Thumb) {
3823	BuildMI(BB: AllocMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::tPUSH))
3824	.add(MOs: predOps(Pred: ARMCC::AL))
3825	.addReg(RegNo: ARM::LR);
3826	} else {
3827	BuildMI(BB: AllocMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::STMDB_UPD))
3828	.addReg(RegNo: ARM::SP, Flags: RegState::Define)
3829	.addReg(RegNo: ARM::SP)
3830	.add(MOs: predOps(Pred: ARMCC::AL))
3831	.addReg(RegNo: ARM::LR);
3832	}
3833
3834	// Emit the DWARF info about the change in stack as well as where to find the
3835	// previous link register
3836	if (!MF.getTarget().getMCAsmInfo().usesWindowsCFI()) {
3837	CFIInstBuilder CFIBuilder(AllocMBB, MachineInstr::NoFlags);
3838	CFIBuilder.buildDefCFAOffset(Offset: `12`);
3839	CFIBuilder.buildOffset(Reg: ARM::LR, Offset: -`12`);
3840	}
3841
3842	// Call __morestack().
3843	if (Thumb) {
3844	BuildMI(BB: AllocMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::tBL))
3845	.add(MOs: predOps(Pred: ARMCC::AL))
3846	.addExternalSymbol(FnName: "__morestack");
3847	} else {
3848	BuildMI(BB: AllocMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::BL))
3849	.addExternalSymbol(FnName: "__morestack");
3850	}
3851
3852	// pop {lr} - Restore return address of this original function.
3853	if (Thumb) {
3854	if (ST->isThumb1Only()) {
3855	BuildMI(BB: AllocMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::tPOP))
3856	.add(MOs: predOps(Pred: ARMCC::AL))
3857	.addReg(RegNo: ScratchReg0);
3858	BuildMI(BB: AllocMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::tMOVr), DestReg: ARM::LR)
3859	.addReg(RegNo: ScratchReg0)
3860	.add(MOs: predOps(Pred: ARMCC::AL));
3861	} else {
3862	BuildMI(BB: AllocMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::t2LDR_POST))
3863	.addReg(RegNo: ARM::LR, Flags: RegState::Define)
3864	.addReg(RegNo: ARM::SP, Flags: RegState::Define)
3865	.addReg(RegNo: ARM::SP)
3866	.addImm(Val: `4`)
3867	.add(MOs: predOps(Pred: ARMCC::AL));
3868	}
3869	} else {
3870	BuildMI(BB: AllocMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::LDMIA_UPD))
3871	.addReg(RegNo: ARM::SP, Flags: RegState::Define)
3872	.addReg(RegNo: ARM::SP)
3873	.add(MOs: predOps(Pred: ARMCC::AL))
3874	.addReg(RegNo: ARM::LR);
3875	}
3876
3877	// Restore SR0 and SR1 in case of __morestack() was called.
3878	// __morestack() will skip PostStackMBB block so we need to restore
3879	// scratch registers from here.
3880	// pop {SR0, SR1}
3881	if (Thumb) {
3882	BuildMI(BB: AllocMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::tPOP))
3883	.add(MOs: predOps(Pred: ARMCC::AL))
3884	.addReg(RegNo: ScratchReg0)
3885	.addReg(RegNo: ScratchReg1);
3886	} else {
3887	BuildMI(BB: AllocMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::LDMIA_UPD))
3888	.addReg(RegNo: ARM::SP, Flags: RegState::Define)
3889	.addReg(RegNo: ARM::SP)
3890	.add(MOs: predOps(Pred: ARMCC::AL))
3891	.addReg(RegNo: ScratchReg0)
3892	.addReg(RegNo: ScratchReg1);
3893	}
3894
3895	// Update the CFA offset now that we've popped
3896	if (!MF.getTarget().getMCAsmInfo().usesWindowsCFI())
3897	CFIInstBuilder (AllocMBB, MachineInstr::NoFlags).buildDefCFAOffset(Offset: `0`);
3898
3899	// Return from this function.
3900	BuildMI(BB: AllocMBB, MIMD: DL, MCID: TII.get(Opcode: ST->getReturnOpcode())).add(MOs: predOps(Pred: ARMCC::AL));
3901
3902	// Restore SR0 and SR1 in case of __morestack() was not called.
3903	// pop {SR0, SR1}
3904	if (Thumb) {
3905	BuildMI(BB: PostStackMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::tPOP))
3906	.add(MOs: predOps(Pred: ARMCC::AL))
3907	.addReg(RegNo: ScratchReg0)
3908	.addReg(RegNo: ScratchReg1);
3909	} else {
3910	BuildMI(BB: PostStackMBB, MIMD: DL, MCID: TII.get(Opcode: ARM::LDMIA_UPD))
3911	.addReg(RegNo: ARM::SP, Flags: RegState::Define)
3912	.addReg(RegNo: ARM::SP)
3913	.add(MOs: predOps(Pred: ARMCC::AL))
3914	.addReg(RegNo: ScratchReg0)
3915	.addReg(RegNo: ScratchReg1);
3916	}
3917
3918	// Update the CFA offset now that we've popped
3919	if (!MF.getTarget().getMCAsmInfo().usesWindowsCFI()) {
3920	CFIInstBuilder CFIBuilder(PostStackMBB, MachineInstr::NoFlags);
3921	CFIBuilder.buildDefCFAOffset(Offset: `0`);
3922
3923	// Tell debuggers that r4 and r5 are now the same as they were in the
3924	// previous function, that they're the "Same Value".
3925	CFIBuilder.buildSameValue(Reg: ScratchReg0);
3926	CFIBuilder.buildSameValue(Reg: ScratchReg1);
3927	}
3928
3929	// Organizing MBB lists
3930	PostStackMBB->addSuccessor(Succ: &PrologueMBB);
3931
3932	AllocMBB->addSuccessor(Succ: PostStackMBB);
3933
3934	GetMBB->addSuccessor(Succ: PostStackMBB);
3935	GetMBB->addSuccessor(Succ: AllocMBB);
3936
3937	McrMBB->addSuccessor(Succ: GetMBB);
3938
3939	PrevStackMBB->addSuccessor(Succ: McrMBB);
3940
3941	#ifdef EXPENSIVE_CHECKS
3942	MF.verify();
3943	#endif
3944	}
3945

Browse the source code of llvm_projects/llvm/lib/Target/ARM/ARMFrameLowering.cpp