1//===- AArch64FrameLowering.cpp - AArch64 Frame Lowering -------*- C++ -*-====//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the AArch64 implementation of TargetFrameLowering class.
10//
11// On AArch64, stack frames are structured as follows:
12//
13// The stack grows downward.
14//
15// All of the individual frame areas on the frame below are optional, i.e. it's
16// possible to create a function so that the particular area isn't present
17// in the frame.
18//
19// At function entry, the "frame" looks as follows:
20//
21// | | Higher address
22// |-----------------------------------|
23// | |
24// | arguments passed on the stack |
25// | |
26// |-----------------------------------| <- sp
27// | | Lower address
28//
29//
30// After the prologue has run, the frame has the following general structure.
31// Note that this doesn't depict the case where a red-zone is used. Also,
32// technically the last frame area (VLAs) doesn't get created until in the
33// main function body, after the prologue is run. However, it's depicted here
34// for completeness.
35//
36// | | Higher address
37// |-----------------------------------|
38// | |
39// | arguments passed on the stack |
40// | |
41// |-----------------------------------|
42// | |
43// | (Win64 only) varargs from reg |
44// | |
45// |-----------------------------------|
46// | |
47// | callee-saved gpr registers | <--.
48// | | | On Darwin platforms these
49// |- - - - - - - - - - - - - - - - - -| | callee saves are swapped,
50// | prev_lr | | (frame record first)
51// | prev_fp | <--'
52// | async context if needed |
53// | (a.k.a. "frame record") |
54// |-----------------------------------| <- fp(=x29)
55// | <hazard padding> |
56// |-----------------------------------|
57// | |
58// | callee-saved fp/simd/SVE regs |
59// | |
60// |-----------------------------------|
61// | |
62// | SVE stack objects |
63// | |
64// |-----------------------------------|
65// |.empty.space.to.make.part.below....|
66// |.aligned.in.case.it.needs.more.than| (size of this area is unknown at
67// |.the.standard.16-byte.alignment....| compile time; if present)
68// |-----------------------------------|
69// | local variables of fixed size |
70// | including spill slots |
71// | <FPR> |
72// | <hazard padding> |
73// | <GPR> |
74// |-----------------------------------| <- bp(not defined by ABI,
75// |.variable-sized.local.variables....| LLVM chooses X19)
76// |.(VLAs)............................| (size of this area is unknown at
77// |...................................| compile time)
78// |-----------------------------------| <- sp
79// | | Lower address
80//
81//
82// To access the data in a frame, at-compile time, a constant offset must be
83// computable from one of the pointers (fp, bp, sp) to access it. The size
84// of the areas with a dotted background cannot be computed at compile-time
85// if they are present, making it required to have all three of fp, bp and
86// sp to be set up to be able to access all contents in the frame areas,
87// assuming all of the frame areas are non-empty.
88//
89// For most functions, some of the frame areas are empty. For those functions,
90// it may not be necessary to set up fp or bp:
91// * A base pointer is definitely needed when there are both VLAs and local
92// variables with more-than-default alignment requirements.
93// * A frame pointer is definitely needed when there are local variables with
94// more-than-default alignment requirements.
95//
96// For Darwin platforms the frame-record (fp, lr) is stored at the top of the
97// callee-saved area, since the unwind encoding does not allow for encoding
98// this dynamically and existing tools depend on this layout. For other
99// platforms, the frame-record is stored at the bottom of the (gpr) callee-saved
100// area to allow SVE stack objects (allocated directly below the callee-saves,
101// if available) to be accessed directly from the framepointer.
102// The SVE spill/fill instructions have VL-scaled addressing modes such
103// as:
104// ldr z8, [fp, #-7 mul vl]
105// For SVE the size of the vector length (VL) is not known at compile-time, so
106// '#-7 mul vl' is an offset that can only be evaluated at runtime. With this
107// layout, we don't need to add an unscaled offset to the framepointer before
108// accessing the SVE object in the frame.
109//
110// In some cases when a base pointer is not strictly needed, it is generated
111// anyway when offsets from the frame pointer to access local variables become
112// so large that the offset can't be encoded in the immediate fields of loads
113// or stores.
114//
115// Outgoing function arguments must be at the bottom of the stack frame when
116// calling another function. If we do not have variable-sized stack objects, we
117// can allocate a "reserved call frame" area at the bottom of the local
118// variable area, large enough for all outgoing calls. If we do have VLAs, then
119// the stack pointer must be decremented and incremented around each call to
120// make space for the arguments below the VLAs.
121//
122// FIXME: also explain the redzone concept.
123//
124// About stack hazards: Under some SME contexts, a coprocessor with its own
125// separate cache can used for FP operations. This can create hazards if the CPU
126// and the SME unit try to access the same area of memory, including if the
127// access is to an area of the stack. To try to alleviate this we attempt to
128// introduce extra padding into the stack frame between FP and GPR accesses,
129// controlled by the StackHazardSize option. Without changing the layout of the
130// stack frame in the diagram above, a stack object of size StackHazardSize is
131// added between GPR and FPR CSRs. Another is added to the stack objects
132// section, and stack objects are sorted so that FPR > Hazard padding slot >
133// GPRs (where possible). Unfortunately some things are not handled well (VLA
134// area, arguments on the stack, object with both GPR and FPR accesses), but if
135// those are controlled by the user then the entire stack frame becomes GPR at
136// the start/end with FPR in the middle, surrounded by Hazard padding.
137//
138// An example of the prologue:
139//
140// .globl __foo
141// .align 2
142// __foo:
143// Ltmp0:
144// .cfi_startproc
145// .cfi_personality 155, ___gxx_personality_v0
146// Leh_func_begin:
147// .cfi_lsda 16, Lexception33
148//
149// stp xa,bx, [sp, -#offset]!
150// ...
151// stp x28, x27, [sp, #offset-32]
152// stp fp, lr, [sp, #offset-16]
153// add fp, sp, #offset - 16
154// sub sp, sp, #1360
155//
156// The Stack:
157// +-------------------------------------------+
158// 10000 | ........ | ........ | ........ | ........ |
159// 10004 | ........ | ........ | ........ | ........ |
160// +-------------------------------------------+
161// 10008 | ........ | ........ | ........ | ........ |
162// 1000c | ........ | ........ | ........ | ........ |
163// +===========================================+
164// 10010 | X28 Register |
165// 10014 | X28 Register |
166// +-------------------------------------------+
167// 10018 | X27 Register |
168// 1001c | X27 Register |
169// +===========================================+
170// 10020 | Frame Pointer |
171// 10024 | Frame Pointer |
172// +-------------------------------------------+
173// 10028 | Link Register |
174// 1002c | Link Register |
175// +===========================================+
176// 10030 | ........ | ........ | ........ | ........ |
177// 10034 | ........ | ........ | ........ | ........ |
178// +-------------------------------------------+
179// 10038 | ........ | ........ | ........ | ........ |
180// 1003c | ........ | ........ | ........ | ........ |
181// +-------------------------------------------+
182//
183// [sp] = 10030 :: >>initial value<<
184// sp = 10020 :: stp fp, lr, [sp, #-16]!
185// fp = sp == 10020 :: mov fp, sp
186// [sp] == 10020 :: stp x28, x27, [sp, #-16]!
187// sp == 10010 :: >>final value<<
188//
189// The frame pointer (w29) points to address 10020. If we use an offset of
190// '16' from 'w29', we get the CFI offsets of -8 for w30, -16 for w29, -24
191// for w27, and -32 for w28:
192//
193// Ltmp1:
194// .cfi_def_cfa w29, 16
195// Ltmp2:
196// .cfi_offset w30, -8
197// Ltmp3:
198// .cfi_offset w29, -16
199// Ltmp4:
200// .cfi_offset w27, -24
201// Ltmp5:
202// .cfi_offset w28, -32
203//
204//===----------------------------------------------------------------------===//
205
206#include "AArch64FrameLowering.h"
207#include "AArch64InstrInfo.h"
208#include "AArch64MachineFunctionInfo.h"
209#include "AArch64RegisterInfo.h"
210#include "AArch64Subtarget.h"
211#include "AArch64TargetMachine.h"
212#include "MCTargetDesc/AArch64AddressingModes.h"
213#include "MCTargetDesc/AArch64MCTargetDesc.h"
214#include "llvm/ADT/ScopeExit.h"
215#include "llvm/ADT/SmallVector.h"
216#include "llvm/ADT/Statistic.h"
217#include "llvm/Analysis/ValueTracking.h"
218#include "llvm/CodeGen/LivePhysRegs.h"
219#include "llvm/CodeGen/MachineBasicBlock.h"
220#include "llvm/CodeGen/MachineFrameInfo.h"
221#include "llvm/CodeGen/MachineFunction.h"
222#include "llvm/CodeGen/MachineInstr.h"
223#include "llvm/CodeGen/MachineInstrBuilder.h"
224#include "llvm/CodeGen/MachineMemOperand.h"
225#include "llvm/CodeGen/MachineModuleInfo.h"
226#include "llvm/CodeGen/MachineOperand.h"
227#include "llvm/CodeGen/MachineRegisterInfo.h"
228#include "llvm/CodeGen/RegisterScavenging.h"
229#include "llvm/CodeGen/TargetInstrInfo.h"
230#include "llvm/CodeGen/TargetRegisterInfo.h"
231#include "llvm/CodeGen/TargetSubtargetInfo.h"
232#include "llvm/CodeGen/WinEHFuncInfo.h"
233#include "llvm/IR/Attributes.h"
234#include "llvm/IR/CallingConv.h"
235#include "llvm/IR/DataLayout.h"
236#include "llvm/IR/DebugLoc.h"
237#include "llvm/IR/Function.h"
238#include "llvm/MC/MCAsmInfo.h"
239#include "llvm/MC/MCDwarf.h"
240#include "llvm/Support/CommandLine.h"
241#include "llvm/Support/Debug.h"
242#include "llvm/Support/ErrorHandling.h"
243#include "llvm/Support/FormatVariadic.h"
244#include "llvm/Support/MathExtras.h"
245#include "llvm/Support/raw_ostream.h"
246#include "llvm/Target/TargetMachine.h"
247#include "llvm/Target/TargetOptions.h"
248#include <cassert>
249#include <cstdint>
250#include <iterator>
251#include <optional>
252#include <vector>
253
254using namespace llvm;
255
256#define DEBUG_TYPE "frame-info"
257
258static cl::opt<bool> EnableRedZone("aarch64-redzone",
259 cl::desc("enable use of redzone on AArch64"),
260 cl::init(Val: false), cl::Hidden);
261
262static cl::opt<bool> StackTaggingMergeSetTag(
263 "stack-tagging-merge-settag",
264 cl::desc("merge settag instruction in function epilog"), cl::init(Val: true),
265 cl::Hidden);
266
267static cl::opt<bool> OrderFrameObjects("aarch64-order-frame-objects",
268 cl::desc("sort stack allocations"),
269 cl::init(Val: true), cl::Hidden);
270
271cl::opt<bool> EnableHomogeneousPrologEpilog(
272 "homogeneous-prolog-epilog", cl::Hidden,
273 cl::desc("Emit homogeneous prologue and epilogue for the size "
274 "optimization (default = off)"));
275
276// Stack hazard padding size. 0 = disabled.
277static cl::opt<unsigned> StackHazardSize("aarch64-stack-hazard-size",
278 cl::init(Val: 0), cl::Hidden);
279// Stack hazard size for analysis remarks. StackHazardSize takes precedence.
280static cl::opt<unsigned>
281 StackHazardRemarkSize("aarch64-stack-hazard-remark-size", cl::init(Val: 0),
282 cl::Hidden);
283// Whether to insert padding into non-streaming functions (for testing).
284static cl::opt<bool>
285 StackHazardInNonStreaming("aarch64-stack-hazard-in-non-streaming",
286 cl::init(Val: false), cl::Hidden);
287
288STATISTIC(NumRedZoneFunctions, "Number of functions using red zone");
289
290/// Returns how much of the incoming argument stack area (in bytes) we should
291/// clean up in an epilogue. For the C calling convention this will be 0, for
292/// guaranteed tail call conventions it can be positive (a normal return or a
293/// tail call to a function that uses less stack space for arguments) or
294/// negative (for a tail call to a function that needs more stack space than us
295/// for arguments).
296static int64_t getArgumentStackToRestore(MachineFunction &MF,
297 MachineBasicBlock &MBB) {
298 MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
299 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
300 bool IsTailCallReturn = (MBB.end() != MBBI)
301 ? AArch64InstrInfo::isTailCallReturnInst(MI: *MBBI)
302 : false;
303
304 int64_t ArgumentPopSize = 0;
305 if (IsTailCallReturn) {
306 MachineOperand &StackAdjust = MBBI->getOperand(i: 1);
307
308 // For a tail-call in a callee-pops-arguments environment, some or all of
309 // the stack may actually be in use for the call's arguments, this is
310 // calculated during LowerCall and consumed here...
311 ArgumentPopSize = StackAdjust.getImm();
312 } else {
313 // ... otherwise the amount to pop is *all* of the argument space,
314 // conveniently stored in the MachineFunctionInfo by
315 // LowerFormalArguments. This will, of course, be zero for the C calling
316 // convention.
317 ArgumentPopSize = AFI->getArgumentStackToRestore();
318 }
319
320 return ArgumentPopSize;
321}
322
323static bool produceCompactUnwindFrame(MachineFunction &MF);
324static bool needsWinCFI(const MachineFunction &MF);
325static StackOffset getSVEStackSize(const MachineFunction &MF);
326static Register findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB);
327
328/// Returns true if a homogeneous prolog or epilog code can be emitted
329/// for the size optimization. If possible, a frame helper call is injected.
330/// When Exit block is given, this check is for epilog.
331bool AArch64FrameLowering::homogeneousPrologEpilog(
332 MachineFunction &MF, MachineBasicBlock *Exit) const {
333 if (!MF.getFunction().hasMinSize())
334 return false;
335 if (!EnableHomogeneousPrologEpilog)
336 return false;
337 if (EnableRedZone)
338 return false;
339
340 // TODO: Window is supported yet.
341 if (needsWinCFI(MF))
342 return false;
343 // TODO: SVE is not supported yet.
344 if (getSVEStackSize(MF))
345 return false;
346
347 // Bail on stack adjustment needed on return for simplicity.
348 const MachineFrameInfo &MFI = MF.getFrameInfo();
349 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
350 if (MFI.hasVarSizedObjects() || RegInfo->hasStackRealignment(MF))
351 return false;
352 if (Exit && getArgumentStackToRestore(MF, MBB&: *Exit))
353 return false;
354
355 auto *AFI = MF.getInfo<AArch64FunctionInfo>();
356 if (AFI->hasSwiftAsyncContext() || AFI->hasStreamingModeChanges())
357 return false;
358
359 // If there are an odd number of GPRs before LR and FP in the CSRs list,
360 // they will not be paired into one RegPairInfo, which is incompatible with
361 // the assumption made by the homogeneous prolog epilog pass.
362 const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
363 unsigned NumGPRs = 0;
364 for (unsigned I = 0; CSRegs[I]; ++I) {
365 Register Reg = CSRegs[I];
366 if (Reg == AArch64::LR) {
367 assert(CSRegs[I + 1] == AArch64::FP);
368 if (NumGPRs % 2 != 0)
369 return false;
370 break;
371 }
372 if (AArch64::GPR64RegClass.contains(Reg))
373 ++NumGPRs;
374 }
375
376 return true;
377}
378
379/// Returns true if CSRs should be paired.
380bool AArch64FrameLowering::producePairRegisters(MachineFunction &MF) const {
381 return produceCompactUnwindFrame(MF) || homogeneousPrologEpilog(MF);
382}
383
384/// This is the biggest offset to the stack pointer we can encode in aarch64
385/// instructions (without using a separate calculation and a temp register).
386/// Note that the exception here are vector stores/loads which cannot encode any
387/// displacements (see estimateRSStackSizeLimit(), isAArch64FrameOffsetLegal()).
388static const unsigned DefaultSafeSPDisplacement = 255;
389
390/// Look at each instruction that references stack frames and return the stack
391/// size limit beyond which some of these instructions will require a scratch
392/// register during their expansion later.
393static unsigned estimateRSStackSizeLimit(MachineFunction &MF) {
394 // FIXME: For now, just conservatively guestimate based on unscaled indexing
395 // range. We'll end up allocating an unnecessary spill slot a lot, but
396 // realistically that's not a big deal at this stage of the game.
397 for (MachineBasicBlock &MBB : MF) {
398 for (MachineInstr &MI : MBB) {
399 if (MI.isDebugInstr() || MI.isPseudo() ||
400 MI.getOpcode() == AArch64::ADDXri ||
401 MI.getOpcode() == AArch64::ADDSXri)
402 continue;
403
404 for (const MachineOperand &MO : MI.operands()) {
405 if (!MO.isFI())
406 continue;
407
408 StackOffset Offset;
409 if (isAArch64FrameOffsetLegal(MI, Offset, OutUseUnscaledOp: nullptr, OutUnscaledOp: nullptr, EmittableOffset: nullptr) ==
410 AArch64FrameOffsetCannotUpdate)
411 return 0;
412 }
413 }
414 }
415 return DefaultSafeSPDisplacement;
416}
417
418TargetStackID::Value
419AArch64FrameLowering::getStackIDForScalableVectors() const {
420 return TargetStackID::ScalableVector;
421}
422
423/// Returns the size of the fixed object area (allocated next to sp on entry)
424/// On Win64 this may include a var args area and an UnwindHelp object for EH.
425static unsigned getFixedObjectSize(const MachineFunction &MF,
426 const AArch64FunctionInfo *AFI, bool IsWin64,
427 bool IsFunclet) {
428 if (!IsWin64 || IsFunclet) {
429 return AFI->getTailCallReservedStack();
430 } else {
431 if (AFI->getTailCallReservedStack() != 0 &&
432 !MF.getFunction().getAttributes().hasAttrSomewhere(
433 Kind: Attribute::SwiftAsync))
434 report_fatal_error(reason: "cannot generate ABI-changing tail call for Win64");
435 // Var args are stored here in the primary function.
436 const unsigned VarArgsArea = AFI->getVarArgsGPRSize();
437 // To support EH funclets we allocate an UnwindHelp object
438 const unsigned UnwindHelpObject = (MF.hasEHFunclets() ? 8 : 0);
439 return AFI->getTailCallReservedStack() +
440 alignTo(Value: VarArgsArea + UnwindHelpObject, Align: 16);
441 }
442}
443
444/// Returns the size of the entire SVE stackframe (calleesaves + spills).
445static StackOffset getSVEStackSize(const MachineFunction &MF) {
446 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
447 return StackOffset::getScalable(Scalable: (int64_t)AFI->getStackSizeSVE());
448}
449
450bool AArch64FrameLowering::canUseRedZone(const MachineFunction &MF) const {
451 if (!EnableRedZone)
452 return false;
453
454 // Don't use the red zone if the function explicitly asks us not to.
455 // This is typically used for kernel code.
456 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
457 const unsigned RedZoneSize =
458 Subtarget.getTargetLowering()->getRedZoneSize(F: MF.getFunction());
459 if (!RedZoneSize)
460 return false;
461
462 const MachineFrameInfo &MFI = MF.getFrameInfo();
463 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
464 uint64_t NumBytes = AFI->getLocalStackSize();
465
466 // If neither NEON or SVE are available, a COPY from one Q-reg to
467 // another requires a spill -> reload sequence. We can do that
468 // using a pre-decrementing store/post-decrementing load, but
469 // if we do so, we can't use the Red Zone.
470 bool LowerQRegCopyThroughMem = Subtarget.hasFPARMv8() &&
471 !Subtarget.isNeonAvailable() &&
472 !Subtarget.hasSVE();
473
474 return !(MFI.hasCalls() || hasFP(MF) || NumBytes > RedZoneSize ||
475 getSVEStackSize(MF) || LowerQRegCopyThroughMem);
476}
477
478/// hasFP - Return true if the specified function should have a dedicated frame
479/// pointer register.
480bool AArch64FrameLowering::hasFP(const MachineFunction &MF) const {
481 const MachineFrameInfo &MFI = MF.getFrameInfo();
482 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
483
484 // Win64 EH requires a frame pointer if funclets are present, as the locals
485 // are accessed off the frame pointer in both the parent function and the
486 // funclets.
487 if (MF.hasEHFunclets())
488 return true;
489 // Retain behavior of always omitting the FP for leaf functions when possible.
490 if (MF.getTarget().Options.DisableFramePointerElim(MF))
491 return true;
492 if (MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken() ||
493 MFI.hasStackMap() || MFI.hasPatchPoint() ||
494 RegInfo->hasStackRealignment(MF))
495 return true;
496 // With large callframes around we may need to use FP to access the scavenging
497 // emergency spillslot.
498 //
499 // Unfortunately some calls to hasFP() like machine verifier ->
500 // getReservedReg() -> hasFP in the middle of global isel are too early
501 // to know the max call frame size. Hopefully conservatively returning "true"
502 // in those cases is fine.
503 // DefaultSafeSPDisplacement is fine as we only emergency spill GP regs.
504 if (!MFI.isMaxCallFrameSizeComputed() ||
505 MFI.getMaxCallFrameSize() > DefaultSafeSPDisplacement)
506 return true;
507
508 return false;
509}
510
511/// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
512/// not required, we reserve argument space for call sites in the function
513/// immediately on entry to the current function. This eliminates the need for
514/// add/sub sp brackets around call sites. Returns true if the call frame is
515/// included as part of the stack frame.
516bool AArch64FrameLowering::hasReservedCallFrame(
517 const MachineFunction &MF) const {
518 // The stack probing code for the dynamically allocated outgoing arguments
519 // area assumes that the stack is probed at the top - either by the prologue
520 // code, which issues a probe if `hasVarSizedObjects` return true, or by the
521 // most recent variable-sized object allocation. Changing the condition here
522 // may need to be followed up by changes to the probe issuing logic.
523 return !MF.getFrameInfo().hasVarSizedObjects();
524}
525
526MachineBasicBlock::iterator AArch64FrameLowering::eliminateCallFramePseudoInstr(
527 MachineFunction &MF, MachineBasicBlock &MBB,
528 MachineBasicBlock::iterator I) const {
529 const AArch64InstrInfo *TII =
530 static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
531 const AArch64TargetLowering *TLI =
532 MF.getSubtarget<AArch64Subtarget>().getTargetLowering();
533 [[maybe_unused]] MachineFrameInfo &MFI = MF.getFrameInfo();
534 DebugLoc DL = I->getDebugLoc();
535 unsigned Opc = I->getOpcode();
536 bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
537 uint64_t CalleePopAmount = IsDestroy ? I->getOperand(i: 1).getImm() : 0;
538
539 if (!hasReservedCallFrame(MF)) {
540 int64_t Amount = I->getOperand(i: 0).getImm();
541 Amount = alignTo(Size: Amount, A: getStackAlign());
542 if (!IsDestroy)
543 Amount = -Amount;
544
545 // N.b. if CalleePopAmount is valid but zero (i.e. callee would pop, but it
546 // doesn't have to pop anything), then the first operand will be zero too so
547 // this adjustment is a no-op.
548 if (CalleePopAmount == 0) {
549 // FIXME: in-function stack adjustment for calls is limited to 24-bits
550 // because there's no guaranteed temporary register available.
551 //
552 // ADD/SUB (immediate) has only LSL #0 and LSL #12 available.
553 // 1) For offset <= 12-bit, we use LSL #0
554 // 2) For 12-bit <= offset <= 24-bit, we use two instructions. One uses
555 // LSL #0, and the other uses LSL #12.
556 //
557 // Most call frames will be allocated at the start of a function so
558 // this is OK, but it is a limitation that needs dealing with.
559 assert(Amount > -0xffffff && Amount < 0xffffff && "call frame too large");
560
561 if (TLI->hasInlineStackProbe(MF) &&
562 -Amount >= AArch64::StackProbeMaxUnprobedStack) {
563 // When stack probing is enabled, the decrement of SP may need to be
564 // probed. We only need to do this if the call site needs 1024 bytes of
565 // space or more, because a region smaller than that is allowed to be
566 // unprobed at an ABI boundary. We rely on the fact that SP has been
567 // probed exactly at this point, either by the prologue or most recent
568 // dynamic allocation.
569 assert(MFI.hasVarSizedObjects() &&
570 "non-reserved call frame without var sized objects?");
571 Register ScratchReg =
572 MF.getRegInfo().createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
573 inlineStackProbeFixed(MBBI: I, ScratchReg, FrameSize: -Amount, CFAOffset: StackOffset::get(Fixed: 0, Scalable: 0));
574 } else {
575 emitFrameOffset(MBB, MBBI: I, DL, DestReg: AArch64::SP, SrcReg: AArch64::SP,
576 Offset: StackOffset::getFixed(Fixed: Amount), TII);
577 }
578 }
579 } else if (CalleePopAmount != 0) {
580 // If the calling convention demands that the callee pops arguments from the
581 // stack, we want to add it back if we have a reserved call frame.
582 assert(CalleePopAmount < 0xffffff && "call frame too large");
583 emitFrameOffset(MBB, MBBI: I, DL, DestReg: AArch64::SP, SrcReg: AArch64::SP,
584 Offset: StackOffset::getFixed(Fixed: -(int64_t)CalleePopAmount), TII);
585 }
586 return MBB.erase(I);
587}
588
589void AArch64FrameLowering::emitCalleeSavedGPRLocations(
590 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
591 MachineFunction &MF = *MBB.getParent();
592 MachineFrameInfo &MFI = MF.getFrameInfo();
593 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
594 SMEAttrs Attrs(MF.getFunction());
595 bool LocallyStreaming =
596 Attrs.hasStreamingBody() && !Attrs.hasStreamingInterface();
597
598 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
599 if (CSI.empty())
600 return;
601
602 const TargetSubtargetInfo &STI = MF.getSubtarget();
603 const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
604 const TargetInstrInfo &TII = *STI.getInstrInfo();
605 DebugLoc DL = MBB.findDebugLoc(MBBI);
606
607 for (const auto &Info : CSI) {
608 unsigned FrameIdx = Info.getFrameIdx();
609 if (MFI.getStackID(ObjectIdx: FrameIdx) == TargetStackID::ScalableVector)
610 continue;
611
612 assert(!Info.isSpilledToReg() && "Spilling to registers not implemented");
613 int64_t DwarfReg = TRI.getDwarfRegNum(RegNum: Info.getReg(), isEH: true);
614 int64_t Offset = MFI.getObjectOffset(ObjectIdx: FrameIdx) - getOffsetOfLocalArea();
615
616 // The location of VG will be emitted before each streaming-mode change in
617 // the function. Only locally-streaming functions require emitting the
618 // non-streaming VG location here.
619 if ((LocallyStreaming && FrameIdx == AFI->getStreamingVGIdx()) ||
620 (!LocallyStreaming &&
621 DwarfReg == TRI.getDwarfRegNum(RegNum: AArch64::VG, isEH: true)))
622 continue;
623
624 unsigned CFIIndex = MF.addFrameInst(
625 Inst: MCCFIInstruction::createOffset(L: nullptr, Register: DwarfReg, Offset));
626 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::CFI_INSTRUCTION))
627 .addCFIIndex(CFIIndex)
628 .setMIFlags(MachineInstr::FrameSetup);
629 }
630}
631
632void AArch64FrameLowering::emitCalleeSavedSVELocations(
633 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
634 MachineFunction &MF = *MBB.getParent();
635 MachineFrameInfo &MFI = MF.getFrameInfo();
636
637 // Add callee saved registers to move list.
638 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
639 if (CSI.empty())
640 return;
641
642 const TargetSubtargetInfo &STI = MF.getSubtarget();
643 const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
644 const TargetInstrInfo &TII = *STI.getInstrInfo();
645 DebugLoc DL = MBB.findDebugLoc(MBBI);
646 AArch64FunctionInfo &AFI = *MF.getInfo<AArch64FunctionInfo>();
647
648 for (const auto &Info : CSI) {
649 if (!(MFI.getStackID(ObjectIdx: Info.getFrameIdx()) == TargetStackID::ScalableVector))
650 continue;
651
652 // Not all unwinders may know about SVE registers, so assume the lowest
653 // common demoninator.
654 assert(!Info.isSpilledToReg() && "Spilling to registers not implemented");
655 unsigned Reg = Info.getReg();
656 if (!static_cast<const AArch64RegisterInfo &>(TRI).regNeedsCFI(Reg, RegToUseForCFI&: Reg))
657 continue;
658
659 StackOffset Offset =
660 StackOffset::getScalable(Scalable: MFI.getObjectOffset(ObjectIdx: Info.getFrameIdx())) -
661 StackOffset::getFixed(Fixed: AFI.getCalleeSavedStackSize(MFI));
662
663 unsigned CFIIndex = MF.addFrameInst(Inst: createCFAOffset(MRI: TRI, Reg, OffsetFromDefCFA: Offset));
664 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::CFI_INSTRUCTION))
665 .addCFIIndex(CFIIndex)
666 .setMIFlags(MachineInstr::FrameSetup);
667 }
668}
669
670static void insertCFISameValue(const MCInstrDesc &Desc, MachineFunction &MF,
671 MachineBasicBlock &MBB,
672 MachineBasicBlock::iterator InsertPt,
673 unsigned DwarfReg) {
674 unsigned CFIIndex =
675 MF.addFrameInst(Inst: MCCFIInstruction::createSameValue(L: nullptr, Register: DwarfReg));
676 BuildMI(BB&: MBB, I: InsertPt, MIMD: DebugLoc(), MCID: Desc).addCFIIndex(CFIIndex);
677}
678
679void AArch64FrameLowering::resetCFIToInitialState(
680 MachineBasicBlock &MBB) const {
681
682 MachineFunction &MF = *MBB.getParent();
683 const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
684 const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
685 const auto &TRI =
686 static_cast<const AArch64RegisterInfo &>(*Subtarget.getRegisterInfo());
687 const auto &MFI = *MF.getInfo<AArch64FunctionInfo>();
688
689 const MCInstrDesc &CFIDesc = TII.get(Opcode: TargetOpcode::CFI_INSTRUCTION);
690 DebugLoc DL;
691
692 // Reset the CFA to `SP + 0`.
693 MachineBasicBlock::iterator InsertPt = MBB.begin();
694 unsigned CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::cfiDefCfa(
695 L: nullptr, Register: TRI.getDwarfRegNum(RegNum: AArch64::SP, isEH: true), Offset: 0));
696 BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: CFIDesc).addCFIIndex(CFIIndex);
697
698 // Flip the RA sign state.
699 if (MFI.shouldSignReturnAddress(MF)) {
700 CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::createNegateRAState(L: nullptr));
701 BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: CFIDesc).addCFIIndex(CFIIndex);
702 }
703
704 // Shadow call stack uses X18, reset it.
705 if (MFI.needsShadowCallStackPrologueEpilogue(MF))
706 insertCFISameValue(Desc: CFIDesc, MF, MBB, InsertPt,
707 DwarfReg: TRI.getDwarfRegNum(RegNum: AArch64::X18, isEH: true));
708
709 // Emit .cfi_same_value for callee-saved registers.
710 const std::vector<CalleeSavedInfo> &CSI =
711 MF.getFrameInfo().getCalleeSavedInfo();
712 for (const auto &Info : CSI) {
713 unsigned Reg = Info.getReg();
714 if (!TRI.regNeedsCFI(Reg, RegToUseForCFI&: Reg))
715 continue;
716 insertCFISameValue(Desc: CFIDesc, MF, MBB, InsertPt,
717 DwarfReg: TRI.getDwarfRegNum(RegNum: Reg, isEH: true));
718 }
719}
720
721static void emitCalleeSavedRestores(MachineBasicBlock &MBB,
722 MachineBasicBlock::iterator MBBI,
723 bool SVE) {
724 MachineFunction &MF = *MBB.getParent();
725 MachineFrameInfo &MFI = MF.getFrameInfo();
726
727 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
728 if (CSI.empty())
729 return;
730
731 const TargetSubtargetInfo &STI = MF.getSubtarget();
732 const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
733 const TargetInstrInfo &TII = *STI.getInstrInfo();
734 DebugLoc DL = MBB.findDebugLoc(MBBI);
735
736 for (const auto &Info : CSI) {
737 if (SVE !=
738 (MFI.getStackID(ObjectIdx: Info.getFrameIdx()) == TargetStackID::ScalableVector))
739 continue;
740
741 unsigned Reg = Info.getReg();
742 if (SVE &&
743 !static_cast<const AArch64RegisterInfo &>(TRI).regNeedsCFI(Reg, RegToUseForCFI&: Reg))
744 continue;
745
746 if (!Info.isRestored())
747 continue;
748
749 unsigned CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::createRestore(
750 L: nullptr, Register: TRI.getDwarfRegNum(RegNum: Info.getReg(), isEH: true)));
751 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::CFI_INSTRUCTION))
752 .addCFIIndex(CFIIndex)
753 .setMIFlags(MachineInstr::FrameDestroy);
754 }
755}
756
757void AArch64FrameLowering::emitCalleeSavedGPRRestores(
758 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
759 emitCalleeSavedRestores(MBB, MBBI, SVE: false);
760}
761
762void AArch64FrameLowering::emitCalleeSavedSVERestores(
763 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
764 emitCalleeSavedRestores(MBB, MBBI, SVE: true);
765}
766
767// Return the maximum possible number of bytes for `Size` due to the
768// architectural limit on the size of a SVE register.
769static int64_t upperBound(StackOffset Size) {
770 static const int64_t MAX_BYTES_PER_SCALABLE_BYTE = 16;
771 return Size.getScalable() * MAX_BYTES_PER_SCALABLE_BYTE + Size.getFixed();
772}
773
774void AArch64FrameLowering::allocateStackSpace(
775 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
776 int64_t RealignmentPadding, StackOffset AllocSize, bool NeedsWinCFI,
777 bool *HasWinCFI, bool EmitCFI, StackOffset InitialOffset,
778 bool FollowupAllocs) const {
779
780 if (!AllocSize)
781 return;
782
783 DebugLoc DL;
784 MachineFunction &MF = *MBB.getParent();
785 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
786 const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
787 AArch64FunctionInfo &AFI = *MF.getInfo<AArch64FunctionInfo>();
788 const MachineFrameInfo &MFI = MF.getFrameInfo();
789
790 const int64_t MaxAlign = MFI.getMaxAlign().value();
791 const uint64_t AndMask = ~(MaxAlign - 1);
792
793 if (!Subtarget.getTargetLowering()->hasInlineStackProbe(MF)) {
794 Register TargetReg = RealignmentPadding
795 ? findScratchNonCalleeSaveRegister(MBB: &MBB)
796 : AArch64::SP;
797 // SUB Xd/SP, SP, AllocSize
798 emitFrameOffset(MBB, MBBI, DL, DestReg: TargetReg, SrcReg: AArch64::SP, Offset: -AllocSize, TII: &TII,
799 MachineInstr::FrameSetup, SetNZCV: false, NeedsWinCFI, HasWinCFI,
800 EmitCFAOffset: EmitCFI, InitialOffset);
801
802 if (RealignmentPadding) {
803 // AND SP, X9, 0b11111...0000
804 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AArch64::ANDXri), DestReg: AArch64::SP)
805 .addReg(RegNo: TargetReg, flags: RegState::Kill)
806 .addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: AndMask, regSize: 64))
807 .setMIFlags(MachineInstr::FrameSetup);
808 AFI.setStackRealigned(true);
809
810 // No need for SEH instructions here; if we're realigning the stack,
811 // we've set a frame pointer and already finished the SEH prologue.
812 assert(!NeedsWinCFI);
813 }
814 return;
815 }
816
817 //
818 // Stack probing allocation.
819 //
820
821 // Fixed length allocation. If we don't need to re-align the stack and don't
822 // have SVE objects, we can use a more efficient sequence for stack probing.
823 if (AllocSize.getScalable() == 0 && RealignmentPadding == 0) {
824 Register ScratchReg = findScratchNonCalleeSaveRegister(MBB: &MBB);
825 assert(ScratchReg != AArch64::NoRegister);
826 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AArch64::PROBED_STACKALLOC))
827 .addDef(RegNo: ScratchReg)
828 .addImm(Val: AllocSize.getFixed())
829 .addImm(Val: InitialOffset.getFixed())
830 .addImm(Val: InitialOffset.getScalable());
831 // The fixed allocation may leave unprobed bytes at the top of the
832 // stack. If we have subsequent alocation (e.g. if we have variable-sized
833 // objects), we need to issue an extra probe, so these allocations start in
834 // a known state.
835 if (FollowupAllocs) {
836 // STR XZR, [SP]
837 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AArch64::STRXui))
838 .addReg(RegNo: AArch64::XZR)
839 .addReg(RegNo: AArch64::SP)
840 .addImm(Val: 0)
841 .setMIFlags(MachineInstr::FrameSetup);
842 }
843
844 return;
845 }
846
847 // Variable length allocation.
848
849 // If the (unknown) allocation size cannot exceed the probe size, decrement
850 // the stack pointer right away.
851 int64_t ProbeSize = AFI.getStackProbeSize();
852 if (upperBound(Size: AllocSize) + RealignmentPadding <= ProbeSize) {
853 Register ScratchReg = RealignmentPadding
854 ? findScratchNonCalleeSaveRegister(MBB: &MBB)
855 : AArch64::SP;
856 assert(ScratchReg != AArch64::NoRegister);
857 // SUB Xd, SP, AllocSize
858 emitFrameOffset(MBB, MBBI, DL, DestReg: ScratchReg, SrcReg: AArch64::SP, Offset: -AllocSize, TII: &TII,
859 MachineInstr::FrameSetup, SetNZCV: false, NeedsWinCFI, HasWinCFI,
860 EmitCFAOffset: EmitCFI, InitialOffset);
861 if (RealignmentPadding) {
862 // AND SP, Xn, 0b11111...0000
863 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AArch64::ANDXri), DestReg: AArch64::SP)
864 .addReg(RegNo: ScratchReg, flags: RegState::Kill)
865 .addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: AndMask, regSize: 64))
866 .setMIFlags(MachineInstr::FrameSetup);
867 AFI.setStackRealigned(true);
868 }
869 if (FollowupAllocs || upperBound(Size: AllocSize) + RealignmentPadding >
870 AArch64::StackProbeMaxUnprobedStack) {
871 // STR XZR, [SP]
872 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AArch64::STRXui))
873 .addReg(RegNo: AArch64::XZR)
874 .addReg(RegNo: AArch64::SP)
875 .addImm(Val: 0)
876 .setMIFlags(MachineInstr::FrameSetup);
877 }
878 return;
879 }
880
881 // Emit a variable-length allocation probing loop.
882 // TODO: As an optimisation, the loop can be "unrolled" into a few parts,
883 // each of them guaranteed to adjust the stack by less than the probe size.
884 Register TargetReg = findScratchNonCalleeSaveRegister(MBB: &MBB);
885 assert(TargetReg != AArch64::NoRegister);
886 // SUB Xd, SP, AllocSize
887 emitFrameOffset(MBB, MBBI, DL, DestReg: TargetReg, SrcReg: AArch64::SP, Offset: -AllocSize, TII: &TII,
888 MachineInstr::FrameSetup, SetNZCV: false, NeedsWinCFI, HasWinCFI,
889 EmitCFAOffset: EmitCFI, InitialOffset);
890 if (RealignmentPadding) {
891 // AND Xn, Xn, 0b11111...0000
892 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AArch64::ANDXri), DestReg: TargetReg)
893 .addReg(RegNo: TargetReg, flags: RegState::Kill)
894 .addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: AndMask, regSize: 64))
895 .setMIFlags(MachineInstr::FrameSetup);
896 }
897
898 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AArch64::PROBED_STACKALLOC_VAR))
899 .addReg(RegNo: TargetReg);
900 if (EmitCFI) {
901 // Set the CFA register back to SP.
902 unsigned Reg =
903 Subtarget.getRegisterInfo()->getDwarfRegNum(RegNum: AArch64::SP, isEH: true);
904 unsigned CFIIndex =
905 MF.addFrameInst(Inst: MCCFIInstruction::createDefCfaRegister(L: nullptr, Register: Reg));
906 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::CFI_INSTRUCTION))
907 .addCFIIndex(CFIIndex)
908 .setMIFlags(MachineInstr::FrameSetup);
909 }
910 if (RealignmentPadding)
911 AFI.setStackRealigned(true);
912}
913
914static MCRegister getRegisterOrZero(MCRegister Reg, bool HasSVE) {
915 switch (Reg.id()) {
916 default:
917 // The called routine is expected to preserve r19-r28
918 // r29 and r30 are used as frame pointer and link register resp.
919 return 0;
920
921 // GPRs
922#define CASE(n) \
923 case AArch64::W##n: \
924 case AArch64::X##n: \
925 return AArch64::X##n
926 CASE(0);
927 CASE(1);
928 CASE(2);
929 CASE(3);
930 CASE(4);
931 CASE(5);
932 CASE(6);
933 CASE(7);
934 CASE(8);
935 CASE(9);
936 CASE(10);
937 CASE(11);
938 CASE(12);
939 CASE(13);
940 CASE(14);
941 CASE(15);
942 CASE(16);
943 CASE(17);
944 CASE(18);
945#undef CASE
946
947 // FPRs
948#define CASE(n) \
949 case AArch64::B##n: \
950 case AArch64::H##n: \
951 case AArch64::S##n: \
952 case AArch64::D##n: \
953 case AArch64::Q##n: \
954 return HasSVE ? AArch64::Z##n : AArch64::Q##n
955 CASE(0);
956 CASE(1);
957 CASE(2);
958 CASE(3);
959 CASE(4);
960 CASE(5);
961 CASE(6);
962 CASE(7);
963 CASE(8);
964 CASE(9);
965 CASE(10);
966 CASE(11);
967 CASE(12);
968 CASE(13);
969 CASE(14);
970 CASE(15);
971 CASE(16);
972 CASE(17);
973 CASE(18);
974 CASE(19);
975 CASE(20);
976 CASE(21);
977 CASE(22);
978 CASE(23);
979 CASE(24);
980 CASE(25);
981 CASE(26);
982 CASE(27);
983 CASE(28);
984 CASE(29);
985 CASE(30);
986 CASE(31);
987#undef CASE
988 }
989}
990
991void AArch64FrameLowering::emitZeroCallUsedRegs(BitVector RegsToZero,
992 MachineBasicBlock &MBB) const {
993 // Insertion point.
994 MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
995
996 // Fake a debug loc.
997 DebugLoc DL;
998 if (MBBI != MBB.end())
999 DL = MBBI->getDebugLoc();
1000
1001 const MachineFunction &MF = *MBB.getParent();
1002 const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
1003 const AArch64RegisterInfo &TRI = *STI.getRegisterInfo();
1004
1005 BitVector GPRsToZero(TRI.getNumRegs());
1006 BitVector FPRsToZero(TRI.getNumRegs());
1007 bool HasSVE = STI.hasSVE();
1008 for (MCRegister Reg : RegsToZero.set_bits()) {
1009 if (TRI.isGeneralPurposeRegister(MF, Reg)) {
1010 // For GPRs, we only care to clear out the 64-bit register.
1011 if (MCRegister XReg = getRegisterOrZero(Reg, HasSVE))
1012 GPRsToZero.set(XReg);
1013 } else if (AArch64InstrInfo::isFpOrNEON(Reg)) {
1014 // For FPRs,
1015 if (MCRegister XReg = getRegisterOrZero(Reg, HasSVE))
1016 FPRsToZero.set(XReg);
1017 }
1018 }
1019
1020 const AArch64InstrInfo &TII = *STI.getInstrInfo();
1021
1022 // Zero out GPRs.
1023 for (MCRegister Reg : GPRsToZero.set_bits())
1024 TII.buildClearRegister(Reg, MBB, Iter: MBBI, DL);
1025
1026 // Zero out FP/vector registers.
1027 for (MCRegister Reg : FPRsToZero.set_bits())
1028 TII.buildClearRegister(Reg, MBB, Iter: MBBI, DL);
1029
1030 if (HasSVE) {
1031 for (MCRegister PReg :
1032 {AArch64::P0, AArch64::P1, AArch64::P2, AArch64::P3, AArch64::P4,
1033 AArch64::P5, AArch64::P6, AArch64::P7, AArch64::P8, AArch64::P9,
1034 AArch64::P10, AArch64::P11, AArch64::P12, AArch64::P13, AArch64::P14,
1035 AArch64::P15}) {
1036 if (RegsToZero[PReg])
1037 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AArch64::PFALSE), DestReg: PReg);
1038 }
1039 }
1040}
1041
1042static void getLiveRegsForEntryMBB(LivePhysRegs &LiveRegs,
1043 const MachineBasicBlock &MBB) {
1044 const MachineFunction *MF = MBB.getParent();
1045 LiveRegs.addLiveIns(MBB);
1046 // Mark callee saved registers as used so we will not choose them.
1047 const MCPhysReg *CSRegs = MF->getRegInfo().getCalleeSavedRegs();
1048 for (unsigned i = 0; CSRegs[i]; ++i)
1049 LiveRegs.addReg(Reg: CSRegs[i]);
1050}
1051
1052// Find a scratch register that we can use at the start of the prologue to
1053// re-align the stack pointer. We avoid using callee-save registers since they
1054// may appear to be free when this is called from canUseAsPrologue (during
1055// shrink wrapping), but then no longer be free when this is called from
1056// emitPrologue.
1057//
1058// FIXME: This is a bit conservative, since in the above case we could use one
1059// of the callee-save registers as a scratch temp to re-align the stack pointer,
1060// but we would then have to make sure that we were in fact saving at least one
1061// callee-save register in the prologue, which is additional complexity that
1062// doesn't seem worth the benefit.
1063static Register findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB) {
1064 MachineFunction *MF = MBB->getParent();
1065
1066 // If MBB is an entry block, use X9 as the scratch register
1067 // preserve_none functions may be using X9 to pass arguments,
1068 // so prefer to pick an available register below.
1069 if (&MF->front() == MBB &&
1070 MF->getFunction().getCallingConv() != CallingConv::PreserveNone)
1071 return AArch64::X9;
1072
1073 const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
1074 const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo();
1075 LivePhysRegs LiveRegs(TRI);
1076 getLiveRegsForEntryMBB(LiveRegs, MBB: *MBB);
1077
1078 // Prefer X9 since it was historically used for the prologue scratch reg.
1079 const MachineRegisterInfo &MRI = MF->getRegInfo();
1080 if (LiveRegs.available(MRI, Reg: AArch64::X9))
1081 return AArch64::X9;
1082
1083 for (unsigned Reg : AArch64::GPR64RegClass) {
1084 if (LiveRegs.available(MRI, Reg))
1085 return Reg;
1086 }
1087 return AArch64::NoRegister;
1088}
1089
1090bool AArch64FrameLowering::canUseAsPrologue(
1091 const MachineBasicBlock &MBB) const {
1092 const MachineFunction *MF = MBB.getParent();
1093 MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
1094 const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
1095 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
1096 const AArch64TargetLowering *TLI = Subtarget.getTargetLowering();
1097 const AArch64FunctionInfo *AFI = MF->getInfo<AArch64FunctionInfo>();
1098
1099 if (AFI->hasSwiftAsyncContext()) {
1100 const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo();
1101 const MachineRegisterInfo &MRI = MF->getRegInfo();
1102 LivePhysRegs LiveRegs(TRI);
1103 getLiveRegsForEntryMBB(LiveRegs, MBB);
1104 // The StoreSwiftAsyncContext clobbers X16 and X17. Make sure they are
1105 // available.
1106 if (!LiveRegs.available(MRI, Reg: AArch64::X16) ||
1107 !LiveRegs.available(MRI, Reg: AArch64::X17))
1108 return false;
1109 }
1110
1111 // Certain stack probing sequences might clobber flags, then we can't use
1112 // the block as a prologue if the flags register is a live-in.
1113 if (MF->getInfo<AArch64FunctionInfo>()->hasStackProbing() &&
1114 MBB.isLiveIn(Reg: AArch64::NZCV))
1115 return false;
1116
1117 // Don't need a scratch register if we're not going to re-align the stack or
1118 // emit stack probes.
1119 if (!RegInfo->hasStackRealignment(MF: *MF) && !TLI->hasInlineStackProbe(MF: *MF))
1120 return true;
1121 // Otherwise, we can use any block as long as it has a scratch register
1122 // available.
1123 return findScratchNonCalleeSaveRegister(MBB: TmpMBB) != AArch64::NoRegister;
1124}
1125
1126static bool windowsRequiresStackProbe(MachineFunction &MF,
1127 uint64_t StackSizeInBytes) {
1128 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1129 const AArch64FunctionInfo &MFI = *MF.getInfo<AArch64FunctionInfo>();
1130 // TODO: When implementing stack protectors, take that into account
1131 // for the probe threshold.
1132 return Subtarget.isTargetWindows() && MFI.hasStackProbing() &&
1133 StackSizeInBytes >= uint64_t(MFI.getStackProbeSize());
1134}
1135
1136static bool needsWinCFI(const MachineFunction &MF) {
1137 const Function &F = MF.getFunction();
1138 return MF.getTarget().getMCAsmInfo()->usesWindowsCFI() &&
1139 F.needsUnwindTableEntry();
1140}
1141
1142bool AArch64FrameLowering::shouldCombineCSRLocalStackBump(
1143 MachineFunction &MF, uint64_t StackBumpBytes) const {
1144 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1145 const MachineFrameInfo &MFI = MF.getFrameInfo();
1146 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1147 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
1148 if (homogeneousPrologEpilog(MF))
1149 return false;
1150
1151 if (AFI->getLocalStackSize() == 0)
1152 return false;
1153
1154 // For WinCFI, if optimizing for size, prefer to not combine the stack bump
1155 // (to force a stp with predecrement) to match the packed unwind format,
1156 // provided that there actually are any callee saved registers to merge the
1157 // decrement with.
1158 // This is potentially marginally slower, but allows using the packed
1159 // unwind format for functions that both have a local area and callee saved
1160 // registers. Using the packed unwind format notably reduces the size of
1161 // the unwind info.
1162 if (needsWinCFI(MF) && AFI->getCalleeSavedStackSize() > 0 &&
1163 MF.getFunction().hasOptSize())
1164 return false;
1165
1166 // 512 is the maximum immediate for stp/ldp that will be used for
1167 // callee-save save/restores
1168 if (StackBumpBytes >= 512 || windowsRequiresStackProbe(MF, StackSizeInBytes: StackBumpBytes))
1169 return false;
1170
1171 if (MFI.hasVarSizedObjects())
1172 return false;
1173
1174 if (RegInfo->hasStackRealignment(MF))
1175 return false;
1176
1177 // This isn't strictly necessary, but it simplifies things a bit since the
1178 // current RedZone handling code assumes the SP is adjusted by the
1179 // callee-save save/restore code.
1180 if (canUseRedZone(MF))
1181 return false;
1182
1183 // When there is an SVE area on the stack, always allocate the
1184 // callee-saves and spills/locals separately.
1185 if (getSVEStackSize(MF))
1186 return false;
1187
1188 return true;
1189}
1190
1191bool AArch64FrameLowering::shouldCombineCSRLocalStackBumpInEpilogue(
1192 MachineBasicBlock &MBB, unsigned StackBumpBytes) const {
1193 if (!shouldCombineCSRLocalStackBump(MF&: *MBB.getParent(), StackBumpBytes))
1194 return false;
1195
1196 if (MBB.empty())
1197 return true;
1198
1199 // Disable combined SP bump if the last instruction is an MTE tag store. It
1200 // is almost always better to merge SP adjustment into those instructions.
1201 MachineBasicBlock::iterator LastI = MBB.getFirstTerminator();
1202 MachineBasicBlock::iterator Begin = MBB.begin();
1203 while (LastI != Begin) {
1204 --LastI;
1205 if (LastI->isTransient())
1206 continue;
1207 if (!LastI->getFlag(Flag: MachineInstr::FrameDestroy))
1208 break;
1209 }
1210 switch (LastI->getOpcode()) {
1211 case AArch64::STGloop:
1212 case AArch64::STZGloop:
1213 case AArch64::STGi:
1214 case AArch64::STZGi:
1215 case AArch64::ST2Gi:
1216 case AArch64::STZ2Gi:
1217 return false;
1218 default:
1219 return true;
1220 }
1221 llvm_unreachable("unreachable");
1222}
1223
1224// Given a load or a store instruction, generate an appropriate unwinding SEH
1225// code on Windows.
1226static MachineBasicBlock::iterator InsertSEH(MachineBasicBlock::iterator MBBI,
1227 const TargetInstrInfo &TII,
1228 MachineInstr::MIFlag Flag) {
1229 unsigned Opc = MBBI->getOpcode();
1230 MachineBasicBlock *MBB = MBBI->getParent();
1231 MachineFunction &MF = *MBB->getParent();
1232 DebugLoc DL = MBBI->getDebugLoc();
1233 unsigned ImmIdx = MBBI->getNumOperands() - 1;
1234 int Imm = MBBI->getOperand(i: ImmIdx).getImm();
1235 MachineInstrBuilder MIB;
1236 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1237 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
1238
1239 switch (Opc) {
1240 default:
1241 llvm_unreachable("No SEH Opcode for this instruction");
1242 case AArch64::LDPDpost:
1243 Imm = -Imm;
1244 [[fallthrough]];
1245 case AArch64::STPDpre: {
1246 unsigned Reg0 = RegInfo->getSEHRegNum(i: MBBI->getOperand(i: 1).getReg());
1247 unsigned Reg1 = RegInfo->getSEHRegNum(i: MBBI->getOperand(i: 2).getReg());
1248 MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveFRegP_X))
1249 .addImm(Val: Reg0)
1250 .addImm(Val: Reg1)
1251 .addImm(Val: Imm * 8)
1252 .setMIFlag(Flag);
1253 break;
1254 }
1255 case AArch64::LDPXpost:
1256 Imm = -Imm;
1257 [[fallthrough]];
1258 case AArch64::STPXpre: {
1259 Register Reg0 = MBBI->getOperand(i: 1).getReg();
1260 Register Reg1 = MBBI->getOperand(i: 2).getReg();
1261 if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
1262 MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveFPLR_X))
1263 .addImm(Val: Imm * 8)
1264 .setMIFlag(Flag);
1265 else
1266 MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveRegP_X))
1267 .addImm(Val: RegInfo->getSEHRegNum(i: Reg0))
1268 .addImm(Val: RegInfo->getSEHRegNum(i: Reg1))
1269 .addImm(Val: Imm * 8)
1270 .setMIFlag(Flag);
1271 break;
1272 }
1273 case AArch64::LDRDpost:
1274 Imm = -Imm;
1275 [[fallthrough]];
1276 case AArch64::STRDpre: {
1277 unsigned Reg = RegInfo->getSEHRegNum(i: MBBI->getOperand(i: 1).getReg());
1278 MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveFReg_X))
1279 .addImm(Val: Reg)
1280 .addImm(Val: Imm)
1281 .setMIFlag(Flag);
1282 break;
1283 }
1284 case AArch64::LDRXpost:
1285 Imm = -Imm;
1286 [[fallthrough]];
1287 case AArch64::STRXpre: {
1288 unsigned Reg = RegInfo->getSEHRegNum(i: MBBI->getOperand(i: 1).getReg());
1289 MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveReg_X))
1290 .addImm(Val: Reg)
1291 .addImm(Val: Imm)
1292 .setMIFlag(Flag);
1293 break;
1294 }
1295 case AArch64::STPDi:
1296 case AArch64::LDPDi: {
1297 unsigned Reg0 = RegInfo->getSEHRegNum(i: MBBI->getOperand(i: 0).getReg());
1298 unsigned Reg1 = RegInfo->getSEHRegNum(i: MBBI->getOperand(i: 1).getReg());
1299 MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveFRegP))
1300 .addImm(Val: Reg0)
1301 .addImm(Val: Reg1)
1302 .addImm(Val: Imm * 8)
1303 .setMIFlag(Flag);
1304 break;
1305 }
1306 case AArch64::STPXi:
1307 case AArch64::LDPXi: {
1308 Register Reg0 = MBBI->getOperand(i: 0).getReg();
1309 Register Reg1 = MBBI->getOperand(i: 1).getReg();
1310 if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
1311 MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveFPLR))
1312 .addImm(Val: Imm * 8)
1313 .setMIFlag(Flag);
1314 else
1315 MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveRegP))
1316 .addImm(Val: RegInfo->getSEHRegNum(i: Reg0))
1317 .addImm(Val: RegInfo->getSEHRegNum(i: Reg1))
1318 .addImm(Val: Imm * 8)
1319 .setMIFlag(Flag);
1320 break;
1321 }
1322 case AArch64::STRXui:
1323 case AArch64::LDRXui: {
1324 int Reg = RegInfo->getSEHRegNum(i: MBBI->getOperand(i: 0).getReg());
1325 MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveReg))
1326 .addImm(Val: Reg)
1327 .addImm(Val: Imm * 8)
1328 .setMIFlag(Flag);
1329 break;
1330 }
1331 case AArch64::STRDui:
1332 case AArch64::LDRDui: {
1333 unsigned Reg = RegInfo->getSEHRegNum(i: MBBI->getOperand(i: 0).getReg());
1334 MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveFReg))
1335 .addImm(Val: Reg)
1336 .addImm(Val: Imm * 8)
1337 .setMIFlag(Flag);
1338 break;
1339 }
1340 case AArch64::STPQi:
1341 case AArch64::LDPQi: {
1342 unsigned Reg0 = RegInfo->getSEHRegNum(i: MBBI->getOperand(i: 0).getReg());
1343 unsigned Reg1 = RegInfo->getSEHRegNum(i: MBBI->getOperand(i: 1).getReg());
1344 MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveAnyRegQP))
1345 .addImm(Val: Reg0)
1346 .addImm(Val: Reg1)
1347 .addImm(Val: Imm * 16)
1348 .setMIFlag(Flag);
1349 break;
1350 }
1351 case AArch64::LDPQpost:
1352 Imm = -Imm;
1353 [[fallthrough]];
1354 case AArch64::STPQpre: {
1355 unsigned Reg0 = RegInfo->getSEHRegNum(i: MBBI->getOperand(i: 1).getReg());
1356 unsigned Reg1 = RegInfo->getSEHRegNum(i: MBBI->getOperand(i: 2).getReg());
1357 MIB = BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_SaveAnyRegQPX))
1358 .addImm(Val: Reg0)
1359 .addImm(Val: Reg1)
1360 .addImm(Val: Imm * 16)
1361 .setMIFlag(Flag);
1362 break;
1363 }
1364 }
1365 auto I = MBB->insertAfter(I: MBBI, MI: MIB);
1366 return I;
1367}
1368
1369// Fix up the SEH opcode associated with the save/restore instruction.
1370static void fixupSEHOpcode(MachineBasicBlock::iterator MBBI,
1371 unsigned LocalStackSize) {
1372 MachineOperand *ImmOpnd = nullptr;
1373 unsigned ImmIdx = MBBI->getNumOperands() - 1;
1374 switch (MBBI->getOpcode()) {
1375 default:
1376 llvm_unreachable("Fix the offset in the SEH instruction");
1377 case AArch64::SEH_SaveFPLR:
1378 case AArch64::SEH_SaveRegP:
1379 case AArch64::SEH_SaveReg:
1380 case AArch64::SEH_SaveFRegP:
1381 case AArch64::SEH_SaveFReg:
1382 case AArch64::SEH_SaveAnyRegQP:
1383 case AArch64::SEH_SaveAnyRegQPX:
1384 ImmOpnd = &MBBI->getOperand(i: ImmIdx);
1385 break;
1386 }
1387 if (ImmOpnd)
1388 ImmOpnd->setImm(ImmOpnd->getImm() + LocalStackSize);
1389}
1390
1391bool requiresGetVGCall(MachineFunction &MF) {
1392 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1393 return AFI->hasStreamingModeChanges() &&
1394 !MF.getSubtarget<AArch64Subtarget>().hasSVE();
1395}
1396
1397bool isVGInstruction(MachineBasicBlock::iterator MBBI) {
1398 unsigned Opc = MBBI->getOpcode();
1399 if (Opc == AArch64::CNTD_XPiI || Opc == AArch64::RDSVLI_XI ||
1400 Opc == AArch64::UBFMXri)
1401 return true;
1402
1403 if (requiresGetVGCall(MF&: *MBBI->getMF())) {
1404 if (Opc == AArch64::ORRXrr)
1405 return true;
1406
1407 if (Opc == AArch64::BL) {
1408 auto Op1 = MBBI->getOperand(i: 0);
1409 return Op1.isSymbol() &&
1410 (StringRef(Op1.getSymbolName()) == "__arm_get_current_vg");
1411 }
1412 }
1413
1414 return false;
1415}
1416
1417// Convert callee-save register save/restore instruction to do stack pointer
1418// decrement/increment to allocate/deallocate the callee-save stack area by
1419// converting store/load to use pre/post increment version.
1420static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
1421 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
1422 const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc,
1423 bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFI,
1424 MachineInstr::MIFlag FrameFlag = MachineInstr::FrameSetup,
1425 int CFAOffset = 0) {
1426 unsigned NewOpc;
1427
1428 // If the function contains streaming mode changes, we expect instructions
1429 // to calculate the value of VG before spilling. For locally-streaming
1430 // functions, we need to do this for both the streaming and non-streaming
1431 // vector length. Move past these instructions if necessary.
1432 MachineFunction &MF = *MBB.getParent();
1433 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1434 if (AFI->hasStreamingModeChanges())
1435 while (isVGInstruction(MBBI))
1436 ++MBBI;
1437
1438 switch (MBBI->getOpcode()) {
1439 default:
1440 llvm_unreachable("Unexpected callee-save save/restore opcode!");
1441 case AArch64::STPXi:
1442 NewOpc = AArch64::STPXpre;
1443 break;
1444 case AArch64::STPDi:
1445 NewOpc = AArch64::STPDpre;
1446 break;
1447 case AArch64::STPQi:
1448 NewOpc = AArch64::STPQpre;
1449 break;
1450 case AArch64::STRXui:
1451 NewOpc = AArch64::STRXpre;
1452 break;
1453 case AArch64::STRDui:
1454 NewOpc = AArch64::STRDpre;
1455 break;
1456 case AArch64::STRQui:
1457 NewOpc = AArch64::STRQpre;
1458 break;
1459 case AArch64::LDPXi:
1460 NewOpc = AArch64::LDPXpost;
1461 break;
1462 case AArch64::LDPDi:
1463 NewOpc = AArch64::LDPDpost;
1464 break;
1465 case AArch64::LDPQi:
1466 NewOpc = AArch64::LDPQpost;
1467 break;
1468 case AArch64::LDRXui:
1469 NewOpc = AArch64::LDRXpost;
1470 break;
1471 case AArch64::LDRDui:
1472 NewOpc = AArch64::LDRDpost;
1473 break;
1474 case AArch64::LDRQui:
1475 NewOpc = AArch64::LDRQpost;
1476 break;
1477 }
1478 // Get rid of the SEH code associated with the old instruction.
1479 if (NeedsWinCFI) {
1480 auto SEH = std::next(x: MBBI);
1481 if (AArch64InstrInfo::isSEHInstruction(MI: *SEH))
1482 SEH->eraseFromParent();
1483 }
1484
1485 TypeSize Scale = TypeSize::getFixed(ExactSize: 1), Width = TypeSize::getFixed(ExactSize: 0);
1486 int64_t MinOffset, MaxOffset;
1487 bool Success = static_cast<const AArch64InstrInfo *>(TII)->getMemOpInfo(
1488 Opcode: NewOpc, Scale, Width, MinOffset, MaxOffset);
1489 (void)Success;
1490 assert(Success && "unknown load/store opcode");
1491
1492 // If the first store isn't right where we want SP then we can't fold the
1493 // update in so create a normal arithmetic instruction instead.
1494 if (MBBI->getOperand(i: MBBI->getNumOperands() - 1).getImm() != 0 ||
1495 CSStackSizeInc < MinOffset || CSStackSizeInc > MaxOffset) {
1496 // If we are destroying the frame, make sure we add the increment after the
1497 // last frame operation.
1498 if (FrameFlag == MachineInstr::FrameDestroy)
1499 ++MBBI;
1500 emitFrameOffset(MBB, MBBI, DL, DestReg: AArch64::SP, SrcReg: AArch64::SP,
1501 Offset: StackOffset::getFixed(Fixed: CSStackSizeInc), TII, FrameFlag,
1502 SetNZCV: false, NeedsWinCFI: false, HasWinCFI: nullptr, EmitCFAOffset: EmitCFI,
1503 InitialOffset: StackOffset::getFixed(Fixed: CFAOffset));
1504
1505 return std::prev(x: MBBI);
1506 }
1507
1508 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: NewOpc));
1509 MIB.addReg(RegNo: AArch64::SP, flags: RegState::Define);
1510
1511 // Copy all operands other than the immediate offset.
1512 unsigned OpndIdx = 0;
1513 for (unsigned OpndEnd = MBBI->getNumOperands() - 1; OpndIdx < OpndEnd;
1514 ++OpndIdx)
1515 MIB.add(MO: MBBI->getOperand(i: OpndIdx));
1516
1517 assert(MBBI->getOperand(OpndIdx).getImm() == 0 &&
1518 "Unexpected immediate offset in first/last callee-save save/restore "
1519 "instruction!");
1520 assert(MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP &&
1521 "Unexpected base register in callee-save save/restore instruction!");
1522 assert(CSStackSizeInc % Scale == 0);
1523 MIB.addImm(Val: CSStackSizeInc / (int)Scale);
1524
1525 MIB.setMIFlags(MBBI->getFlags());
1526 MIB.setMemRefs(MBBI->memoperands());
1527
1528 // Generate a new SEH code that corresponds to the new instruction.
1529 if (NeedsWinCFI) {
1530 *HasWinCFI = true;
1531 InsertSEH(MBBI: *MIB, TII: *TII, Flag: FrameFlag);
1532 }
1533
1534 if (EmitCFI) {
1535 unsigned CFIIndex = MF.addFrameInst(
1536 Inst: MCCFIInstruction::cfiDefCfaOffset(L: nullptr, Offset: CFAOffset - CSStackSizeInc));
1537 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
1538 .addCFIIndex(CFIIndex)
1539 .setMIFlags(FrameFlag);
1540 }
1541
1542 return std::prev(x: MBB.erase(I: MBBI));
1543}
1544
1545// Fixup callee-save register save/restore instructions to take into account
1546// combined SP bump by adding the local stack size to the stack offsets.
1547static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI,
1548 uint64_t LocalStackSize,
1549 bool NeedsWinCFI,
1550 bool *HasWinCFI) {
1551 if (AArch64InstrInfo::isSEHInstruction(MI))
1552 return;
1553
1554 unsigned Opc = MI.getOpcode();
1555 unsigned Scale;
1556 switch (Opc) {
1557 case AArch64::STPXi:
1558 case AArch64::STRXui:
1559 case AArch64::STPDi:
1560 case AArch64::STRDui:
1561 case AArch64::LDPXi:
1562 case AArch64::LDRXui:
1563 case AArch64::LDPDi:
1564 case AArch64::LDRDui:
1565 Scale = 8;
1566 break;
1567 case AArch64::STPQi:
1568 case AArch64::STRQui:
1569 case AArch64::LDPQi:
1570 case AArch64::LDRQui:
1571 Scale = 16;
1572 break;
1573 default:
1574 llvm_unreachable("Unexpected callee-save save/restore opcode!");
1575 }
1576
1577 unsigned OffsetIdx = MI.getNumExplicitOperands() - 1;
1578 assert(MI.getOperand(OffsetIdx - 1).getReg() == AArch64::SP &&
1579 "Unexpected base register in callee-save save/restore instruction!");
1580 // Last operand is immediate offset that needs fixing.
1581 MachineOperand &OffsetOpnd = MI.getOperand(i: OffsetIdx);
1582 // All generated opcodes have scaled offsets.
1583 assert(LocalStackSize % Scale == 0);
1584 OffsetOpnd.setImm(OffsetOpnd.getImm() + LocalStackSize / Scale);
1585
1586 if (NeedsWinCFI) {
1587 *HasWinCFI = true;
1588 auto MBBI = std::next(x: MachineBasicBlock::iterator(MI));
1589 assert(MBBI != MI.getParent()->end() && "Expecting a valid instruction");
1590 assert(AArch64InstrInfo::isSEHInstruction(*MBBI) &&
1591 "Expecting a SEH instruction");
1592 fixupSEHOpcode(MBBI, LocalStackSize);
1593 }
1594}
1595
1596static bool isTargetWindows(const MachineFunction &MF) {
1597 return MF.getSubtarget<AArch64Subtarget>().isTargetWindows();
1598}
1599
1600// Convenience function to determine whether I is an SVE callee save.
1601static bool IsSVECalleeSave(MachineBasicBlock::iterator I) {
1602 switch (I->getOpcode()) {
1603 default:
1604 return false;
1605 case AArch64::PTRUE_C_B:
1606 case AArch64::LD1B_2Z_IMM:
1607 case AArch64::ST1B_2Z_IMM:
1608 case AArch64::STR_ZXI:
1609 case AArch64::STR_PXI:
1610 case AArch64::LDR_ZXI:
1611 case AArch64::LDR_PXI:
1612 return I->getFlag(Flag: MachineInstr::FrameSetup) ||
1613 I->getFlag(Flag: MachineInstr::FrameDestroy);
1614 }
1615}
1616
1617static void emitShadowCallStackPrologue(const TargetInstrInfo &TII,
1618 MachineFunction &MF,
1619 MachineBasicBlock &MBB,
1620 MachineBasicBlock::iterator MBBI,
1621 const DebugLoc &DL, bool NeedsWinCFI,
1622 bool NeedsUnwindInfo) {
1623 // Shadow call stack prolog: str x30, [x18], #8
1624 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AArch64::STRXpost))
1625 .addReg(RegNo: AArch64::X18, flags: RegState::Define)
1626 .addReg(RegNo: AArch64::LR)
1627 .addReg(RegNo: AArch64::X18)
1628 .addImm(Val: 8)
1629 .setMIFlag(MachineInstr::FrameSetup);
1630
1631 // This instruction also makes x18 live-in to the entry block.
1632 MBB.addLiveIn(PhysReg: AArch64::X18);
1633
1634 if (NeedsWinCFI)
1635 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AArch64::SEH_Nop))
1636 .setMIFlag(MachineInstr::FrameSetup);
1637
1638 if (NeedsUnwindInfo) {
1639 // Emit a CFI instruction that causes 8 to be subtracted from the value of
1640 // x18 when unwinding past this frame.
1641 static const char CFIInst[] = {
1642 dwarf::DW_CFA_val_expression,
1643 18, // register
1644 2, // length
1645 static_cast<char>(unsigned(dwarf::DW_OP_breg18)),
1646 static_cast<char>(-8) & 0x7f, // addend (sleb128)
1647 };
1648 unsigned CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::createEscape(
1649 L: nullptr, Vals: StringRef(CFIInst, sizeof(CFIInst))));
1650 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AArch64::CFI_INSTRUCTION))
1651 .addCFIIndex(CFIIndex)
1652 .setMIFlag(MachineInstr::FrameSetup);
1653 }
1654}
1655
1656static void emitShadowCallStackEpilogue(const TargetInstrInfo &TII,
1657 MachineFunction &MF,
1658 MachineBasicBlock &MBB,
1659 MachineBasicBlock::iterator MBBI,
1660 const DebugLoc &DL) {
1661 // Shadow call stack epilog: ldr x30, [x18, #-8]!
1662 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AArch64::LDRXpre))
1663 .addReg(RegNo: AArch64::X18, flags: RegState::Define)
1664 .addReg(RegNo: AArch64::LR, flags: RegState::Define)
1665 .addReg(RegNo: AArch64::X18)
1666 .addImm(Val: -8)
1667 .setMIFlag(MachineInstr::FrameDestroy);
1668
1669 if (MF.getInfo<AArch64FunctionInfo>()->needsAsyncDwarfUnwindInfo(MF)) {
1670 unsigned CFIIndex =
1671 MF.addFrameInst(Inst: MCCFIInstruction::createRestore(L: nullptr, Register: 18));
1672 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::CFI_INSTRUCTION))
1673 .addCFIIndex(CFIIndex)
1674 .setMIFlags(MachineInstr::FrameDestroy);
1675 }
1676}
1677
1678// Define the current CFA rule to use the provided FP.
1679static void emitDefineCFAWithFP(MachineFunction &MF, MachineBasicBlock &MBB,
1680 MachineBasicBlock::iterator MBBI,
1681 const DebugLoc &DL, unsigned FixedObject) {
1682 const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
1683 const AArch64RegisterInfo *TRI = STI.getRegisterInfo();
1684 const TargetInstrInfo *TII = STI.getInstrInfo();
1685 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1686
1687 const int OffsetToFirstCalleeSaveFromFP =
1688 AFI->getCalleeSaveBaseToFrameRecordOffset() -
1689 AFI->getCalleeSavedStackSize();
1690 Register FramePtr = TRI->getFrameRegister(MF);
1691 unsigned Reg = TRI->getDwarfRegNum(RegNum: FramePtr, isEH: true);
1692 unsigned CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::cfiDefCfa(
1693 L: nullptr, Register: Reg, Offset: FixedObject - OffsetToFirstCalleeSaveFromFP));
1694 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
1695 .addCFIIndex(CFIIndex)
1696 .setMIFlags(MachineInstr::FrameSetup);
1697}
1698
1699#ifndef NDEBUG
1700/// Collect live registers from the end of \p MI's parent up to (including) \p
1701/// MI in \p LiveRegs.
1702static void getLivePhysRegsUpTo(MachineInstr &MI, const TargetRegisterInfo &TRI,
1703 LivePhysRegs &LiveRegs) {
1704
1705 MachineBasicBlock &MBB = *MI.getParent();
1706 LiveRegs.addLiveOuts(MBB);
1707 for (const MachineInstr &MI :
1708 reverse(make_range(MI.getIterator(), MBB.instr_end())))
1709 LiveRegs.stepBackward(MI);
1710}
1711#endif
1712
1713void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
1714 MachineBasicBlock &MBB) const {
1715 MachineBasicBlock::iterator MBBI = MBB.begin();
1716 const MachineFrameInfo &MFI = MF.getFrameInfo();
1717 const Function &F = MF.getFunction();
1718 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1719 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
1720 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1721
1722 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1723 bool EmitCFI = AFI->needsDwarfUnwindInfo(MF);
1724 bool EmitAsyncCFI = AFI->needsAsyncDwarfUnwindInfo(MF);
1725 bool HasFP = hasFP(MF);
1726 bool NeedsWinCFI = needsWinCFI(MF);
1727 bool HasWinCFI = false;
1728 auto Cleanup = make_scope_exit(F: [&]() { MF.setHasWinCFI(HasWinCFI); });
1729
1730 MachineBasicBlock::iterator End = MBB.end();
1731#ifndef NDEBUG
1732 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
1733 // Collect live register from the end of MBB up to the start of the existing
1734 // frame setup instructions.
1735 MachineBasicBlock::iterator NonFrameStart = MBB.begin();
1736 while (NonFrameStart != End &&
1737 NonFrameStart->getFlag(MachineInstr::FrameSetup))
1738 ++NonFrameStart;
1739
1740 LivePhysRegs LiveRegs(*TRI);
1741 if (NonFrameStart != MBB.end()) {
1742 getLivePhysRegsUpTo(*NonFrameStart, *TRI, LiveRegs);
1743 // Ignore registers used for stack management for now.
1744 LiveRegs.removeReg(AArch64::SP);
1745 LiveRegs.removeReg(AArch64::X19);
1746 LiveRegs.removeReg(AArch64::FP);
1747 LiveRegs.removeReg(AArch64::LR);
1748
1749 // X0 will be clobbered by a call to __arm_get_current_vg in the prologue.
1750 // This is necessary to spill VG if required where SVE is unavailable, but
1751 // X0 is preserved around this call.
1752 if (requiresGetVGCall(MF))
1753 LiveRegs.removeReg(AArch64::X0);
1754 }
1755
1756 auto VerifyClobberOnExit = make_scope_exit([&]() {
1757 if (NonFrameStart == MBB.end())
1758 return;
1759 // Check if any of the newly instructions clobber any of the live registers.
1760 for (MachineInstr &MI :
1761 make_range(MBB.instr_begin(), NonFrameStart->getIterator())) {
1762 for (auto &Op : MI.operands())
1763 if (Op.isReg() && Op.isDef())
1764 assert(!LiveRegs.contains(Op.getReg()) &&
1765 "live register clobbered by inserted prologue instructions");
1766 }
1767 });
1768#endif
1769
1770 bool IsFunclet = MBB.isEHFuncletEntry();
1771
1772 // At this point, we're going to decide whether or not the function uses a
1773 // redzone. In most cases, the function doesn't have a redzone so let's
1774 // assume that's false and set it to true in the case that there's a redzone.
1775 AFI->setHasRedZone(false);
1776
1777 // Debug location must be unknown since the first debug location is used
1778 // to determine the end of the prologue.
1779 DebugLoc DL;
1780
1781 const auto &MFnI = *MF.getInfo<AArch64FunctionInfo>();
1782 if (MFnI.needsShadowCallStackPrologueEpilogue(MF))
1783 emitShadowCallStackPrologue(TII: *TII, MF, MBB, MBBI, DL, NeedsWinCFI,
1784 NeedsUnwindInfo: MFnI.needsDwarfUnwindInfo(MF));
1785
1786 if (MFnI.shouldSignReturnAddress(MF)) {
1787 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PAUTH_PROLOGUE))
1788 .setMIFlag(MachineInstr::FrameSetup);
1789 if (NeedsWinCFI)
1790 HasWinCFI = true; // AArch64PointerAuth pass will insert SEH_PACSignLR
1791 }
1792
1793 if (EmitCFI && MFnI.isMTETagged()) {
1794 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::EMITMTETAGGED))
1795 .setMIFlag(MachineInstr::FrameSetup);
1796 }
1797
1798 // We signal the presence of a Swift extended frame to external tools by
1799 // storing FP with 0b0001 in bits 63:60. In normal userland operation a simple
1800 // ORR is sufficient, it is assumed a Swift kernel would initialize the TBI
1801 // bits so that is still true.
1802 if (HasFP && AFI->hasSwiftAsyncContext()) {
1803 switch (MF.getTarget().Options.SwiftAsyncFramePointer) {
1804 case SwiftAsyncFramePointerMode::DeploymentBased:
1805 if (Subtarget.swiftAsyncContextIsDynamicallySet()) {
1806 // The special symbol below is absolute and has a *value* that can be
1807 // combined with the frame pointer to signal an extended frame.
1808 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::LOADgot), DestReg: AArch64::X16)
1809 .addExternalSymbol(FnName: "swift_async_extendedFramePointerFlags",
1810 TargetFlags: AArch64II::MO_GOT);
1811 if (NeedsWinCFI) {
1812 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_Nop))
1813 .setMIFlags(MachineInstr::FrameSetup);
1814 HasWinCFI = true;
1815 }
1816 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ORRXrs), DestReg: AArch64::FP)
1817 .addUse(RegNo: AArch64::FP)
1818 .addUse(RegNo: AArch64::X16)
1819 .addImm(Val: Subtarget.isTargetILP32() ? 32 : 0);
1820 if (NeedsWinCFI) {
1821 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_Nop))
1822 .setMIFlags(MachineInstr::FrameSetup);
1823 HasWinCFI = true;
1824 }
1825 break;
1826 }
1827 [[fallthrough]];
1828
1829 case SwiftAsyncFramePointerMode::Always:
1830 // ORR x29, x29, #0x1000_0000_0000_0000
1831 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ORRXri), DestReg: AArch64::FP)
1832 .addUse(RegNo: AArch64::FP)
1833 .addImm(Val: 0x1100)
1834 .setMIFlag(MachineInstr::FrameSetup);
1835 if (NeedsWinCFI) {
1836 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_Nop))
1837 .setMIFlags(MachineInstr::FrameSetup);
1838 HasWinCFI = true;
1839 }
1840 break;
1841
1842 case SwiftAsyncFramePointerMode::Never:
1843 break;
1844 }
1845 }
1846
1847 // All calls are tail calls in GHC calling conv, and functions have no
1848 // prologue/epilogue.
1849 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
1850 return;
1851
1852 // Set tagged base pointer to the requested stack slot.
1853 // Ideally it should match SP value after prologue.
1854 std::optional<int> TBPI = AFI->getTaggedBasePointerIndex();
1855 if (TBPI)
1856 AFI->setTaggedBasePointerOffset(-MFI.getObjectOffset(ObjectIdx: *TBPI));
1857 else
1858 AFI->setTaggedBasePointerOffset(MFI.getStackSize());
1859
1860 const StackOffset &SVEStackSize = getSVEStackSize(MF);
1861
1862 // getStackSize() includes all the locals in its size calculation. We don't
1863 // include these locals when computing the stack size of a funclet, as they
1864 // are allocated in the parent's stack frame and accessed via the frame
1865 // pointer from the funclet. We only save the callee saved registers in the
1866 // funclet, which are really the callee saved registers of the parent
1867 // function, including the funclet.
1868 int64_t NumBytes =
1869 IsFunclet ? getWinEHFuncletFrameSize(MF) : MFI.getStackSize();
1870 if (!AFI->hasStackFrame() && !windowsRequiresStackProbe(MF, StackSizeInBytes: NumBytes)) {
1871 assert(!HasFP && "unexpected function without stack frame but with FP");
1872 assert(!SVEStackSize &&
1873 "unexpected function without stack frame but with SVE objects");
1874 // All of the stack allocation is for locals.
1875 AFI->setLocalStackSize(NumBytes);
1876 if (!NumBytes)
1877 return;
1878 // REDZONE: If the stack size is less than 128 bytes, we don't need
1879 // to actually allocate.
1880 if (canUseRedZone(MF)) {
1881 AFI->setHasRedZone(true);
1882 ++NumRedZoneFunctions;
1883 } else {
1884 emitFrameOffset(MBB, MBBI, DL, DestReg: AArch64::SP, SrcReg: AArch64::SP,
1885 Offset: StackOffset::getFixed(Fixed: -NumBytes), TII,
1886 MachineInstr::FrameSetup, SetNZCV: false, NeedsWinCFI, HasWinCFI: &HasWinCFI);
1887 if (EmitCFI) {
1888 // Label used to tie together the PROLOG_LABEL and the MachineMoves.
1889 MCSymbol *FrameLabel = MF.getContext().createTempSymbol();
1890 // Encode the stack size of the leaf function.
1891 unsigned CFIIndex = MF.addFrameInst(
1892 Inst: MCCFIInstruction::cfiDefCfaOffset(L: FrameLabel, Offset: NumBytes));
1893 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
1894 .addCFIIndex(CFIIndex)
1895 .setMIFlags(MachineInstr::FrameSetup);
1896 }
1897 }
1898
1899 if (NeedsWinCFI) {
1900 HasWinCFI = true;
1901 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_PrologEnd))
1902 .setMIFlag(MachineInstr::FrameSetup);
1903 }
1904
1905 return;
1906 }
1907
1908 bool IsWin64 = Subtarget.isCallingConvWin64(CC: F.getCallingConv(), IsVarArg: F.isVarArg());
1909 unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
1910
1911 auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
1912 // All of the remaining stack allocations are for locals.
1913 AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
1914 bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, StackBumpBytes: NumBytes);
1915 bool HomPrologEpilog = homogeneousPrologEpilog(MF);
1916 if (CombineSPBump) {
1917 assert(!SVEStackSize && "Cannot combine SP bump with SVE");
1918 emitFrameOffset(MBB, MBBI, DL, DestReg: AArch64::SP, SrcReg: AArch64::SP,
1919 Offset: StackOffset::getFixed(Fixed: -NumBytes), TII,
1920 MachineInstr::FrameSetup, SetNZCV: false, NeedsWinCFI, HasWinCFI: &HasWinCFI,
1921 EmitCFAOffset: EmitAsyncCFI);
1922 NumBytes = 0;
1923 } else if (HomPrologEpilog) {
1924 // Stack has been already adjusted.
1925 NumBytes -= PrologueSaveSize;
1926 } else if (PrologueSaveSize != 0) {
1927 MBBI = convertCalleeSaveRestoreToSPPrePostIncDec(
1928 MBB, MBBI, DL, TII, CSStackSizeInc: -PrologueSaveSize, NeedsWinCFI, HasWinCFI: &HasWinCFI,
1929 EmitCFI: EmitAsyncCFI);
1930 NumBytes -= PrologueSaveSize;
1931 }
1932 assert(NumBytes >= 0 && "Negative stack allocation size!?");
1933
1934 // Move past the saves of the callee-saved registers, fixing up the offsets
1935 // and pre-inc if we decided to combine the callee-save and local stack
1936 // pointer bump above.
1937 while (MBBI != End && MBBI->getFlag(Flag: MachineInstr::FrameSetup) &&
1938 !IsSVECalleeSave(I: MBBI)) {
1939 // Move past instructions generated to calculate VG
1940 if (AFI->hasStreamingModeChanges())
1941 while (isVGInstruction(MBBI))
1942 ++MBBI;
1943
1944 if (CombineSPBump)
1945 fixupCalleeSaveRestoreStackOffset(MI&: *MBBI, LocalStackSize: AFI->getLocalStackSize(),
1946 NeedsWinCFI, HasWinCFI: &HasWinCFI);
1947 ++MBBI;
1948 }
1949
1950 // For funclets the FP belongs to the containing function.
1951 if (!IsFunclet && HasFP) {
1952 // Only set up FP if we actually need to.
1953 int64_t FPOffset = AFI->getCalleeSaveBaseToFrameRecordOffset();
1954
1955 if (CombineSPBump)
1956 FPOffset += AFI->getLocalStackSize();
1957
1958 if (AFI->hasSwiftAsyncContext()) {
1959 // Before we update the live FP we have to ensure there's a valid (or
1960 // null) asynchronous context in its slot just before FP in the frame
1961 // record, so store it now.
1962 const auto &Attrs = MF.getFunction().getAttributes();
1963 bool HaveInitialContext = Attrs.hasAttrSomewhere(Kind: Attribute::SwiftAsync);
1964 if (HaveInitialContext)
1965 MBB.addLiveIn(PhysReg: AArch64::X22);
1966 Register Reg = HaveInitialContext ? AArch64::X22 : AArch64::XZR;
1967 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::StoreSwiftAsyncContext))
1968 .addUse(RegNo: Reg)
1969 .addUse(RegNo: AArch64::SP)
1970 .addImm(Val: FPOffset - 8)
1971 .setMIFlags(MachineInstr::FrameSetup);
1972 if (NeedsWinCFI) {
1973 // WinCFI and arm64e, where StoreSwiftAsyncContext is expanded
1974 // to multiple instructions, should be mutually-exclusive.
1975 assert(Subtarget.getTargetTriple().getArchName() != "arm64e");
1976 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_Nop))
1977 .setMIFlags(MachineInstr::FrameSetup);
1978 HasWinCFI = true;
1979 }
1980 }
1981
1982 if (HomPrologEpilog) {
1983 auto Prolog = MBBI;
1984 --Prolog;
1985 assert(Prolog->getOpcode() == AArch64::HOM_Prolog);
1986 Prolog->addOperand(Op: MachineOperand::CreateImm(Val: FPOffset));
1987 } else {
1988 // Issue sub fp, sp, FPOffset or
1989 // mov fp,sp when FPOffset is zero.
1990 // Note: All stores of callee-saved registers are marked as "FrameSetup".
1991 // This code marks the instruction(s) that set the FP also.
1992 emitFrameOffset(MBB, MBBI, DL, DestReg: AArch64::FP, SrcReg: AArch64::SP,
1993 Offset: StackOffset::getFixed(Fixed: FPOffset), TII,
1994 MachineInstr::FrameSetup, SetNZCV: false, NeedsWinCFI, HasWinCFI: &HasWinCFI);
1995 if (NeedsWinCFI && HasWinCFI) {
1996 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_PrologEnd))
1997 .setMIFlag(MachineInstr::FrameSetup);
1998 // After setting up the FP, the rest of the prolog doesn't need to be
1999 // included in the SEH unwind info.
2000 NeedsWinCFI = false;
2001 }
2002 }
2003 if (EmitAsyncCFI)
2004 emitDefineCFAWithFP(MF, MBB, MBBI, DL, FixedObject);
2005 }
2006
2007 // Now emit the moves for whatever callee saved regs we have (including FP,
2008 // LR if those are saved). Frame instructions for SVE register are emitted
2009 // later, after the instruction which actually save SVE regs.
2010 if (EmitAsyncCFI)
2011 emitCalleeSavedGPRLocations(MBB, MBBI);
2012
2013 // Alignment is required for the parent frame, not the funclet
2014 const bool NeedsRealignment =
2015 NumBytes && !IsFunclet && RegInfo->hasStackRealignment(MF);
2016 const int64_t RealignmentPadding =
2017 (NeedsRealignment && MFI.getMaxAlign() > Align(16))
2018 ? MFI.getMaxAlign().value() - 16
2019 : 0;
2020
2021 if (windowsRequiresStackProbe(MF, StackSizeInBytes: NumBytes + RealignmentPadding)) {
2022 uint64_t NumWords = (NumBytes + RealignmentPadding) >> 4;
2023 if (NeedsWinCFI) {
2024 HasWinCFI = true;
2025 // alloc_l can hold at most 256MB, so assume that NumBytes doesn't
2026 // exceed this amount. We need to move at most 2^24 - 1 into x15.
2027 // This is at most two instructions, MOVZ follwed by MOVK.
2028 // TODO: Fix to use multiple stack alloc unwind codes for stacks
2029 // exceeding 256MB in size.
2030 if (NumBytes >= (1 << 28))
2031 report_fatal_error(reason: "Stack size cannot exceed 256MB for stack "
2032 "unwinding purposes");
2033
2034 uint32_t LowNumWords = NumWords & 0xFFFF;
2035 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::MOVZXi), DestReg: AArch64::X15)
2036 .addImm(Val: LowNumWords)
2037 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0))
2038 .setMIFlag(MachineInstr::FrameSetup);
2039 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_Nop))
2040 .setMIFlag(MachineInstr::FrameSetup);
2041 if ((NumWords & 0xFFFF0000) != 0) {
2042 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::MOVKXi), DestReg: AArch64::X15)
2043 .addReg(RegNo: AArch64::X15)
2044 .addImm(Val: (NumWords & 0xFFFF0000) >> 16) // High half
2045 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 16))
2046 .setMIFlag(MachineInstr::FrameSetup);
2047 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_Nop))
2048 .setMIFlag(MachineInstr::FrameSetup);
2049 }
2050 } else {
2051 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::MOVi64imm), DestReg: AArch64::X15)
2052 .addImm(Val: NumWords)
2053 .setMIFlags(MachineInstr::FrameSetup);
2054 }
2055
2056 const char *ChkStk = Subtarget.getChkStkName();
2057 switch (MF.getTarget().getCodeModel()) {
2058 case CodeModel::Tiny:
2059 case CodeModel::Small:
2060 case CodeModel::Medium:
2061 case CodeModel::Kernel:
2062 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::BL))
2063 .addExternalSymbol(FnName: ChkStk)
2064 .addReg(RegNo: AArch64::X15, flags: RegState::Implicit)
2065 .addReg(RegNo: AArch64::X16, flags: RegState::Implicit | RegState::Define | RegState::Dead)
2066 .addReg(RegNo: AArch64::X17, flags: RegState::Implicit | RegState::Define | RegState::Dead)
2067 .addReg(RegNo: AArch64::NZCV, flags: RegState::Implicit | RegState::Define | RegState::Dead)
2068 .setMIFlags(MachineInstr::FrameSetup);
2069 if (NeedsWinCFI) {
2070 HasWinCFI = true;
2071 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_Nop))
2072 .setMIFlag(MachineInstr::FrameSetup);
2073 }
2074 break;
2075 case CodeModel::Large:
2076 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::MOVaddrEXT))
2077 .addReg(RegNo: AArch64::X16, flags: RegState::Define)
2078 .addExternalSymbol(FnName: ChkStk)
2079 .addExternalSymbol(FnName: ChkStk)
2080 .setMIFlags(MachineInstr::FrameSetup);
2081 if (NeedsWinCFI) {
2082 HasWinCFI = true;
2083 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_Nop))
2084 .setMIFlag(MachineInstr::FrameSetup);
2085 }
2086
2087 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: getBLRCallOpcode(MF)))
2088 .addReg(RegNo: AArch64::X16, flags: RegState::Kill)
2089 .addReg(RegNo: AArch64::X15, flags: RegState::Implicit | RegState::Define)
2090 .addReg(RegNo: AArch64::X16, flags: RegState::Implicit | RegState::Define | RegState::Dead)
2091 .addReg(RegNo: AArch64::X17, flags: RegState::Implicit | RegState::Define | RegState::Dead)
2092 .addReg(RegNo: AArch64::NZCV, flags: RegState::Implicit | RegState::Define | RegState::Dead)
2093 .setMIFlags(MachineInstr::FrameSetup);
2094 if (NeedsWinCFI) {
2095 HasWinCFI = true;
2096 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_Nop))
2097 .setMIFlag(MachineInstr::FrameSetup);
2098 }
2099 break;
2100 }
2101
2102 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SUBXrx64), DestReg: AArch64::SP)
2103 .addReg(RegNo: AArch64::SP, flags: RegState::Kill)
2104 .addReg(RegNo: AArch64::X15, flags: RegState::Kill)
2105 .addImm(Val: AArch64_AM::getArithExtendImm(ET: AArch64_AM::UXTX, Imm: 4))
2106 .setMIFlags(MachineInstr::FrameSetup);
2107 if (NeedsWinCFI) {
2108 HasWinCFI = true;
2109 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_StackAlloc))
2110 .addImm(Val: NumBytes)
2111 .setMIFlag(MachineInstr::FrameSetup);
2112 }
2113 NumBytes = 0;
2114
2115 if (RealignmentPadding > 0) {
2116 if (RealignmentPadding >= 4096) {
2117 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::MOVi64imm))
2118 .addReg(RegNo: AArch64::X16, flags: RegState::Define)
2119 .addImm(Val: RealignmentPadding)
2120 .setMIFlags(MachineInstr::FrameSetup);
2121 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADDXrx64), DestReg: AArch64::X15)
2122 .addReg(RegNo: AArch64::SP)
2123 .addReg(RegNo: AArch64::X16, flags: RegState::Kill)
2124 .addImm(Val: AArch64_AM::getArithExtendImm(ET: AArch64_AM::UXTX, Imm: 0))
2125 .setMIFlag(MachineInstr::FrameSetup);
2126 } else {
2127 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADDXri), DestReg: AArch64::X15)
2128 .addReg(RegNo: AArch64::SP)
2129 .addImm(Val: RealignmentPadding)
2130 .addImm(Val: 0)
2131 .setMIFlag(MachineInstr::FrameSetup);
2132 }
2133
2134 uint64_t AndMask = ~(MFI.getMaxAlign().value() - 1);
2135 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ANDXri), DestReg: AArch64::SP)
2136 .addReg(RegNo: AArch64::X15, flags: RegState::Kill)
2137 .addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: AndMask, regSize: 64));
2138 AFI->setStackRealigned(true);
2139
2140 // No need for SEH instructions here; if we're realigning the stack,
2141 // we've set a frame pointer and already finished the SEH prologue.
2142 assert(!NeedsWinCFI);
2143 }
2144 }
2145
2146 StackOffset SVECalleeSavesSize = {}, SVELocalsSize = SVEStackSize;
2147 MachineBasicBlock::iterator CalleeSavesBegin = MBBI, CalleeSavesEnd = MBBI;
2148
2149 // Process the SVE callee-saves to determine what space needs to be
2150 // allocated.
2151 if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
2152 LLVM_DEBUG(dbgs() << "SVECalleeSavedStackSize = " << CalleeSavedSize
2153 << "\n");
2154 // Find callee save instructions in frame.
2155 CalleeSavesBegin = MBBI;
2156 assert(IsSVECalleeSave(CalleeSavesBegin) && "Unexpected instruction");
2157 while (IsSVECalleeSave(I: MBBI) && MBBI != MBB.getFirstTerminator())
2158 ++MBBI;
2159 CalleeSavesEnd = MBBI;
2160
2161 SVECalleeSavesSize = StackOffset::getScalable(Scalable: CalleeSavedSize);
2162 SVELocalsSize = SVEStackSize - SVECalleeSavesSize;
2163 }
2164
2165 // Allocate space for the callee saves (if any).
2166 StackOffset CFAOffset =
2167 StackOffset::getFixed(Fixed: (int64_t)MFI.getStackSize() - NumBytes);
2168 StackOffset LocalsSize = SVELocalsSize + StackOffset::getFixed(Fixed: NumBytes);
2169 allocateStackSpace(MBB, MBBI: CalleeSavesBegin, RealignmentPadding: 0, AllocSize: SVECalleeSavesSize, NeedsWinCFI: false,
2170 HasWinCFI: nullptr, EmitCFI: EmitAsyncCFI && !HasFP, InitialOffset: CFAOffset,
2171 FollowupAllocs: MFI.hasVarSizedObjects() || LocalsSize);
2172 CFAOffset += SVECalleeSavesSize;
2173
2174 if (EmitAsyncCFI)
2175 emitCalleeSavedSVELocations(MBB, MBBI: CalleeSavesEnd);
2176
2177 // Allocate space for the rest of the frame including SVE locals. Align the
2178 // stack as necessary.
2179 assert(!(canUseRedZone(MF) && NeedsRealignment) &&
2180 "Cannot use redzone with stack realignment");
2181 if (!canUseRedZone(MF)) {
2182 // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have
2183 // the correct value here, as NumBytes also includes padding bytes,
2184 // which shouldn't be counted here.
2185 allocateStackSpace(MBB, MBBI: CalleeSavesEnd, RealignmentPadding,
2186 AllocSize: SVELocalsSize + StackOffset::getFixed(Fixed: NumBytes),
2187 NeedsWinCFI, HasWinCFI: &HasWinCFI, EmitCFI: EmitAsyncCFI && !HasFP,
2188 InitialOffset: CFAOffset, FollowupAllocs: MFI.hasVarSizedObjects());
2189 }
2190
2191 // If we need a base pointer, set it up here. It's whatever the value of the
2192 // stack pointer is at this point. Any variable size objects will be allocated
2193 // after this, so we can still use the base pointer to reference locals.
2194 //
2195 // FIXME: Clarify FrameSetup flags here.
2196 // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is
2197 // needed.
2198 // For funclets the BP belongs to the containing function.
2199 if (!IsFunclet && RegInfo->hasBasePointer(MF)) {
2200 TII->copyPhysReg(MBB, MI: MBBI, DL, DestReg: RegInfo->getBaseRegister(), SrcReg: AArch64::SP,
2201 KillSrc: false);
2202 if (NeedsWinCFI) {
2203 HasWinCFI = true;
2204 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_Nop))
2205 .setMIFlag(MachineInstr::FrameSetup);
2206 }
2207 }
2208
2209 // The very last FrameSetup instruction indicates the end of prologue. Emit a
2210 // SEH opcode indicating the prologue end.
2211 if (NeedsWinCFI && HasWinCFI) {
2212 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_PrologEnd))
2213 .setMIFlag(MachineInstr::FrameSetup);
2214 }
2215
2216 // SEH funclets are passed the frame pointer in X1. If the parent
2217 // function uses the base register, then the base register is used
2218 // directly, and is not retrieved from X1.
2219 if (IsFunclet && F.hasPersonalityFn()) {
2220 EHPersonality Per = classifyEHPersonality(Pers: F.getPersonalityFn());
2221 if (isAsynchronousEHPersonality(Pers: Per)) {
2222 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: AArch64::FP)
2223 .addReg(RegNo: AArch64::X1)
2224 .setMIFlag(MachineInstr::FrameSetup);
2225 MBB.addLiveIn(PhysReg: AArch64::X1);
2226 }
2227 }
2228
2229 if (EmitCFI && !EmitAsyncCFI) {
2230 if (HasFP) {
2231 emitDefineCFAWithFP(MF, MBB, MBBI, DL, FixedObject);
2232 } else {
2233 StackOffset TotalSize =
2234 SVEStackSize + StackOffset::getFixed(Fixed: (int64_t)MFI.getStackSize());
2235 unsigned CFIIndex = MF.addFrameInst(Inst: createDefCFA(
2236 TRI: *RegInfo, /*FrameReg=*/AArch64::SP, /*Reg=*/AArch64::SP, Offset: TotalSize,
2237 /*LastAdjustmentWasScalable=*/false));
2238 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
2239 .addCFIIndex(CFIIndex)
2240 .setMIFlags(MachineInstr::FrameSetup);
2241 }
2242 emitCalleeSavedGPRLocations(MBB, MBBI);
2243 emitCalleeSavedSVELocations(MBB, MBBI);
2244 }
2245}
2246
2247static bool isFuncletReturnInstr(const MachineInstr &MI) {
2248 switch (MI.getOpcode()) {
2249 default:
2250 return false;
2251 case AArch64::CATCHRET:
2252 case AArch64::CLEANUPRET:
2253 return true;
2254 }
2255}
2256
2257void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
2258 MachineBasicBlock &MBB) const {
2259 MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
2260 MachineFrameInfo &MFI = MF.getFrameInfo();
2261 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
2262 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
2263 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
2264 DebugLoc DL;
2265 bool NeedsWinCFI = needsWinCFI(MF);
2266 bool EmitCFI = AFI->needsAsyncDwarfUnwindInfo(MF);
2267 bool HasWinCFI = false;
2268 bool IsFunclet = false;
2269
2270 if (MBB.end() != MBBI) {
2271 DL = MBBI->getDebugLoc();
2272 IsFunclet = isFuncletReturnInstr(MI: *MBBI);
2273 }
2274
2275 MachineBasicBlock::iterator EpilogStartI = MBB.end();
2276
2277 auto FinishingTouches = make_scope_exit(F: [&]() {
2278 if (AFI->shouldSignReturnAddress(MF)) {
2279 BuildMI(BB&: MBB, I: MBB.getFirstTerminator(), MIMD: DL,
2280 MCID: TII->get(Opcode: AArch64::PAUTH_EPILOGUE))
2281 .setMIFlag(MachineInstr::FrameDestroy);
2282 if (NeedsWinCFI)
2283 HasWinCFI = true; // AArch64PointerAuth pass will insert SEH_PACSignLR
2284 }
2285 if (AFI->needsShadowCallStackPrologueEpilogue(MF))
2286 emitShadowCallStackEpilogue(TII: *TII, MF, MBB, MBBI: MBB.getFirstTerminator(), DL);
2287 if (EmitCFI)
2288 emitCalleeSavedGPRRestores(MBB, MBBI: MBB.getFirstTerminator());
2289 if (HasWinCFI) {
2290 BuildMI(BB&: MBB, I: MBB.getFirstTerminator(), MIMD: DL,
2291 MCID: TII->get(Opcode: AArch64::SEH_EpilogEnd))
2292 .setMIFlag(MachineInstr::FrameDestroy);
2293 if (!MF.hasWinCFI())
2294 MF.setHasWinCFI(true);
2295 }
2296 if (NeedsWinCFI) {
2297 assert(EpilogStartI != MBB.end());
2298 if (!HasWinCFI)
2299 MBB.erase(I: EpilogStartI);
2300 }
2301 });
2302
2303 int64_t NumBytes = IsFunclet ? getWinEHFuncletFrameSize(MF)
2304 : MFI.getStackSize();
2305
2306 // All calls are tail calls in GHC calling conv, and functions have no
2307 // prologue/epilogue.
2308 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
2309 return;
2310
2311 // How much of the stack used by incoming arguments this function is expected
2312 // to restore in this particular epilogue.
2313 int64_t ArgumentStackToRestore = getArgumentStackToRestore(MF, MBB);
2314 bool IsWin64 = Subtarget.isCallingConvWin64(CC: MF.getFunction().getCallingConv(),
2315 IsVarArg: MF.getFunction().isVarArg());
2316 unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
2317
2318 int64_t AfterCSRPopSize = ArgumentStackToRestore;
2319 auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
2320 // We cannot rely on the local stack size set in emitPrologue if the function
2321 // has funclets, as funclets have different local stack size requirements, and
2322 // the current value set in emitPrologue may be that of the containing
2323 // function.
2324 if (MF.hasEHFunclets())
2325 AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
2326 if (homogeneousPrologEpilog(MF, Exit: &MBB)) {
2327 assert(!NeedsWinCFI);
2328 auto LastPopI = MBB.getFirstTerminator();
2329 if (LastPopI != MBB.begin()) {
2330 auto HomogeneousEpilog = std::prev(x: LastPopI);
2331 if (HomogeneousEpilog->getOpcode() == AArch64::HOM_Epilog)
2332 LastPopI = HomogeneousEpilog;
2333 }
2334
2335 // Adjust local stack
2336 emitFrameOffset(MBB, MBBI: LastPopI, DL, DestReg: AArch64::SP, SrcReg: AArch64::SP,
2337 Offset: StackOffset::getFixed(Fixed: AFI->getLocalStackSize()), TII,
2338 MachineInstr::FrameDestroy, SetNZCV: false, NeedsWinCFI, HasWinCFI: &HasWinCFI);
2339
2340 // SP has been already adjusted while restoring callee save regs.
2341 // We've bailed-out the case with adjusting SP for arguments.
2342 assert(AfterCSRPopSize == 0);
2343 return;
2344 }
2345 bool CombineSPBump = shouldCombineCSRLocalStackBumpInEpilogue(MBB, StackBumpBytes: NumBytes);
2346 // Assume we can't combine the last pop with the sp restore.
2347
2348 bool CombineAfterCSRBump = false;
2349 if (!CombineSPBump && PrologueSaveSize != 0) {
2350 MachineBasicBlock::iterator Pop = std::prev(x: MBB.getFirstTerminator());
2351 while (Pop->getOpcode() == TargetOpcode::CFI_INSTRUCTION ||
2352 AArch64InstrInfo::isSEHInstruction(MI: *Pop))
2353 Pop = std::prev(x: Pop);
2354 // Converting the last ldp to a post-index ldp is valid only if the last
2355 // ldp's offset is 0.
2356 const MachineOperand &OffsetOp = Pop->getOperand(i: Pop->getNumOperands() - 1);
2357 // If the offset is 0 and the AfterCSR pop is not actually trying to
2358 // allocate more stack for arguments (in space that an untimely interrupt
2359 // may clobber), convert it to a post-index ldp.
2360 if (OffsetOp.getImm() == 0 && AfterCSRPopSize >= 0) {
2361 convertCalleeSaveRestoreToSPPrePostIncDec(
2362 MBB, MBBI: Pop, DL, TII, CSStackSizeInc: PrologueSaveSize, NeedsWinCFI, HasWinCFI: &HasWinCFI, EmitCFI,
2363 FrameFlag: MachineInstr::FrameDestroy, CFAOffset: PrologueSaveSize);
2364 } else {
2365 // If not, make sure to emit an add after the last ldp.
2366 // We're doing this by transfering the size to be restored from the
2367 // adjustment *before* the CSR pops to the adjustment *after* the CSR
2368 // pops.
2369 AfterCSRPopSize += PrologueSaveSize;
2370 CombineAfterCSRBump = true;
2371 }
2372 }
2373
2374 // Move past the restores of the callee-saved registers.
2375 // If we plan on combining the sp bump of the local stack size and the callee
2376 // save stack size, we might need to adjust the CSR save and restore offsets.
2377 MachineBasicBlock::iterator LastPopI = MBB.getFirstTerminator();
2378 MachineBasicBlock::iterator Begin = MBB.begin();
2379 while (LastPopI != Begin) {
2380 --LastPopI;
2381 if (!LastPopI->getFlag(Flag: MachineInstr::FrameDestroy) ||
2382 IsSVECalleeSave(I: LastPopI)) {
2383 ++LastPopI;
2384 break;
2385 } else if (CombineSPBump)
2386 fixupCalleeSaveRestoreStackOffset(MI&: *LastPopI, LocalStackSize: AFI->getLocalStackSize(),
2387 NeedsWinCFI, HasWinCFI: &HasWinCFI);
2388 }
2389
2390 if (NeedsWinCFI) {
2391 // Note that there are cases where we insert SEH opcodes in the
2392 // epilogue when we had no SEH opcodes in the prologue. For
2393 // example, when there is no stack frame but there are stack
2394 // arguments. Insert the SEH_EpilogStart and remove it later if it
2395 // we didn't emit any SEH opcodes to avoid generating WinCFI for
2396 // functions that don't need it.
2397 BuildMI(BB&: MBB, I: LastPopI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_EpilogStart))
2398 .setMIFlag(MachineInstr::FrameDestroy);
2399 EpilogStartI = LastPopI;
2400 --EpilogStartI;
2401 }
2402
2403 if (hasFP(MF) && AFI->hasSwiftAsyncContext()) {
2404 switch (MF.getTarget().Options.SwiftAsyncFramePointer) {
2405 case SwiftAsyncFramePointerMode::DeploymentBased:
2406 // Avoid the reload as it is GOT relative, and instead fall back to the
2407 // hardcoded value below. This allows a mismatch between the OS and
2408 // application without immediately terminating on the difference.
2409 [[fallthrough]];
2410 case SwiftAsyncFramePointerMode::Always:
2411 // We need to reset FP to its untagged state on return. Bit 60 is
2412 // currently used to show the presence of an extended frame.
2413
2414 // BIC x29, x29, #0x1000_0000_0000_0000
2415 BuildMI(BB&: MBB, I: MBB.getFirstTerminator(), MIMD: DL, MCID: TII->get(Opcode: AArch64::ANDXri),
2416 DestReg: AArch64::FP)
2417 .addUse(RegNo: AArch64::FP)
2418 .addImm(Val: 0x10fe)
2419 .setMIFlag(MachineInstr::FrameDestroy);
2420 if (NeedsWinCFI) {
2421 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_Nop))
2422 .setMIFlags(MachineInstr::FrameDestroy);
2423 HasWinCFI = true;
2424 }
2425 break;
2426
2427 case SwiftAsyncFramePointerMode::Never:
2428 break;
2429 }
2430 }
2431
2432 const StackOffset &SVEStackSize = getSVEStackSize(MF);
2433
2434 // If there is a single SP update, insert it before the ret and we're done.
2435 if (CombineSPBump) {
2436 assert(!SVEStackSize && "Cannot combine SP bump with SVE");
2437
2438 // When we are about to restore the CSRs, the CFA register is SP again.
2439 if (EmitCFI && hasFP(MF)) {
2440 const AArch64RegisterInfo &RegInfo = *Subtarget.getRegisterInfo();
2441 unsigned Reg = RegInfo.getDwarfRegNum(RegNum: AArch64::SP, isEH: true);
2442 unsigned CFIIndex =
2443 MF.addFrameInst(Inst: MCCFIInstruction::cfiDefCfa(L: nullptr, Register: Reg, Offset: NumBytes));
2444 BuildMI(BB&: MBB, I: LastPopI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
2445 .addCFIIndex(CFIIndex)
2446 .setMIFlags(MachineInstr::FrameDestroy);
2447 }
2448
2449 emitFrameOffset(MBB, MBBI: MBB.getFirstTerminator(), DL, DestReg: AArch64::SP, SrcReg: AArch64::SP,
2450 Offset: StackOffset::getFixed(Fixed: NumBytes + (int64_t)AfterCSRPopSize),
2451 TII, MachineInstr::FrameDestroy, SetNZCV: false, NeedsWinCFI,
2452 HasWinCFI: &HasWinCFI, EmitCFAOffset: EmitCFI, InitialOffset: StackOffset::getFixed(Fixed: NumBytes));
2453 return;
2454 }
2455
2456 NumBytes -= PrologueSaveSize;
2457 assert(NumBytes >= 0 && "Negative stack allocation size!?");
2458
2459 // Process the SVE callee-saves to determine what space needs to be
2460 // deallocated.
2461 StackOffset DeallocateBefore = {}, DeallocateAfter = SVEStackSize;
2462 MachineBasicBlock::iterator RestoreBegin = LastPopI, RestoreEnd = LastPopI;
2463 if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
2464 RestoreBegin = std::prev(x: RestoreEnd);
2465 while (RestoreBegin != MBB.begin() &&
2466 IsSVECalleeSave(I: std::prev(x: RestoreBegin)))
2467 --RestoreBegin;
2468
2469 assert(IsSVECalleeSave(RestoreBegin) &&
2470 IsSVECalleeSave(std::prev(RestoreEnd)) && "Unexpected instruction");
2471
2472 StackOffset CalleeSavedSizeAsOffset =
2473 StackOffset::getScalable(Scalable: CalleeSavedSize);
2474 DeallocateBefore = SVEStackSize - CalleeSavedSizeAsOffset;
2475 DeallocateAfter = CalleeSavedSizeAsOffset;
2476 }
2477
2478 // Deallocate the SVE area.
2479 if (SVEStackSize) {
2480 // If we have stack realignment or variable sized objects on the stack,
2481 // restore the stack pointer from the frame pointer prior to SVE CSR
2482 // restoration.
2483 if (AFI->isStackRealigned() || MFI.hasVarSizedObjects()) {
2484 if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
2485 // Set SP to start of SVE callee-save area from which they can
2486 // be reloaded. The code below will deallocate the stack space
2487 // space by moving FP -> SP.
2488 emitFrameOffset(MBB, MBBI: RestoreBegin, DL, DestReg: AArch64::SP, SrcReg: AArch64::FP,
2489 Offset: StackOffset::getScalable(Scalable: -CalleeSavedSize), TII,
2490 MachineInstr::FrameDestroy);
2491 }
2492 } else {
2493 if (AFI->getSVECalleeSavedStackSize()) {
2494 // Deallocate the non-SVE locals first before we can deallocate (and
2495 // restore callee saves) from the SVE area.
2496 emitFrameOffset(
2497 MBB, MBBI: RestoreBegin, DL, DestReg: AArch64::SP, SrcReg: AArch64::SP,
2498 Offset: StackOffset::getFixed(Fixed: NumBytes), TII, MachineInstr::FrameDestroy,
2499 SetNZCV: false, NeedsWinCFI: false, HasWinCFI: nullptr, EmitCFAOffset: EmitCFI && !hasFP(MF),
2500 InitialOffset: SVEStackSize + StackOffset::getFixed(Fixed: NumBytes + PrologueSaveSize));
2501 NumBytes = 0;
2502 }
2503
2504 emitFrameOffset(MBB, MBBI: RestoreBegin, DL, DestReg: AArch64::SP, SrcReg: AArch64::SP,
2505 Offset: DeallocateBefore, TII, MachineInstr::FrameDestroy, SetNZCV: false,
2506 NeedsWinCFI: false, HasWinCFI: nullptr, EmitCFAOffset: EmitCFI && !hasFP(MF),
2507 InitialOffset: SVEStackSize +
2508 StackOffset::getFixed(Fixed: NumBytes + PrologueSaveSize));
2509
2510 emitFrameOffset(MBB, MBBI: RestoreEnd, DL, DestReg: AArch64::SP, SrcReg: AArch64::SP,
2511 Offset: DeallocateAfter, TII, MachineInstr::FrameDestroy, SetNZCV: false,
2512 NeedsWinCFI: false, HasWinCFI: nullptr, EmitCFAOffset: EmitCFI && !hasFP(MF),
2513 InitialOffset: DeallocateAfter +
2514 StackOffset::getFixed(Fixed: NumBytes + PrologueSaveSize));
2515 }
2516 if (EmitCFI)
2517 emitCalleeSavedSVERestores(MBB, MBBI: RestoreEnd);
2518 }
2519
2520 if (!hasFP(MF)) {
2521 bool RedZone = canUseRedZone(MF);
2522 // If this was a redzone leaf function, we don't need to restore the
2523 // stack pointer (but we may need to pop stack args for fastcc).
2524 if (RedZone && AfterCSRPopSize == 0)
2525 return;
2526
2527 // Pop the local variables off the stack. If there are no callee-saved
2528 // registers, it means we are actually positioned at the terminator and can
2529 // combine stack increment for the locals and the stack increment for
2530 // callee-popped arguments into (possibly) a single instruction and be done.
2531 bool NoCalleeSaveRestore = PrologueSaveSize == 0;
2532 int64_t StackRestoreBytes = RedZone ? 0 : NumBytes;
2533 if (NoCalleeSaveRestore)
2534 StackRestoreBytes += AfterCSRPopSize;
2535
2536 emitFrameOffset(
2537 MBB, MBBI: LastPopI, DL, DestReg: AArch64::SP, SrcReg: AArch64::SP,
2538 Offset: StackOffset::getFixed(Fixed: StackRestoreBytes), TII,
2539 MachineInstr::FrameDestroy, SetNZCV: false, NeedsWinCFI, HasWinCFI: &HasWinCFI, EmitCFAOffset: EmitCFI,
2540 InitialOffset: StackOffset::getFixed(Fixed: (RedZone ? 0 : NumBytes) + PrologueSaveSize));
2541
2542 // If we were able to combine the local stack pop with the argument pop,
2543 // then we're done.
2544 if (NoCalleeSaveRestore || AfterCSRPopSize == 0) {
2545 return;
2546 }
2547
2548 NumBytes = 0;
2549 }
2550
2551 // Restore the original stack pointer.
2552 // FIXME: Rather than doing the math here, we should instead just use
2553 // non-post-indexed loads for the restores if we aren't actually going to
2554 // be able to save any instructions.
2555 if (!IsFunclet && (MFI.hasVarSizedObjects() || AFI->isStackRealigned())) {
2556 emitFrameOffset(
2557 MBB, MBBI: LastPopI, DL, DestReg: AArch64::SP, SrcReg: AArch64::FP,
2558 Offset: StackOffset::getFixed(Fixed: -AFI->getCalleeSaveBaseToFrameRecordOffset()),
2559 TII, MachineInstr::FrameDestroy, SetNZCV: false, NeedsWinCFI, HasWinCFI: &HasWinCFI);
2560 } else if (NumBytes)
2561 emitFrameOffset(MBB, MBBI: LastPopI, DL, DestReg: AArch64::SP, SrcReg: AArch64::SP,
2562 Offset: StackOffset::getFixed(Fixed: NumBytes), TII,
2563 MachineInstr::FrameDestroy, SetNZCV: false, NeedsWinCFI, HasWinCFI: &HasWinCFI);
2564
2565 // When we are about to restore the CSRs, the CFA register is SP again.
2566 if (EmitCFI && hasFP(MF)) {
2567 const AArch64RegisterInfo &RegInfo = *Subtarget.getRegisterInfo();
2568 unsigned Reg = RegInfo.getDwarfRegNum(RegNum: AArch64::SP, isEH: true);
2569 unsigned CFIIndex = MF.addFrameInst(
2570 Inst: MCCFIInstruction::cfiDefCfa(L: nullptr, Register: Reg, Offset: PrologueSaveSize));
2571 BuildMI(BB&: MBB, I: LastPopI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
2572 .addCFIIndex(CFIIndex)
2573 .setMIFlags(MachineInstr::FrameDestroy);
2574 }
2575
2576 // This must be placed after the callee-save restore code because that code
2577 // assumes the SP is at the same location as it was after the callee-save save
2578 // code in the prologue.
2579 if (AfterCSRPopSize) {
2580 assert(AfterCSRPopSize > 0 && "attempting to reallocate arg stack that an "
2581 "interrupt may have clobbered");
2582
2583 emitFrameOffset(
2584 MBB, MBBI: MBB.getFirstTerminator(), DL, DestReg: AArch64::SP, SrcReg: AArch64::SP,
2585 Offset: StackOffset::getFixed(Fixed: AfterCSRPopSize), TII, MachineInstr::FrameDestroy,
2586 SetNZCV: false, NeedsWinCFI, HasWinCFI: &HasWinCFI, EmitCFAOffset: EmitCFI,
2587 InitialOffset: StackOffset::getFixed(Fixed: CombineAfterCSRBump ? PrologueSaveSize : 0));
2588 }
2589}
2590
2591bool AArch64FrameLowering::enableCFIFixup(MachineFunction &MF) const {
2592 return TargetFrameLowering::enableCFIFixup(MF) &&
2593 MF.getInfo<AArch64FunctionInfo>()->needsAsyncDwarfUnwindInfo(MF);
2594}
2595
2596/// getFrameIndexReference - Provide a base+offset reference to an FI slot for
2597/// debug info. It's the same as what we use for resolving the code-gen
2598/// references for now. FIXME: This can go wrong when references are
2599/// SP-relative and simple call frames aren't used.
2600StackOffset
2601AArch64FrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
2602 Register &FrameReg) const {
2603 return resolveFrameIndexReference(
2604 MF, FI, FrameReg,
2605 /*PreferFP=*/
2606 MF.getFunction().hasFnAttribute(Kind: Attribute::SanitizeHWAddress) ||
2607 MF.getFunction().hasFnAttribute(Kind: Attribute::SanitizeMemTag),
2608 /*ForSimm=*/false);
2609}
2610
2611StackOffset
2612AArch64FrameLowering::getFrameIndexReferenceFromSP(const MachineFunction &MF,
2613 int FI) const {
2614 // This function serves to provide a comparable offset from a single reference
2615 // point (the value of SP at function entry) that can be used for analysis,
2616 // e.g. the stack-frame-layout analysis pass. It is not guaranteed to be
2617 // correct for all objects in the presence of VLA-area objects or dynamic
2618 // stack re-alignment.
2619
2620 const auto &MFI = MF.getFrameInfo();
2621
2622 int64_t ObjectOffset = MFI.getObjectOffset(ObjectIdx: FI);
2623 StackOffset SVEStackSize = getSVEStackSize(MF);
2624
2625 // For VLA-area objects, just emit an offset at the end of the stack frame.
2626 // Whilst not quite correct, these objects do live at the end of the frame and
2627 // so it is more useful for analysis for the offset to reflect this.
2628 if (MFI.isVariableSizedObjectIndex(ObjectIdx: FI)) {
2629 return StackOffset::getFixed(Fixed: -((int64_t)MFI.getStackSize())) - SVEStackSize;
2630 }
2631
2632 // This is correct in the absence of any SVE stack objects.
2633 if (!SVEStackSize)
2634 return StackOffset::getFixed(Fixed: ObjectOffset - getOffsetOfLocalArea());
2635
2636 const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
2637 if (MFI.getStackID(ObjectIdx: FI) == TargetStackID::ScalableVector) {
2638 return StackOffset::get(Fixed: -((int64_t)AFI->getCalleeSavedStackSize()),
2639 Scalable: ObjectOffset);
2640 }
2641
2642 bool IsFixed = MFI.isFixedObjectIndex(ObjectIdx: FI);
2643 bool IsCSR =
2644 !IsFixed && ObjectOffset >= -((int)AFI->getCalleeSavedStackSize(MFI));
2645
2646 StackOffset ScalableOffset = {};
2647 if (!IsFixed && !IsCSR)
2648 ScalableOffset = -SVEStackSize;
2649
2650 return StackOffset::getFixed(Fixed: ObjectOffset) + ScalableOffset;
2651}
2652
2653StackOffset
2654AArch64FrameLowering::getNonLocalFrameIndexReference(const MachineFunction &MF,
2655 int FI) const {
2656 return StackOffset::getFixed(Fixed: getSEHFrameIndexOffset(MF, FI));
2657}
2658
2659static StackOffset getFPOffset(const MachineFunction &MF,
2660 int64_t ObjectOffset) {
2661 const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
2662 const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
2663 const Function &F = MF.getFunction();
2664 bool IsWin64 = Subtarget.isCallingConvWin64(CC: F.getCallingConv(), IsVarArg: F.isVarArg());
2665 unsigned FixedObject =
2666 getFixedObjectSize(MF, AFI, IsWin64, /*IsFunclet=*/false);
2667 int64_t CalleeSaveSize = AFI->getCalleeSavedStackSize(MFI: MF.getFrameInfo());
2668 int64_t FPAdjust =
2669 CalleeSaveSize - AFI->getCalleeSaveBaseToFrameRecordOffset();
2670 return StackOffset::getFixed(Fixed: ObjectOffset + FixedObject + FPAdjust);
2671}
2672
2673static StackOffset getStackOffset(const MachineFunction &MF,
2674 int64_t ObjectOffset) {
2675 const auto &MFI = MF.getFrameInfo();
2676 return StackOffset::getFixed(Fixed: ObjectOffset + (int64_t)MFI.getStackSize());
2677}
2678
2679// TODO: This function currently does not work for scalable vectors.
2680int AArch64FrameLowering::getSEHFrameIndexOffset(const MachineFunction &MF,
2681 int FI) const {
2682 const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
2683 MF.getSubtarget().getRegisterInfo());
2684 int ObjectOffset = MF.getFrameInfo().getObjectOffset(ObjectIdx: FI);
2685 return RegInfo->getLocalAddressRegister(MF) == AArch64::FP
2686 ? getFPOffset(MF, ObjectOffset).getFixed()
2687 : getStackOffset(MF, ObjectOffset).getFixed();
2688}
2689
2690StackOffset AArch64FrameLowering::resolveFrameIndexReference(
2691 const MachineFunction &MF, int FI, Register &FrameReg, bool PreferFP,
2692 bool ForSimm) const {
2693 const auto &MFI = MF.getFrameInfo();
2694 int64_t ObjectOffset = MFI.getObjectOffset(ObjectIdx: FI);
2695 bool isFixed = MFI.isFixedObjectIndex(ObjectIdx: FI);
2696 bool isSVE = MFI.getStackID(ObjectIdx: FI) == TargetStackID::ScalableVector;
2697 return resolveFrameOffsetReference(MF, ObjectOffset, isFixed, isSVE, FrameReg,
2698 PreferFP, ForSimm);
2699}
2700
2701StackOffset AArch64FrameLowering::resolveFrameOffsetReference(
2702 const MachineFunction &MF, int64_t ObjectOffset, bool isFixed, bool isSVE,
2703 Register &FrameReg, bool PreferFP, bool ForSimm) const {
2704 const auto &MFI = MF.getFrameInfo();
2705 const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
2706 MF.getSubtarget().getRegisterInfo());
2707 const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
2708 const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
2709
2710 int64_t FPOffset = getFPOffset(MF, ObjectOffset).getFixed();
2711 int64_t Offset = getStackOffset(MF, ObjectOffset).getFixed();
2712 bool isCSR =
2713 !isFixed && ObjectOffset >= -((int)AFI->getCalleeSavedStackSize(MFI));
2714
2715 const StackOffset &SVEStackSize = getSVEStackSize(MF);
2716
2717 // Use frame pointer to reference fixed objects. Use it for locals if
2718 // there are VLAs or a dynamically realigned SP (and thus the SP isn't
2719 // reliable as a base). Make sure useFPForScavengingIndex() does the
2720 // right thing for the emergency spill slot.
2721 bool UseFP = false;
2722 if (AFI->hasStackFrame() && !isSVE) {
2723 // We shouldn't prefer using the FP to access fixed-sized stack objects when
2724 // there are scalable (SVE) objects in between the FP and the fixed-sized
2725 // objects.
2726 PreferFP &= !SVEStackSize;
2727
2728 // Note: Keeping the following as multiple 'if' statements rather than
2729 // merging to a single expression for readability.
2730 //
2731 // Argument access should always use the FP.
2732 if (isFixed) {
2733 UseFP = hasFP(MF);
2734 } else if (isCSR && RegInfo->hasStackRealignment(MF)) {
2735 // References to the CSR area must use FP if we're re-aligning the stack
2736 // since the dynamically-sized alignment padding is between the SP/BP and
2737 // the CSR area.
2738 assert(hasFP(MF) && "Re-aligned stack must have frame pointer");
2739 UseFP = true;
2740 } else if (hasFP(MF) && !RegInfo->hasStackRealignment(MF)) {
2741 // If the FPOffset is negative and we're producing a signed immediate, we
2742 // have to keep in mind that the available offset range for negative
2743 // offsets is smaller than for positive ones. If an offset is available
2744 // via the FP and the SP, use whichever is closest.
2745 bool FPOffsetFits = !ForSimm || FPOffset >= -256;
2746 PreferFP |= Offset > -FPOffset && !SVEStackSize;
2747
2748 if (MFI.hasVarSizedObjects()) {
2749 // If we have variable sized objects, we can use either FP or BP, as the
2750 // SP offset is unknown. We can use the base pointer if we have one and
2751 // FP is not preferred. If not, we're stuck with using FP.
2752 bool CanUseBP = RegInfo->hasBasePointer(MF);
2753 if (FPOffsetFits && CanUseBP) // Both are ok. Pick the best.
2754 UseFP = PreferFP;
2755 else if (!CanUseBP) // Can't use BP. Forced to use FP.
2756 UseFP = true;
2757 // else we can use BP and FP, but the offset from FP won't fit.
2758 // That will make us scavenge registers which we can probably avoid by
2759 // using BP. If it won't fit for BP either, we'll scavenge anyway.
2760 } else if (FPOffset >= 0) {
2761 // Use SP or FP, whichever gives us the best chance of the offset
2762 // being in range for direct access. If the FPOffset is positive,
2763 // that'll always be best, as the SP will be even further away.
2764 UseFP = true;
2765 } else if (MF.hasEHFunclets() && !RegInfo->hasBasePointer(MF)) {
2766 // Funclets access the locals contained in the parent's stack frame
2767 // via the frame pointer, so we have to use the FP in the parent
2768 // function.
2769 (void) Subtarget;
2770 assert(Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv(),
2771 MF.getFunction().isVarArg()) &&
2772 "Funclets should only be present on Win64");
2773 UseFP = true;
2774 } else {
2775 // We have the choice between FP and (SP or BP).
2776 if (FPOffsetFits && PreferFP) // If FP is the best fit, use it.
2777 UseFP = true;
2778 }
2779 }
2780 }
2781
2782 assert(
2783 ((isFixed || isCSR) || !RegInfo->hasStackRealignment(MF) || !UseFP) &&
2784 "In the presence of dynamic stack pointer realignment, "
2785 "non-argument/CSR objects cannot be accessed through the frame pointer");
2786
2787 if (isSVE) {
2788 StackOffset FPOffset =
2789 StackOffset::get(Fixed: -AFI->getCalleeSaveBaseToFrameRecordOffset(), Scalable: ObjectOffset);
2790 StackOffset SPOffset =
2791 SVEStackSize +
2792 StackOffset::get(Fixed: MFI.getStackSize() - AFI->getCalleeSavedStackSize(),
2793 Scalable: ObjectOffset);
2794 // Always use the FP for SVE spills if available and beneficial.
2795 if (hasFP(MF) && (SPOffset.getFixed() ||
2796 FPOffset.getScalable() < SPOffset.getScalable() ||
2797 RegInfo->hasStackRealignment(MF))) {
2798 FrameReg = RegInfo->getFrameRegister(MF);
2799 return FPOffset;
2800 }
2801
2802 FrameReg = RegInfo->hasBasePointer(MF) ? RegInfo->getBaseRegister()
2803 : (unsigned)AArch64::SP;
2804 return SPOffset;
2805 }
2806
2807 StackOffset ScalableOffset = {};
2808 if (UseFP && !(isFixed || isCSR))
2809 ScalableOffset = -SVEStackSize;
2810 if (!UseFP && (isFixed || isCSR))
2811 ScalableOffset = SVEStackSize;
2812
2813 if (UseFP) {
2814 FrameReg = RegInfo->getFrameRegister(MF);
2815 return StackOffset::getFixed(Fixed: FPOffset) + ScalableOffset;
2816 }
2817
2818 // Use the base pointer if we have one.
2819 if (RegInfo->hasBasePointer(MF))
2820 FrameReg = RegInfo->getBaseRegister();
2821 else {
2822 assert(!MFI.hasVarSizedObjects() &&
2823 "Can't use SP when we have var sized objects.");
2824 FrameReg = AArch64::SP;
2825 // If we're using the red zone for this function, the SP won't actually
2826 // be adjusted, so the offsets will be negative. They're also all
2827 // within range of the signed 9-bit immediate instructions.
2828 if (canUseRedZone(MF))
2829 Offset -= AFI->getLocalStackSize();
2830 }
2831
2832 return StackOffset::getFixed(Fixed: Offset) + ScalableOffset;
2833}
2834
2835static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg) {
2836 // Do not set a kill flag on values that are also marked as live-in. This
2837 // happens with the @llvm-returnaddress intrinsic and with arguments passed in
2838 // callee saved registers.
2839 // Omitting the kill flags is conservatively correct even if the live-in
2840 // is not used after all.
2841 bool IsLiveIn = MF.getRegInfo().isLiveIn(Reg);
2842 return getKillRegState(B: !IsLiveIn);
2843}
2844
2845static bool produceCompactUnwindFrame(MachineFunction &MF) {
2846 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
2847 AttributeList Attrs = MF.getFunction().getAttributes();
2848 return Subtarget.isTargetMachO() &&
2849 !(Subtarget.getTargetLowering()->supportSwiftError() &&
2850 Attrs.hasAttrSomewhere(Kind: Attribute::SwiftError)) &&
2851 MF.getFunction().getCallingConv() != CallingConv::SwiftTail;
2852}
2853
2854static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2,
2855 bool NeedsWinCFI, bool IsFirst,
2856 const TargetRegisterInfo *TRI) {
2857 // If we are generating register pairs for a Windows function that requires
2858 // EH support, then pair consecutive registers only. There are no unwind
2859 // opcodes for saves/restores of non-consectuve register pairs.
2860 // The unwind opcodes are save_regp, save_regp_x, save_fregp, save_frepg_x,
2861 // save_lrpair.
2862 // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
2863
2864 if (Reg2 == AArch64::FP)
2865 return true;
2866 if (!NeedsWinCFI)
2867 return false;
2868 if (TRI->getEncodingValue(RegNo: Reg2) == TRI->getEncodingValue(RegNo: Reg1) + 1)
2869 return false;
2870 // If pairing a GPR with LR, the pair can be described by the save_lrpair
2871 // opcode. If this is the first register pair, it would end up with a
2872 // predecrement, but there's no save_lrpair_x opcode, so we can only do this
2873 // if LR is paired with something else than the first register.
2874 // The save_lrpair opcode requires the first register to be an odd one.
2875 if (Reg1 >= AArch64::X19 && Reg1 <= AArch64::X27 &&
2876 (Reg1 - AArch64::X19) % 2 == 0 && Reg2 == AArch64::LR && !IsFirst)
2877 return false;
2878 return true;
2879}
2880
2881/// Returns true if Reg1 and Reg2 cannot be paired using a ldp/stp instruction.
2882/// WindowsCFI requires that only consecutive registers can be paired.
2883/// LR and FP need to be allocated together when the frame needs to save
2884/// the frame-record. This means any other register pairing with LR is invalid.
2885static bool invalidateRegisterPairing(unsigned Reg1, unsigned Reg2,
2886 bool UsesWinAAPCS, bool NeedsWinCFI,
2887 bool NeedsFrameRecord, bool IsFirst,
2888 const TargetRegisterInfo *TRI) {
2889 if (UsesWinAAPCS)
2890 return invalidateWindowsRegisterPairing(Reg1, Reg2, NeedsWinCFI, IsFirst,
2891 TRI);
2892
2893 // If we need to store the frame record, don't pair any register
2894 // with LR other than FP.
2895 if (NeedsFrameRecord)
2896 return Reg2 == AArch64::LR;
2897
2898 return false;
2899}
2900
2901namespace {
2902
2903struct RegPairInfo {
2904 unsigned Reg1 = AArch64::NoRegister;
2905 unsigned Reg2 = AArch64::NoRegister;
2906 int FrameIdx;
2907 int Offset;
2908 enum RegType { GPR, FPR64, FPR128, PPR, ZPR, VG } Type;
2909
2910 RegPairInfo() = default;
2911
2912 bool isPaired() const { return Reg2 != AArch64::NoRegister; }
2913
2914 unsigned getScale() const {
2915 switch (Type) {
2916 case PPR:
2917 return 2;
2918 case GPR:
2919 case FPR64:
2920 case VG:
2921 return 8;
2922 case ZPR:
2923 case FPR128:
2924 return 16;
2925 }
2926 llvm_unreachable("Unsupported type");
2927 }
2928
2929 bool isScalable() const { return Type == PPR || Type == ZPR; }
2930};
2931
2932} // end anonymous namespace
2933
2934static void computeCalleeSaveRegisterPairs(
2935 MachineFunction &MF, ArrayRef<CalleeSavedInfo> CSI,
2936 const TargetRegisterInfo *TRI, SmallVectorImpl<RegPairInfo> &RegPairs,
2937 bool NeedsFrameRecord) {
2938
2939 if (CSI.empty())
2940 return;
2941
2942 bool IsWindows = isTargetWindows(MF);
2943 bool NeedsWinCFI = needsWinCFI(MF);
2944 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
2945 MachineFrameInfo &MFI = MF.getFrameInfo();
2946 CallingConv::ID CC = MF.getFunction().getCallingConv();
2947 unsigned Count = CSI.size();
2948 (void)CC;
2949 // MachO's compact unwind format relies on all registers being stored in
2950 // pairs.
2951 assert((!produceCompactUnwindFrame(MF) || CC == CallingConv::PreserveMost ||
2952 CC == CallingConv::PreserveAll || CC == CallingConv::CXX_FAST_TLS ||
2953 CC == CallingConv::Win64 || (Count & 1) == 0) &&
2954 "Odd number of callee-saved regs to spill!");
2955 int ByteOffset = AFI->getCalleeSavedStackSize();
2956 int StackFillDir = -1;
2957 int RegInc = 1;
2958 unsigned FirstReg = 0;
2959 if (NeedsWinCFI) {
2960 // For WinCFI, fill the stack from the bottom up.
2961 ByteOffset = 0;
2962 StackFillDir = 1;
2963 // As the CSI array is reversed to match PrologEpilogInserter, iterate
2964 // backwards, to pair up registers starting from lower numbered registers.
2965 RegInc = -1;
2966 FirstReg = Count - 1;
2967 }
2968 int ScalableByteOffset = AFI->getSVECalleeSavedStackSize();
2969 bool NeedGapToAlignStack = AFI->hasCalleeSaveStackFreeSpace();
2970 Register LastReg = 0;
2971
2972 // When iterating backwards, the loop condition relies on unsigned wraparound.
2973 for (unsigned i = FirstReg; i < Count; i += RegInc) {
2974 RegPairInfo RPI;
2975 RPI.Reg1 = CSI[i].getReg();
2976
2977 if (AArch64::GPR64RegClass.contains(Reg: RPI.Reg1))
2978 RPI.Type = RegPairInfo::GPR;
2979 else if (AArch64::FPR64RegClass.contains(Reg: RPI.Reg1))
2980 RPI.Type = RegPairInfo::FPR64;
2981 else if (AArch64::FPR128RegClass.contains(Reg: RPI.Reg1))
2982 RPI.Type = RegPairInfo::FPR128;
2983 else if (AArch64::ZPRRegClass.contains(Reg: RPI.Reg1))
2984 RPI.Type = RegPairInfo::ZPR;
2985 else if (AArch64::PPRRegClass.contains(Reg: RPI.Reg1))
2986 RPI.Type = RegPairInfo::PPR;
2987 else if (RPI.Reg1 == AArch64::VG)
2988 RPI.Type = RegPairInfo::VG;
2989 else
2990 llvm_unreachable("Unsupported register class.");
2991
2992 // Add the stack hazard size as we transition from GPR->FPR CSRs.
2993 if (AFI->hasStackHazardSlotIndex() &&
2994 (!LastReg || !AArch64InstrInfo::isFpOrNEON(Reg: LastReg)) &&
2995 AArch64InstrInfo::isFpOrNEON(Reg: RPI.Reg1))
2996 ByteOffset += StackFillDir * StackHazardSize;
2997 LastReg = RPI.Reg1;
2998
2999 // Add the next reg to the pair if it is in the same register class.
3000 if (unsigned(i + RegInc) < Count && !AFI->hasStackHazardSlotIndex()) {
3001 Register NextReg = CSI[i + RegInc].getReg();
3002 bool IsFirst = i == FirstReg;
3003 switch (RPI.Type) {
3004 case RegPairInfo::GPR:
3005 if (AArch64::GPR64RegClass.contains(Reg: NextReg) &&
3006 !invalidateRegisterPairing(Reg1: RPI.Reg1, Reg2: NextReg, UsesWinAAPCS: IsWindows,
3007 NeedsWinCFI, NeedsFrameRecord, IsFirst,
3008 TRI))
3009 RPI.Reg2 = NextReg;
3010 break;
3011 case RegPairInfo::FPR64:
3012 if (AArch64::FPR64RegClass.contains(Reg: NextReg) &&
3013 !invalidateWindowsRegisterPairing(Reg1: RPI.Reg1, Reg2: NextReg, NeedsWinCFI,
3014 IsFirst, TRI))
3015 RPI.Reg2 = NextReg;
3016 break;
3017 case RegPairInfo::FPR128:
3018 if (AArch64::FPR128RegClass.contains(Reg: NextReg))
3019 RPI.Reg2 = NextReg;
3020 break;
3021 case RegPairInfo::PPR:
3022 break;
3023 case RegPairInfo::ZPR:
3024 if (AFI->getPredicateRegForFillSpill() != 0)
3025 if (((RPI.Reg1 - AArch64::Z0) & 1) == 0 && (NextReg == RPI.Reg1 + 1))
3026 RPI.Reg2 = NextReg;
3027 break;
3028 case RegPairInfo::VG:
3029 break;
3030 }
3031 }
3032
3033 // GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI
3034 // list to come in sorted by frame index so that we can issue the store
3035 // pair instructions directly. Assert if we see anything otherwise.
3036 //
3037 // The order of the registers in the list is controlled by
3038 // getCalleeSavedRegs(), so they will always be in-order, as well.
3039 assert((!RPI.isPaired() ||
3040 (CSI[i].getFrameIdx() + RegInc == CSI[i + RegInc].getFrameIdx())) &&
3041 "Out of order callee saved regs!");
3042
3043 assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg2 != AArch64::FP ||
3044 RPI.Reg1 == AArch64::LR) &&
3045 "FrameRecord must be allocated together with LR");
3046
3047 // Windows AAPCS has FP and LR reversed.
3048 assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg1 != AArch64::FP ||
3049 RPI.Reg2 == AArch64::LR) &&
3050 "FrameRecord must be allocated together with LR");
3051
3052 // MachO's compact unwind format relies on all registers being stored in
3053 // adjacent register pairs.
3054 assert((!produceCompactUnwindFrame(MF) || CC == CallingConv::PreserveMost ||
3055 CC == CallingConv::PreserveAll || CC == CallingConv::CXX_FAST_TLS ||
3056 CC == CallingConv::Win64 ||
3057 (RPI.isPaired() &&
3058 ((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
3059 RPI.Reg1 + 1 == RPI.Reg2))) &&
3060 "Callee-save registers not saved as adjacent register pair!");
3061
3062 RPI.FrameIdx = CSI[i].getFrameIdx();
3063 if (NeedsWinCFI &&
3064 RPI.isPaired()) // RPI.FrameIdx must be the lower index of the pair
3065 RPI.FrameIdx = CSI[i + RegInc].getFrameIdx();
3066 int Scale = RPI.getScale();
3067
3068 int OffsetPre = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
3069 assert(OffsetPre % Scale == 0);
3070
3071 if (RPI.isScalable())
3072 ScalableByteOffset += StackFillDir * (RPI.isPaired() ? 2 * Scale : Scale);
3073 else
3074 ByteOffset += StackFillDir * (RPI.isPaired() ? 2 * Scale : Scale);
3075
3076 // Swift's async context is directly before FP, so allocate an extra
3077 // 8 bytes for it.
3078 if (NeedsFrameRecord && AFI->hasSwiftAsyncContext() &&
3079 ((!IsWindows && RPI.Reg2 == AArch64::FP) ||
3080 (IsWindows && RPI.Reg2 == AArch64::LR)))
3081 ByteOffset += StackFillDir * 8;
3082
3083 // Round up size of non-pair to pair size if we need to pad the
3084 // callee-save area to ensure 16-byte alignment.
3085 if (NeedGapToAlignStack && !NeedsWinCFI && !RPI.isScalable() &&
3086 RPI.Type != RegPairInfo::FPR128 && !RPI.isPaired() &&
3087 ByteOffset % 16 != 0) {
3088 ByteOffset += 8 * StackFillDir;
3089 assert(MFI.getObjectAlign(RPI.FrameIdx) <= Align(16));
3090 // A stack frame with a gap looks like this, bottom up:
3091 // d9, d8. x21, gap, x20, x19.
3092 // Set extra alignment on the x21 object to create the gap above it.
3093 MFI.setObjectAlignment(ObjectIdx: RPI.FrameIdx, Alignment: Align(16));
3094 NeedGapToAlignStack = false;
3095 }
3096
3097 int OffsetPost = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
3098 assert(OffsetPost % Scale == 0);
3099 // If filling top down (default), we want the offset after incrementing it.
3100 // If filling bottom up (WinCFI) we need the original offset.
3101 int Offset = NeedsWinCFI ? OffsetPre : OffsetPost;
3102
3103 // The FP, LR pair goes 8 bytes into our expanded 24-byte slot so that the
3104 // Swift context can directly precede FP.
3105 if (NeedsFrameRecord && AFI->hasSwiftAsyncContext() &&
3106 ((!IsWindows && RPI.Reg2 == AArch64::FP) ||
3107 (IsWindows && RPI.Reg2 == AArch64::LR)))
3108 Offset += 8;
3109 RPI.Offset = Offset / Scale;
3110
3111 assert((!RPI.isPaired() ||
3112 (!RPI.isScalable() && RPI.Offset >= -64 && RPI.Offset <= 63) ||
3113 (RPI.isScalable() && RPI.Offset >= -256 && RPI.Offset <= 255)) &&
3114 "Offset out of bounds for LDP/STP immediate");
3115
3116 // Save the offset to frame record so that the FP register can point to the
3117 // innermost frame record (spilled FP and LR registers).
3118 if (NeedsFrameRecord &&
3119 ((!IsWindows && RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
3120 (IsWindows && RPI.Reg1 == AArch64::FP && RPI.Reg2 == AArch64::LR)))
3121 AFI->setCalleeSaveBaseToFrameRecordOffset(Offset);
3122
3123 RegPairs.push_back(Elt: RPI);
3124 if (RPI.isPaired())
3125 i += RegInc;
3126 }
3127 if (NeedsWinCFI) {
3128 // If we need an alignment gap in the stack, align the topmost stack
3129 // object. A stack frame with a gap looks like this, bottom up:
3130 // x19, d8. d9, gap.
3131 // Set extra alignment on the topmost stack object (the first element in
3132 // CSI, which goes top down), to create the gap above it.
3133 if (AFI->hasCalleeSaveStackFreeSpace())
3134 MFI.setObjectAlignment(ObjectIdx: CSI[0].getFrameIdx(), Alignment: Align(16));
3135 // We iterated bottom up over the registers; flip RegPairs back to top
3136 // down order.
3137 std::reverse(first: RegPairs.begin(), last: RegPairs.end());
3138 }
3139}
3140
3141bool AArch64FrameLowering::spillCalleeSavedRegisters(
3142 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
3143 ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
3144 MachineFunction &MF = *MBB.getParent();
3145 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
3146 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
3147 bool NeedsWinCFI = needsWinCFI(MF);
3148 DebugLoc DL;
3149 SmallVector<RegPairInfo, 8> RegPairs;
3150
3151 computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs, NeedsFrameRecord: hasFP(MF));
3152
3153 MachineRegisterInfo &MRI = MF.getRegInfo();
3154 // Refresh the reserved regs in case there are any potential changes since the
3155 // last freeze.
3156 MRI.freezeReservedRegs();
3157
3158 if (homogeneousPrologEpilog(MF)) {
3159 auto MIB = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AArch64::HOM_Prolog))
3160 .setMIFlag(MachineInstr::FrameSetup);
3161
3162 for (auto &RPI : RegPairs) {
3163 MIB.addReg(RegNo: RPI.Reg1);
3164 MIB.addReg(RegNo: RPI.Reg2);
3165
3166 // Update register live in.
3167 if (!MRI.isReserved(PhysReg: RPI.Reg1))
3168 MBB.addLiveIn(PhysReg: RPI.Reg1);
3169 if (RPI.isPaired() && !MRI.isReserved(PhysReg: RPI.Reg2))
3170 MBB.addLiveIn(PhysReg: RPI.Reg2);
3171 }
3172 return true;
3173 }
3174 bool PTrueCreated = false;
3175 for (const RegPairInfo &RPI : llvm::reverse(C&: RegPairs)) {
3176 unsigned Reg1 = RPI.Reg1;
3177 unsigned Reg2 = RPI.Reg2;
3178 unsigned StrOpc;
3179
3180 // Issue sequence of spills for cs regs. The first spill may be converted
3181 // to a pre-decrement store later by emitPrologue if the callee-save stack
3182 // area allocation can't be combined with the local stack area allocation.
3183 // For example:
3184 // stp x22, x21, [sp, #0] // addImm(+0)
3185 // stp x20, x19, [sp, #16] // addImm(+2)
3186 // stp fp, lr, [sp, #32] // addImm(+4)
3187 // Rationale: This sequence saves uop updates compared to a sequence of
3188 // pre-increment spills like stp xi,xj,[sp,#-16]!
3189 // Note: Similar rationale and sequence for restores in epilog.
3190 unsigned Size;
3191 Align Alignment;
3192 switch (RPI.Type) {
3193 case RegPairInfo::GPR:
3194 StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
3195 Size = 8;
3196 Alignment = Align(8);
3197 break;
3198 case RegPairInfo::FPR64:
3199 StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
3200 Size = 8;
3201 Alignment = Align(8);
3202 break;
3203 case RegPairInfo::FPR128:
3204 StrOpc = RPI.isPaired() ? AArch64::STPQi : AArch64::STRQui;
3205 Size = 16;
3206 Alignment = Align(16);
3207 break;
3208 case RegPairInfo::ZPR:
3209 StrOpc = RPI.isPaired() ? AArch64::ST1B_2Z_IMM : AArch64::STR_ZXI;
3210 Size = 16;
3211 Alignment = Align(16);
3212 break;
3213 case RegPairInfo::PPR:
3214 StrOpc = AArch64::STR_PXI;
3215 Size = 2;
3216 Alignment = Align(2);
3217 break;
3218 case RegPairInfo::VG:
3219 StrOpc = AArch64::STRXui;
3220 Size = 8;
3221 Alignment = Align(8);
3222 break;
3223 }
3224
3225 unsigned X0Scratch = AArch64::NoRegister;
3226 if (Reg1 == AArch64::VG) {
3227 // Find an available register to store value of VG to.
3228 Reg1 = findScratchNonCalleeSaveRegister(MBB: &MBB);
3229 assert(Reg1 != AArch64::NoRegister);
3230 SMEAttrs Attrs(MF.getFunction());
3231
3232 if (Attrs.hasStreamingBody() && !Attrs.hasStreamingInterface() &&
3233 AFI->getStreamingVGIdx() == std::numeric_limits<int>::max()) {
3234 // For locally-streaming functions, we need to store both the streaming
3235 // & non-streaming VG. Spill the streaming value first.
3236 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AArch64::RDSVLI_XI), DestReg: Reg1)
3237 .addImm(Val: 1)
3238 .setMIFlag(MachineInstr::FrameSetup);
3239 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AArch64::UBFMXri), DestReg: Reg1)
3240 .addReg(RegNo: Reg1)
3241 .addImm(Val: 3)
3242 .addImm(Val: 63)
3243 .setMIFlag(MachineInstr::FrameSetup);
3244
3245 AFI->setStreamingVGIdx(RPI.FrameIdx);
3246 } else if (MF.getSubtarget<AArch64Subtarget>().hasSVE()) {
3247 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AArch64::CNTD_XPiI), DestReg: Reg1)
3248 .addImm(Val: 31)
3249 .addImm(Val: 1)
3250 .setMIFlag(MachineInstr::FrameSetup);
3251 AFI->setVGIdx(RPI.FrameIdx);
3252 } else {
3253 const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
3254 if (llvm::any_of(
3255 Range: MBB.liveins(),
3256 P: [&STI](const MachineBasicBlock::RegisterMaskPair &LiveIn) {
3257 return STI.getRegisterInfo()->isSuperOrSubRegisterEq(
3258 RegA: AArch64::X0, RegB: LiveIn.PhysReg);
3259 }))
3260 X0Scratch = Reg1;
3261
3262 if (X0Scratch != AArch64::NoRegister)
3263 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AArch64::ORRXrr), DestReg: Reg1)
3264 .addReg(RegNo: AArch64::XZR)
3265 .addReg(RegNo: AArch64::X0, flags: RegState::Undef)
3266 .addReg(RegNo: AArch64::X0, flags: RegState::Implicit)
3267 .setMIFlag(MachineInstr::FrameSetup);
3268
3269 const uint32_t *RegMask = TRI->getCallPreservedMask(
3270 MF,
3271 CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1);
3272 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AArch64::BL))
3273 .addExternalSymbol(FnName: "__arm_get_current_vg")
3274 .addRegMask(Mask: RegMask)
3275 .addReg(RegNo: AArch64::X0, flags: RegState::ImplicitDefine)
3276 .setMIFlag(MachineInstr::FrameSetup);
3277 Reg1 = AArch64::X0;
3278 AFI->setVGIdx(RPI.FrameIdx);
3279 }
3280 }
3281
3282 LLVM_DEBUG(dbgs() << "CSR spill: (" << printReg(Reg1, TRI);
3283 if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
3284 dbgs() << ") -> fi#(" << RPI.FrameIdx;
3285 if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
3286 dbgs() << ")\n");
3287
3288 assert((!NeedsWinCFI || !(Reg1 == AArch64::LR && Reg2 == AArch64::FP)) &&
3289 "Windows unwdinding requires a consecutive (FP,LR) pair");
3290 // Windows unwind codes require consecutive registers if registers are
3291 // paired. Make the switch here, so that the code below will save (x,x+1)
3292 // and not (x+1,x).
3293 unsigned FrameIdxReg1 = RPI.FrameIdx;
3294 unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
3295 if (NeedsWinCFI && RPI.isPaired()) {
3296 std::swap(a&: Reg1, b&: Reg2);
3297 std::swap(a&: FrameIdxReg1, b&: FrameIdxReg2);
3298 }
3299
3300 if (RPI.isPaired() && RPI.isScalable()) {
3301 [[maybe_unused]] const AArch64Subtarget &Subtarget =
3302 MF.getSubtarget<AArch64Subtarget>();
3303 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
3304 unsigned PnReg = AFI->getPredicateRegForFillSpill();
3305 assert(((Subtarget.hasSVE2p1() || Subtarget.hasSME2()) && PnReg != 0) &&
3306 "Expects SVE2.1 or SME2 target and a predicate register");
3307#ifdef EXPENSIVE_CHECKS
3308 auto IsPPR = [](const RegPairInfo &c) {
3309 return c.Reg1 == RegPairInfo::PPR;
3310 };
3311 auto PPRBegin = std::find_if(RegPairs.begin(), RegPairs.end(), IsPPR);
3312 auto IsZPR = [](const RegPairInfo &c) {
3313 return c.Type == RegPairInfo::ZPR;
3314 };
3315 auto ZPRBegin = std::find_if(RegPairs.begin(), RegPairs.end(), IsZPR);
3316 assert(!(PPRBegin < ZPRBegin) &&
3317 "Expected callee save predicate to be handled first");
3318#endif
3319 if (!PTrueCreated) {
3320 PTrueCreated = true;
3321 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AArch64::PTRUE_C_B), DestReg: PnReg)
3322 .setMIFlags(MachineInstr::FrameSetup);
3323 }
3324 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: StrOpc));
3325 if (!MRI.isReserved(PhysReg: Reg1))
3326 MBB.addLiveIn(PhysReg: Reg1);
3327 if (!MRI.isReserved(PhysReg: Reg2))
3328 MBB.addLiveIn(PhysReg: Reg2);
3329 MIB.addReg(/*PairRegs*/ RegNo: AArch64::Z0_Z1 + (RPI.Reg1 - AArch64::Z0));
3330 MIB.addMemOperand(MMO: MF.getMachineMemOperand(
3331 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIdxReg2),
3332 F: MachineMemOperand::MOStore, Size, BaseAlignment: Alignment));
3333 MIB.addReg(RegNo: PnReg);
3334 MIB.addReg(RegNo: AArch64::SP)
3335 .addImm(Val: RPI.Offset) // [sp, #offset*scale],
3336 // where factor*scale is implicit
3337 .setMIFlag(MachineInstr::FrameSetup);
3338 MIB.addMemOperand(MMO: MF.getMachineMemOperand(
3339 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIdxReg1),
3340 F: MachineMemOperand::MOStore, Size, BaseAlignment: Alignment));
3341 if (NeedsWinCFI)
3342 InsertSEH(MBBI: MIB, TII, Flag: MachineInstr::FrameSetup);
3343 } else { // The code when the pair of ZReg is not present
3344 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: StrOpc));
3345 if (!MRI.isReserved(PhysReg: Reg1))
3346 MBB.addLiveIn(PhysReg: Reg1);
3347 if (RPI.isPaired()) {
3348 if (!MRI.isReserved(PhysReg: Reg2))
3349 MBB.addLiveIn(PhysReg: Reg2);
3350 MIB.addReg(RegNo: Reg2, flags: getPrologueDeath(MF, Reg: Reg2));
3351 MIB.addMemOperand(MMO: MF.getMachineMemOperand(
3352 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIdxReg2),
3353 F: MachineMemOperand::MOStore, Size, BaseAlignment: Alignment));
3354 }
3355 MIB.addReg(RegNo: Reg1, flags: getPrologueDeath(MF, Reg: Reg1))
3356 .addReg(RegNo: AArch64::SP)
3357 .addImm(Val: RPI.Offset) // [sp, #offset*scale],
3358 // where factor*scale is implicit
3359 .setMIFlag(MachineInstr::FrameSetup);
3360 MIB.addMemOperand(MMO: MF.getMachineMemOperand(
3361 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIdxReg1),
3362 F: MachineMemOperand::MOStore, Size, BaseAlignment: Alignment));
3363 if (NeedsWinCFI)
3364 InsertSEH(MBBI: MIB, TII, Flag: MachineInstr::FrameSetup);
3365 }
3366 // Update the StackIDs of the SVE stack slots.
3367 MachineFrameInfo &MFI = MF.getFrameInfo();
3368 if (RPI.Type == RegPairInfo::ZPR || RPI.Type == RegPairInfo::PPR) {
3369 MFI.setStackID(ObjectIdx: FrameIdxReg1, ID: TargetStackID::ScalableVector);
3370 if (RPI.isPaired())
3371 MFI.setStackID(ObjectIdx: FrameIdxReg2, ID: TargetStackID::ScalableVector);
3372 }
3373
3374 if (X0Scratch != AArch64::NoRegister)
3375 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AArch64::ORRXrr), DestReg: AArch64::X0)
3376 .addReg(RegNo: AArch64::XZR)
3377 .addReg(RegNo: X0Scratch, flags: RegState::Undef)
3378 .addReg(RegNo: X0Scratch, flags: RegState::Implicit)
3379 .setMIFlag(MachineInstr::FrameSetup);
3380 }
3381 return true;
3382}
3383
3384bool AArch64FrameLowering::restoreCalleeSavedRegisters(
3385 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
3386 MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
3387 MachineFunction &MF = *MBB.getParent();
3388 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
3389 DebugLoc DL;
3390 SmallVector<RegPairInfo, 8> RegPairs;
3391 bool NeedsWinCFI = needsWinCFI(MF);
3392
3393 if (MBBI != MBB.end())
3394 DL = MBBI->getDebugLoc();
3395
3396 computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs, NeedsFrameRecord: hasFP(MF));
3397 if (homogeneousPrologEpilog(MF, Exit: &MBB)) {
3398 auto MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AArch64::HOM_Epilog))
3399 .setMIFlag(MachineInstr::FrameDestroy);
3400 for (auto &RPI : RegPairs) {
3401 MIB.addReg(RegNo: RPI.Reg1, flags: RegState::Define);
3402 MIB.addReg(RegNo: RPI.Reg2, flags: RegState::Define);
3403 }
3404 return true;
3405 }
3406
3407 // For performance reasons restore SVE register in increasing order
3408 auto IsPPR = [](const RegPairInfo &c) { return c.Type == RegPairInfo::PPR; };
3409 auto PPRBegin = std::find_if(first: RegPairs.begin(), last: RegPairs.end(), pred: IsPPR);
3410 auto PPREnd = std::find_if_not(first: PPRBegin, last: RegPairs.end(), pred: IsPPR);
3411 std::reverse(first: PPRBegin, last: PPREnd);
3412 auto IsZPR = [](const RegPairInfo &c) { return c.Type == RegPairInfo::ZPR; };
3413 auto ZPRBegin = std::find_if(first: RegPairs.begin(), last: RegPairs.end(), pred: IsZPR);
3414 auto ZPREnd = std::find_if_not(first: ZPRBegin, last: RegPairs.end(), pred: IsZPR);
3415 std::reverse(first: ZPRBegin, last: ZPREnd);
3416
3417 bool PTrueCreated = false;
3418 for (const RegPairInfo &RPI : RegPairs) {
3419 unsigned Reg1 = RPI.Reg1;
3420 unsigned Reg2 = RPI.Reg2;
3421
3422 // Issue sequence of restores for cs regs. The last restore may be converted
3423 // to a post-increment load later by emitEpilogue if the callee-save stack
3424 // area allocation can't be combined with the local stack area allocation.
3425 // For example:
3426 // ldp fp, lr, [sp, #32] // addImm(+4)
3427 // ldp x20, x19, [sp, #16] // addImm(+2)
3428 // ldp x22, x21, [sp, #0] // addImm(+0)
3429 // Note: see comment in spillCalleeSavedRegisters()
3430 unsigned LdrOpc;
3431 unsigned Size;
3432 Align Alignment;
3433 switch (RPI.Type) {
3434 case RegPairInfo::GPR:
3435 LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
3436 Size = 8;
3437 Alignment = Align(8);
3438 break;
3439 case RegPairInfo::FPR64:
3440 LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
3441 Size = 8;
3442 Alignment = Align(8);
3443 break;
3444 case RegPairInfo::FPR128:
3445 LdrOpc = RPI.isPaired() ? AArch64::LDPQi : AArch64::LDRQui;
3446 Size = 16;
3447 Alignment = Align(16);
3448 break;
3449 case RegPairInfo::ZPR:
3450 LdrOpc = RPI.isPaired() ? AArch64::LD1B_2Z_IMM : AArch64::LDR_ZXI;
3451 Size = 16;
3452 Alignment = Align(16);
3453 break;
3454 case RegPairInfo::PPR:
3455 LdrOpc = AArch64::LDR_PXI;
3456 Size = 2;
3457 Alignment = Align(2);
3458 break;
3459 case RegPairInfo::VG:
3460 continue;
3461 }
3462 LLVM_DEBUG(dbgs() << "CSR restore: (" << printReg(Reg1, TRI);
3463 if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
3464 dbgs() << ") -> fi#(" << RPI.FrameIdx;
3465 if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
3466 dbgs() << ")\n");
3467
3468 // Windows unwind codes require consecutive registers if registers are
3469 // paired. Make the switch here, so that the code below will save (x,x+1)
3470 // and not (x+1,x).
3471 unsigned FrameIdxReg1 = RPI.FrameIdx;
3472 unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
3473 if (NeedsWinCFI && RPI.isPaired()) {
3474 std::swap(a&: Reg1, b&: Reg2);
3475 std::swap(a&: FrameIdxReg1, b&: FrameIdxReg2);
3476 }
3477
3478 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
3479 if (RPI.isPaired() && RPI.isScalable()) {
3480 [[maybe_unused]] const AArch64Subtarget &Subtarget =
3481 MF.getSubtarget<AArch64Subtarget>();
3482 unsigned PnReg = AFI->getPredicateRegForFillSpill();
3483 assert(((Subtarget.hasSVE2p1() || Subtarget.hasSME2()) && PnReg != 0) &&
3484 "Expects SVE2.1 or SME2 target and a predicate register");
3485#ifdef EXPENSIVE_CHECKS
3486 assert(!(PPRBegin < ZPRBegin) &&
3487 "Expected callee save predicate to be handled first");
3488#endif
3489 if (!PTrueCreated) {
3490 PTrueCreated = true;
3491 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AArch64::PTRUE_C_B), DestReg: PnReg)
3492 .setMIFlags(MachineInstr::FrameDestroy);
3493 }
3494 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: LdrOpc));
3495 MIB.addReg(/*PairRegs*/ RegNo: AArch64::Z0_Z1 + (RPI.Reg1 - AArch64::Z0),
3496 flags: getDefRegState(B: true));
3497 MIB.addMemOperand(MMO: MF.getMachineMemOperand(
3498 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIdxReg2),
3499 F: MachineMemOperand::MOLoad, Size, BaseAlignment: Alignment));
3500 MIB.addReg(RegNo: PnReg);
3501 MIB.addReg(RegNo: AArch64::SP)
3502 .addImm(Val: RPI.Offset) // [sp, #offset*scale]
3503 // where factor*scale is implicit
3504 .setMIFlag(MachineInstr::FrameDestroy);
3505 MIB.addMemOperand(MMO: MF.getMachineMemOperand(
3506 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIdxReg1),
3507 F: MachineMemOperand::MOLoad, Size, BaseAlignment: Alignment));
3508 if (NeedsWinCFI)
3509 InsertSEH(MBBI: MIB, TII, Flag: MachineInstr::FrameDestroy);
3510 } else {
3511 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: LdrOpc));
3512 if (RPI.isPaired()) {
3513 MIB.addReg(RegNo: Reg2, flags: getDefRegState(B: true));
3514 MIB.addMemOperand(MMO: MF.getMachineMemOperand(
3515 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIdxReg2),
3516 F: MachineMemOperand::MOLoad, Size, BaseAlignment: Alignment));
3517 }
3518 MIB.addReg(RegNo: Reg1, flags: getDefRegState(B: true));
3519 MIB.addReg(RegNo: AArch64::SP)
3520 .addImm(Val: RPI.Offset) // [sp, #offset*scale]
3521 // where factor*scale is implicit
3522 .setMIFlag(MachineInstr::FrameDestroy);
3523 MIB.addMemOperand(MMO: MF.getMachineMemOperand(
3524 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIdxReg1),
3525 F: MachineMemOperand::MOLoad, Size, BaseAlignment: Alignment));
3526 if (NeedsWinCFI)
3527 InsertSEH(MBBI: MIB, TII, Flag: MachineInstr::FrameDestroy);
3528 }
3529 }
3530 return true;
3531}
3532
3533// Return the FrameID for a MMO.
3534static std::optional<int> getMMOFrameID(MachineMemOperand *MMO,
3535 const MachineFrameInfo &MFI) {
3536 auto *PSV =
3537 dyn_cast_or_null<FixedStackPseudoSourceValue>(Val: MMO->getPseudoValue());
3538 if (PSV)
3539 return std::optional<int>(PSV->getFrameIndex());
3540
3541 if (MMO->getValue()) {
3542 if (auto *Al = dyn_cast<AllocaInst>(Val: getUnderlyingObject(V: MMO->getValue()))) {
3543 for (int FI = MFI.getObjectIndexBegin(); FI < MFI.getObjectIndexEnd();
3544 FI++)
3545 if (MFI.getObjectAllocation(ObjectIdx: FI) == Al)
3546 return FI;
3547 }
3548 }
3549
3550 return std::nullopt;
3551}
3552
3553// Return the FrameID for a Load/Store instruction by looking at the first MMO.
3554static std::optional<int> getLdStFrameID(const MachineInstr &MI,
3555 const MachineFrameInfo &MFI) {
3556 if (!MI.mayLoadOrStore() || MI.getNumMemOperands() < 1)
3557 return std::nullopt;
3558
3559 return getMMOFrameID(MMO: *MI.memoperands_begin(), MFI);
3560}
3561
3562// Check if a Hazard slot is needed for the current function, and if so create
3563// one for it. The index is stored in AArch64FunctionInfo->StackHazardSlotIndex,
3564// which can be used to determine if any hazard padding is needed.
3565void AArch64FrameLowering::determineStackHazardSlot(
3566 MachineFunction &MF, BitVector &SavedRegs) const {
3567 if (StackHazardSize == 0 || StackHazardSize % 16 != 0 ||
3568 MF.getInfo<AArch64FunctionInfo>()->hasStackHazardSlotIndex())
3569 return;
3570
3571 // Stack hazards are only needed in streaming functions.
3572 SMEAttrs Attrs(MF.getFunction());
3573 if (!StackHazardInNonStreaming && Attrs.hasNonStreamingInterfaceAndBody())
3574 return;
3575
3576 MachineFrameInfo &MFI = MF.getFrameInfo();
3577
3578 // Add a hazard slot if there are any CSR FPR registers, or are any fp-only
3579 // stack objects.
3580 bool HasFPRCSRs = any_of(Range: SavedRegs.set_bits(), P: [](unsigned Reg) {
3581 return AArch64::FPR64RegClass.contains(Reg) ||
3582 AArch64::FPR128RegClass.contains(Reg) ||
3583 AArch64::ZPRRegClass.contains(Reg) ||
3584 AArch64::PPRRegClass.contains(Reg);
3585 });
3586 bool HasFPRStackObjects = false;
3587 if (!HasFPRCSRs) {
3588 std::vector<unsigned> FrameObjects(MFI.getObjectIndexEnd());
3589 for (auto &MBB : MF) {
3590 for (auto &MI : MBB) {
3591 std::optional<int> FI = getLdStFrameID(MI, MFI);
3592 if (FI && *FI >= 0 && *FI < (int)FrameObjects.size()) {
3593 if (MFI.getStackID(ObjectIdx: *FI) == TargetStackID::ScalableVector ||
3594 AArch64InstrInfo::isFpOrNEON(MI))
3595 FrameObjects[*FI] |= 2;
3596 else
3597 FrameObjects[*FI] |= 1;
3598 }
3599 }
3600 }
3601 HasFPRStackObjects =
3602 any_of(Range&: FrameObjects, P: [](unsigned B) { return (B & 3) == 2; });
3603 }
3604
3605 if (HasFPRCSRs || HasFPRStackObjects) {
3606 int ID = MFI.CreateStackObject(Size: StackHazardSize, Alignment: Align(16), isSpillSlot: false);
3607 LLVM_DEBUG(dbgs() << "Created Hazard slot at " << ID << " size "
3608 << StackHazardSize << "\n");
3609 MF.getInfo<AArch64FunctionInfo>()->setStackHazardSlotIndex(ID);
3610 }
3611}
3612
3613void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
3614 BitVector &SavedRegs,
3615 RegScavenger *RS) const {
3616 // All calls are tail calls in GHC calling conv, and functions have no
3617 // prologue/epilogue.
3618 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
3619 return;
3620
3621 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
3622 const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
3623 MF.getSubtarget().getRegisterInfo());
3624 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
3625 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
3626 unsigned UnspilledCSGPR = AArch64::NoRegister;
3627 unsigned UnspilledCSGPRPaired = AArch64::NoRegister;
3628
3629 MachineFrameInfo &MFI = MF.getFrameInfo();
3630 const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
3631
3632 unsigned BasePointerReg = RegInfo->hasBasePointer(MF)
3633 ? RegInfo->getBaseRegister()
3634 : (unsigned)AArch64::NoRegister;
3635
3636 unsigned ExtraCSSpill = 0;
3637 bool HasUnpairedGPR64 = false;
3638 // Figure out which callee-saved registers to save/restore.
3639 for (unsigned i = 0; CSRegs[i]; ++i) {
3640 const unsigned Reg = CSRegs[i];
3641
3642 // Add the base pointer register to SavedRegs if it is callee-save.
3643 if (Reg == BasePointerReg)
3644 SavedRegs.set(Reg);
3645
3646 bool RegUsed = SavedRegs.test(Idx: Reg);
3647 unsigned PairedReg = AArch64::NoRegister;
3648 const bool RegIsGPR64 = AArch64::GPR64RegClass.contains(Reg);
3649 if (RegIsGPR64 || AArch64::FPR64RegClass.contains(Reg) ||
3650 AArch64::FPR128RegClass.contains(Reg)) {
3651 // Compensate for odd numbers of GP CSRs.
3652 // For now, all the known cases of odd number of CSRs are of GPRs.
3653 if (HasUnpairedGPR64)
3654 PairedReg = CSRegs[i % 2 == 0 ? i - 1 : i + 1];
3655 else
3656 PairedReg = CSRegs[i ^ 1];
3657 }
3658
3659 // If the function requires all the GP registers to save (SavedRegs),
3660 // and there are an odd number of GP CSRs at the same time (CSRegs),
3661 // PairedReg could be in a different register class from Reg, which would
3662 // lead to a FPR (usually D8) accidentally being marked saved.
3663 if (RegIsGPR64 && !AArch64::GPR64RegClass.contains(Reg: PairedReg)) {
3664 PairedReg = AArch64::NoRegister;
3665 HasUnpairedGPR64 = true;
3666 }
3667 assert(PairedReg == AArch64::NoRegister ||
3668 AArch64::GPR64RegClass.contains(Reg, PairedReg) ||
3669 AArch64::FPR64RegClass.contains(Reg, PairedReg) ||
3670 AArch64::FPR128RegClass.contains(Reg, PairedReg));
3671
3672 if (!RegUsed) {
3673 if (AArch64::GPR64RegClass.contains(Reg) &&
3674 !RegInfo->isReservedReg(MF, Reg)) {
3675 UnspilledCSGPR = Reg;
3676 UnspilledCSGPRPaired = PairedReg;
3677 }
3678 continue;
3679 }
3680
3681 // MachO's compact unwind format relies on all registers being stored in
3682 // pairs.
3683 // FIXME: the usual format is actually better if unwinding isn't needed.
3684 if (producePairRegisters(MF) && PairedReg != AArch64::NoRegister &&
3685 !SavedRegs.test(Idx: PairedReg)) {
3686 SavedRegs.set(PairedReg);
3687 if (AArch64::GPR64RegClass.contains(Reg: PairedReg) &&
3688 !RegInfo->isReservedReg(MF, Reg: PairedReg))
3689 ExtraCSSpill = PairedReg;
3690 }
3691 }
3692
3693 if (MF.getFunction().getCallingConv() == CallingConv::Win64 &&
3694 !Subtarget.isTargetWindows()) {
3695 // For Windows calling convention on a non-windows OS, where X18 is treated
3696 // as reserved, back up X18 when entering non-windows code (marked with the
3697 // Windows calling convention) and restore when returning regardless of
3698 // whether the individual function uses it - it might call other functions
3699 // that clobber it.
3700 SavedRegs.set(AArch64::X18);
3701 }
3702
3703 // Calculates the callee saved stack size.
3704 unsigned CSStackSize = 0;
3705 unsigned SVECSStackSize = 0;
3706 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
3707 const MachineRegisterInfo &MRI = MF.getRegInfo();
3708 for (unsigned Reg : SavedRegs.set_bits()) {
3709 auto RegSize = TRI->getRegSizeInBits(Reg, MRI) / 8;
3710 if (AArch64::PPRRegClass.contains(Reg) ||
3711 AArch64::ZPRRegClass.contains(Reg))
3712 SVECSStackSize += RegSize;
3713 else
3714 CSStackSize += RegSize;
3715 }
3716
3717 // Increase the callee-saved stack size if the function has streaming mode
3718 // changes, as we will need to spill the value of the VG register.
3719 // For locally streaming functions, we spill both the streaming and
3720 // non-streaming VG value.
3721 const Function &F = MF.getFunction();
3722 SMEAttrs Attrs(F);
3723 if (AFI->hasStreamingModeChanges()) {
3724 if (Attrs.hasStreamingBody() && !Attrs.hasStreamingInterface())
3725 CSStackSize += 16;
3726 else
3727 CSStackSize += 8;
3728 }
3729
3730 // Determine if a Hazard slot should be used, and increase the CSStackSize by
3731 // StackHazardSize if so.
3732 determineStackHazardSlot(MF, SavedRegs);
3733 if (AFI->hasStackHazardSlotIndex())
3734 CSStackSize += StackHazardSize;
3735
3736 // Save number of saved regs, so we can easily update CSStackSize later.
3737 unsigned NumSavedRegs = SavedRegs.count();
3738
3739 // The frame record needs to be created by saving the appropriate registers
3740 uint64_t EstimatedStackSize = MFI.estimateStackSize(MF);
3741 if (hasFP(MF) ||
3742 windowsRequiresStackProbe(MF, StackSizeInBytes: EstimatedStackSize + CSStackSize + 16)) {
3743 SavedRegs.set(AArch64::FP);
3744 SavedRegs.set(AArch64::LR);
3745 }
3746
3747 LLVM_DEBUG({
3748 dbgs() << "*** determineCalleeSaves\nSaved CSRs:";
3749 for (unsigned Reg : SavedRegs.set_bits())
3750 dbgs() << ' ' << printReg(Reg, RegInfo);
3751 dbgs() << "\n";
3752 });
3753
3754 // If any callee-saved registers are used, the frame cannot be eliminated.
3755 int64_t SVEStackSize =
3756 alignTo(Value: SVECSStackSize + estimateSVEStackObjectOffsets(MF&: MFI), Align: 16);
3757 bool CanEliminateFrame = (SavedRegs.count() == 0) && !SVEStackSize;
3758
3759 // The CSR spill slots have not been allocated yet, so estimateStackSize
3760 // won't include them.
3761 unsigned EstimatedStackSizeLimit = estimateRSStackSizeLimit(MF);
3762
3763 // We may address some of the stack above the canonical frame address, either
3764 // for our own arguments or during a call. Include that in calculating whether
3765 // we have complicated addressing concerns.
3766 int64_t CalleeStackUsed = 0;
3767 for (int I = MFI.getObjectIndexBegin(); I != 0; ++I) {
3768 int64_t FixedOff = MFI.getObjectOffset(ObjectIdx: I);
3769 if (FixedOff > CalleeStackUsed)
3770 CalleeStackUsed = FixedOff;
3771 }
3772
3773 // Conservatively always assume BigStack when there are SVE spills.
3774 bool BigStack = SVEStackSize || (EstimatedStackSize + CSStackSize +
3775 CalleeStackUsed) > EstimatedStackSizeLimit;
3776 if (BigStack || !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF))
3777 AFI->setHasStackFrame(true);
3778
3779 // Estimate if we might need to scavenge a register at some point in order
3780 // to materialize a stack offset. If so, either spill one additional
3781 // callee-saved register or reserve a special spill slot to facilitate
3782 // register scavenging. If we already spilled an extra callee-saved register
3783 // above to keep the number of spills even, we don't need to do anything else
3784 // here.
3785 if (BigStack) {
3786 if (!ExtraCSSpill && UnspilledCSGPR != AArch64::NoRegister) {
3787 LLVM_DEBUG(dbgs() << "Spilling " << printReg(UnspilledCSGPR, RegInfo)
3788 << " to get a scratch register.\n");
3789 SavedRegs.set(UnspilledCSGPR);
3790 ExtraCSSpill = UnspilledCSGPR;
3791
3792 // MachO's compact unwind format relies on all registers being stored in
3793 // pairs, so if we need to spill one extra for BigStack, then we need to
3794 // store the pair.
3795 if (producePairRegisters(MF)) {
3796 if (UnspilledCSGPRPaired == AArch64::NoRegister) {
3797 // Failed to make a pair for compact unwind format, revert spilling.
3798 if (produceCompactUnwindFrame(MF)) {
3799 SavedRegs.reset(Idx: UnspilledCSGPR);
3800 ExtraCSSpill = AArch64::NoRegister;
3801 }
3802 } else
3803 SavedRegs.set(UnspilledCSGPRPaired);
3804 }
3805 }
3806
3807 // If we didn't find an extra callee-saved register to spill, create
3808 // an emergency spill slot.
3809 if (!ExtraCSSpill || MF.getRegInfo().isPhysRegUsed(PhysReg: ExtraCSSpill)) {
3810 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
3811 const TargetRegisterClass &RC = AArch64::GPR64RegClass;
3812 unsigned Size = TRI->getSpillSize(RC);
3813 Align Alignment = TRI->getSpillAlign(RC);
3814 int FI = MFI.CreateStackObject(Size, Alignment, isSpillSlot: false);
3815 RS->addScavengingFrameIndex(FI);
3816 LLVM_DEBUG(dbgs() << "No available CS registers, allocated fi#" << FI
3817 << " as the emergency spill slot.\n");
3818 }
3819 }
3820
3821 // Adding the size of additional 64bit GPR saves.
3822 CSStackSize += 8 * (SavedRegs.count() - NumSavedRegs);
3823
3824 // A Swift asynchronous context extends the frame record with a pointer
3825 // directly before FP.
3826 if (hasFP(MF) && AFI->hasSwiftAsyncContext())
3827 CSStackSize += 8;
3828
3829 uint64_t AlignedCSStackSize = alignTo(Value: CSStackSize, Align: 16);
3830 LLVM_DEBUG(dbgs() << "Estimated stack frame size: "
3831 << EstimatedStackSize + AlignedCSStackSize << " bytes.\n");
3832
3833 assert((!MFI.isCalleeSavedInfoValid() ||
3834 AFI->getCalleeSavedStackSize() == AlignedCSStackSize) &&
3835 "Should not invalidate callee saved info");
3836
3837 // Round up to register pair alignment to avoid additional SP adjustment
3838 // instructions.
3839 AFI->setCalleeSavedStackSize(AlignedCSStackSize);
3840 AFI->setCalleeSaveStackHasFreeSpace(AlignedCSStackSize != CSStackSize);
3841 AFI->setSVECalleeSavedStackSize(alignTo(Value: SVECSStackSize, Align: 16));
3842}
3843
3844bool AArch64FrameLowering::assignCalleeSavedSpillSlots(
3845 MachineFunction &MF, const TargetRegisterInfo *RegInfo,
3846 std::vector<CalleeSavedInfo> &CSI, unsigned &MinCSFrameIndex,
3847 unsigned &MaxCSFrameIndex) const {
3848 bool NeedsWinCFI = needsWinCFI(MF);
3849 // To match the canonical windows frame layout, reverse the list of
3850 // callee saved registers to get them laid out by PrologEpilogInserter
3851 // in the right order. (PrologEpilogInserter allocates stack objects top
3852 // down. Windows canonical prologs store higher numbered registers at
3853 // the top, thus have the CSI array start from the highest registers.)
3854 if (NeedsWinCFI)
3855 std::reverse(first: CSI.begin(), last: CSI.end());
3856
3857 if (CSI.empty())
3858 return true; // Early exit if no callee saved registers are modified!
3859
3860 // Now that we know which registers need to be saved and restored, allocate
3861 // stack slots for them.
3862 MachineFrameInfo &MFI = MF.getFrameInfo();
3863 auto *AFI = MF.getInfo<AArch64FunctionInfo>();
3864
3865 bool UsesWinAAPCS = isTargetWindows(MF);
3866 if (UsesWinAAPCS && hasFP(MF) && AFI->hasSwiftAsyncContext()) {
3867 int FrameIdx = MFI.CreateStackObject(Size: 8, Alignment: Align(16), isSpillSlot: true);
3868 AFI->setSwiftAsyncContextFrameIdx(FrameIdx);
3869 if ((unsigned)FrameIdx < MinCSFrameIndex)
3870 MinCSFrameIndex = FrameIdx;
3871 if ((unsigned)FrameIdx > MaxCSFrameIndex)
3872 MaxCSFrameIndex = FrameIdx;
3873 }
3874
3875 // Insert VG into the list of CSRs, immediately before LR if saved.
3876 if (AFI->hasStreamingModeChanges()) {
3877 std::vector<CalleeSavedInfo> VGSaves;
3878 SMEAttrs Attrs(MF.getFunction());
3879
3880 auto VGInfo = CalleeSavedInfo(AArch64::VG);
3881 VGInfo.setRestored(false);
3882 VGSaves.push_back(x: VGInfo);
3883
3884 // Add VG again if the function is locally-streaming, as we will spill two
3885 // values.
3886 if (Attrs.hasStreamingBody() && !Attrs.hasStreamingInterface())
3887 VGSaves.push_back(x: VGInfo);
3888
3889 bool InsertBeforeLR = false;
3890
3891 for (unsigned I = 0; I < CSI.size(); I++)
3892 if (CSI[I].getReg() == AArch64::LR) {
3893 InsertBeforeLR = true;
3894 CSI.insert(position: CSI.begin() + I, first: VGSaves.begin(), last: VGSaves.end());
3895 break;
3896 }
3897
3898 if (!InsertBeforeLR)
3899 CSI.insert(position: CSI.end(), first: VGSaves.begin(), last: VGSaves.end());
3900 }
3901
3902 Register LastReg = 0;
3903 int HazardSlotIndex = std::numeric_limits<int>::max();
3904 for (auto &CS : CSI) {
3905 Register Reg = CS.getReg();
3906 const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg);
3907
3908 // Create a hazard slot as we switch between GPR and FPR CSRs.
3909 if (AFI->hasStackHazardSlotIndex() &&
3910 (!LastReg || !AArch64InstrInfo::isFpOrNEON(Reg: LastReg)) &&
3911 AArch64InstrInfo::isFpOrNEON(Reg)) {
3912 assert(HazardSlotIndex == std::numeric_limits<int>::max() &&
3913 "Unexpected register order for hazard slot");
3914 HazardSlotIndex = MFI.CreateStackObject(Size: StackHazardSize, Alignment: Align(8), isSpillSlot: true);
3915 LLVM_DEBUG(dbgs() << "Created CSR Hazard at slot " << HazardSlotIndex
3916 << "\n");
3917 AFI->setStackHazardCSRSlotIndex(HazardSlotIndex);
3918 if ((unsigned)HazardSlotIndex < MinCSFrameIndex)
3919 MinCSFrameIndex = HazardSlotIndex;
3920 if ((unsigned)HazardSlotIndex > MaxCSFrameIndex)
3921 MaxCSFrameIndex = HazardSlotIndex;
3922 }
3923
3924 unsigned Size = RegInfo->getSpillSize(RC: *RC);
3925 Align Alignment(RegInfo->getSpillAlign(RC: *RC));
3926 int FrameIdx = MFI.CreateStackObject(Size, Alignment, isSpillSlot: true);
3927 CS.setFrameIdx(FrameIdx);
3928
3929 if ((unsigned)FrameIdx < MinCSFrameIndex)
3930 MinCSFrameIndex = FrameIdx;
3931 if ((unsigned)FrameIdx > MaxCSFrameIndex)
3932 MaxCSFrameIndex = FrameIdx;
3933
3934 // Grab 8 bytes below FP for the extended asynchronous frame info.
3935 if (hasFP(MF) && AFI->hasSwiftAsyncContext() && !UsesWinAAPCS &&
3936 Reg == AArch64::FP) {
3937 FrameIdx = MFI.CreateStackObject(Size: 8, Alignment, isSpillSlot: true);
3938 AFI->setSwiftAsyncContextFrameIdx(FrameIdx);
3939 if ((unsigned)FrameIdx < MinCSFrameIndex)
3940 MinCSFrameIndex = FrameIdx;
3941 if ((unsigned)FrameIdx > MaxCSFrameIndex)
3942 MaxCSFrameIndex = FrameIdx;
3943 }
3944 LastReg = Reg;
3945 }
3946
3947 // Add hazard slot in the case where no FPR CSRs are present.
3948 if (AFI->hasStackHazardSlotIndex() &&
3949 HazardSlotIndex == std::numeric_limits<int>::max()) {
3950 HazardSlotIndex = MFI.CreateStackObject(Size: StackHazardSize, Alignment: Align(8), isSpillSlot: true);
3951 LLVM_DEBUG(dbgs() << "Created CSR Hazard at slot " << HazardSlotIndex
3952 << "\n");
3953 AFI->setStackHazardCSRSlotIndex(HazardSlotIndex);
3954 if ((unsigned)HazardSlotIndex < MinCSFrameIndex)
3955 MinCSFrameIndex = HazardSlotIndex;
3956 if ((unsigned)HazardSlotIndex > MaxCSFrameIndex)
3957 MaxCSFrameIndex = HazardSlotIndex;
3958 }
3959
3960 return true;
3961}
3962
3963bool AArch64FrameLowering::enableStackSlotScavenging(
3964 const MachineFunction &MF) const {
3965 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
3966 // If the function has streaming-mode changes, don't scavenge a
3967 // spillslot in the callee-save area, as that might require an
3968 // 'addvl' in the streaming-mode-changing call-sequence when the
3969 // function doesn't use a FP.
3970 if (AFI->hasStreamingModeChanges() && !hasFP(MF))
3971 return false;
3972 // Don't allow register salvaging with hazard slots, in case it moves objects
3973 // into the wrong place.
3974 if (AFI->hasStackHazardSlotIndex())
3975 return false;
3976 return AFI->hasCalleeSaveStackFreeSpace();
3977}
3978
3979/// returns true if there are any SVE callee saves.
3980static bool getSVECalleeSaveSlotRange(const MachineFrameInfo &MFI,
3981 int &Min, int &Max) {
3982 Min = std::numeric_limits<int>::max();
3983 Max = std::numeric_limits<int>::min();
3984
3985 if (!MFI.isCalleeSavedInfoValid())
3986 return false;
3987
3988 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
3989 for (auto &CS : CSI) {
3990 if (AArch64::ZPRRegClass.contains(Reg: CS.getReg()) ||
3991 AArch64::PPRRegClass.contains(Reg: CS.getReg())) {
3992 assert((Max == std::numeric_limits<int>::min() ||
3993 Max + 1 == CS.getFrameIdx()) &&
3994 "SVE CalleeSaves are not consecutive");
3995
3996 Min = std::min(a: Min, b: CS.getFrameIdx());
3997 Max = std::max(a: Max, b: CS.getFrameIdx());
3998 }
3999 }
4000 return Min != std::numeric_limits<int>::max();
4001}
4002
4003// Process all the SVE stack objects and determine offsets for each
4004// object. If AssignOffsets is true, the offsets get assigned.
4005// Fills in the first and last callee-saved frame indices into
4006// Min/MaxCSFrameIndex, respectively.
4007// Returns the size of the stack.
4008static int64_t determineSVEStackObjectOffsets(MachineFrameInfo &MFI,
4009 int &MinCSFrameIndex,
4010 int &MaxCSFrameIndex,
4011 bool AssignOffsets) {
4012#ifndef NDEBUG
4013 // First process all fixed stack objects.
4014 for (int I = MFI.getObjectIndexBegin(); I != 0; ++I)
4015 assert(MFI.getStackID(I) != TargetStackID::ScalableVector &&
4016 "SVE vectors should never be passed on the stack by value, only by "
4017 "reference.");
4018#endif
4019
4020 auto Assign = [&MFI](int FI, int64_t Offset) {
4021 LLVM_DEBUG(dbgs() << "alloc FI(" << FI << ") at SP[" << Offset << "]\n");
4022 MFI.setObjectOffset(ObjectIdx: FI, SPOffset: Offset);
4023 };
4024
4025 int64_t Offset = 0;
4026
4027 // Then process all callee saved slots.
4028 if (getSVECalleeSaveSlotRange(MFI, Min&: MinCSFrameIndex, Max&: MaxCSFrameIndex)) {
4029 // Assign offsets to the callee save slots.
4030 for (int I = MinCSFrameIndex; I <= MaxCSFrameIndex; ++I) {
4031 Offset += MFI.getObjectSize(ObjectIdx: I);
4032 Offset = alignTo(Size: Offset, A: MFI.getObjectAlign(ObjectIdx: I));
4033 if (AssignOffsets)
4034 Assign(I, -Offset);
4035 }
4036 }
4037
4038 // Ensure that the Callee-save area is aligned to 16bytes.
4039 Offset = alignTo(Size: Offset, A: Align(16U));
4040
4041 // Create a buffer of SVE objects to allocate and sort it.
4042 SmallVector<int, 8> ObjectsToAllocate;
4043 // If we have a stack protector, and we've previously decided that we have SVE
4044 // objects on the stack and thus need it to go in the SVE stack area, then it
4045 // needs to go first.
4046 int StackProtectorFI = -1;
4047 if (MFI.hasStackProtectorIndex()) {
4048 StackProtectorFI = MFI.getStackProtectorIndex();
4049 if (MFI.getStackID(ObjectIdx: StackProtectorFI) == TargetStackID::ScalableVector)
4050 ObjectsToAllocate.push_back(Elt: StackProtectorFI);
4051 }
4052 for (int I = 0, E = MFI.getObjectIndexEnd(); I != E; ++I) {
4053 unsigned StackID = MFI.getStackID(ObjectIdx: I);
4054 if (StackID != TargetStackID::ScalableVector)
4055 continue;
4056 if (I == StackProtectorFI)
4057 continue;
4058 if (MaxCSFrameIndex >= I && I >= MinCSFrameIndex)
4059 continue;
4060 if (MFI.isDeadObjectIndex(ObjectIdx: I))
4061 continue;
4062
4063 ObjectsToAllocate.push_back(Elt: I);
4064 }
4065
4066 // Allocate all SVE locals and spills
4067 for (unsigned FI : ObjectsToAllocate) {
4068 Align Alignment = MFI.getObjectAlign(ObjectIdx: FI);
4069 // FIXME: Given that the length of SVE vectors is not necessarily a power of
4070 // two, we'd need to align every object dynamically at runtime if the
4071 // alignment is larger than 16. This is not yet supported.
4072 if (Alignment > Align(16))
4073 report_fatal_error(
4074 reason: "Alignment of scalable vectors > 16 bytes is not yet supported");
4075
4076 Offset = alignTo(Size: Offset + MFI.getObjectSize(ObjectIdx: FI), A: Alignment);
4077 if (AssignOffsets)
4078 Assign(FI, -Offset);
4079 }
4080
4081 return Offset;
4082}
4083
4084int64_t AArch64FrameLowering::estimateSVEStackObjectOffsets(
4085 MachineFrameInfo &MFI) const {
4086 int MinCSFrameIndex, MaxCSFrameIndex;
4087 return determineSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex, AssignOffsets: false);
4088}
4089
4090int64_t AArch64FrameLowering::assignSVEStackObjectOffsets(
4091 MachineFrameInfo &MFI, int &MinCSFrameIndex, int &MaxCSFrameIndex) const {
4092 return determineSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex,
4093 AssignOffsets: true);
4094}
4095
4096void AArch64FrameLowering::processFunctionBeforeFrameFinalized(
4097 MachineFunction &MF, RegScavenger *RS) const {
4098 MachineFrameInfo &MFI = MF.getFrameInfo();
4099
4100 assert(getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown &&
4101 "Upwards growing stack unsupported");
4102
4103 int MinCSFrameIndex, MaxCSFrameIndex;
4104 int64_t SVEStackSize =
4105 assignSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex);
4106
4107 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
4108 AFI->setStackSizeSVE(alignTo(Value: SVEStackSize, Align: 16U));
4109 AFI->setMinMaxSVECSFrameIndex(Min: MinCSFrameIndex, Max: MaxCSFrameIndex);
4110
4111 // If this function isn't doing Win64-style C++ EH, we don't need to do
4112 // anything.
4113 if (!MF.hasEHFunclets())
4114 return;
4115 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
4116 WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo();
4117
4118 MachineBasicBlock &MBB = MF.front();
4119 auto MBBI = MBB.begin();
4120 while (MBBI != MBB.end() && MBBI->getFlag(Flag: MachineInstr::FrameSetup))
4121 ++MBBI;
4122
4123 // Create an UnwindHelp object.
4124 // The UnwindHelp object is allocated at the start of the fixed object area
4125 int64_t FixedObject =
4126 getFixedObjectSize(MF, AFI, /*IsWin64*/ true, /*IsFunclet*/ false);
4127 int UnwindHelpFI = MFI.CreateFixedObject(/*Size*/ 8,
4128 /*SPOffset*/ -FixedObject,
4129 /*IsImmutable=*/false);
4130 EHInfo.UnwindHelpFrameIdx = UnwindHelpFI;
4131
4132 // We need to store -2 into the UnwindHelp object at the start of the
4133 // function.
4134 DebugLoc DL;
4135 RS->enterBasicBlockEnd(MBB);
4136 RS->backward(I: MBBI);
4137 Register DstReg = RS->FindUnusedReg(RC: &AArch64::GPR64commonRegClass);
4138 assert(DstReg && "There must be a free register after frame setup");
4139 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AArch64::MOVi64imm), DestReg: DstReg).addImm(Val: -2);
4140 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AArch64::STURXi))
4141 .addReg(RegNo: DstReg, flags: getKillRegState(B: true))
4142 .addFrameIndex(Idx: UnwindHelpFI)
4143 .addImm(Val: 0);
4144}
4145
4146namespace {
4147struct TagStoreInstr {
4148 MachineInstr *MI;
4149 int64_t Offset, Size;
4150 explicit TagStoreInstr(MachineInstr *MI, int64_t Offset, int64_t Size)
4151 : MI(MI), Offset(Offset), Size(Size) {}
4152};
4153
4154class TagStoreEdit {
4155 MachineFunction *MF;
4156 MachineBasicBlock *MBB;
4157 MachineRegisterInfo *MRI;
4158 // Tag store instructions that are being replaced.
4159 SmallVector<TagStoreInstr, 8> TagStores;
4160 // Combined memref arguments of the above instructions.
4161 SmallVector<MachineMemOperand *, 8> CombinedMemRefs;
4162
4163 // Replace allocation tags in [FrameReg + FrameRegOffset, FrameReg +
4164 // FrameRegOffset + Size) with the address tag of SP.
4165 Register FrameReg;
4166 StackOffset FrameRegOffset;
4167 int64_t Size;
4168 // If not std::nullopt, move FrameReg to (FrameReg + FrameRegUpdate) at the
4169 // end.
4170 std::optional<int64_t> FrameRegUpdate;
4171 // MIFlags for any FrameReg updating instructions.
4172 unsigned FrameRegUpdateFlags;
4173
4174 // Use zeroing instruction variants.
4175 bool ZeroData;
4176 DebugLoc DL;
4177
4178 void emitUnrolled(MachineBasicBlock::iterator InsertI);
4179 void emitLoop(MachineBasicBlock::iterator InsertI);
4180
4181public:
4182 TagStoreEdit(MachineBasicBlock *MBB, bool ZeroData)
4183 : MBB(MBB), ZeroData(ZeroData) {
4184 MF = MBB->getParent();
4185 MRI = &MF->getRegInfo();
4186 }
4187 // Add an instruction to be replaced. Instructions must be added in the
4188 // ascending order of Offset, and have to be adjacent.
4189 void addInstruction(TagStoreInstr I) {
4190 assert((TagStores.empty() ||
4191 TagStores.back().Offset + TagStores.back().Size == I.Offset) &&
4192 "Non-adjacent tag store instructions.");
4193 TagStores.push_back(Elt: I);
4194 }
4195 void clear() { TagStores.clear(); }
4196 // Emit equivalent code at the given location, and erase the current set of
4197 // instructions. May skip if the replacement is not profitable. May invalidate
4198 // the input iterator and replace it with a valid one.
4199 void emitCode(MachineBasicBlock::iterator &InsertI,
4200 const AArch64FrameLowering *TFI, bool TryMergeSPUpdate);
4201};
4202
4203void TagStoreEdit::emitUnrolled(MachineBasicBlock::iterator InsertI) {
4204 const AArch64InstrInfo *TII =
4205 MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
4206
4207 const int64_t kMinOffset = -256 * 16;
4208 const int64_t kMaxOffset = 255 * 16;
4209
4210 Register BaseReg = FrameReg;
4211 int64_t BaseRegOffsetBytes = FrameRegOffset.getFixed();
4212 if (BaseRegOffsetBytes < kMinOffset ||
4213 BaseRegOffsetBytes + (Size - Size % 32) > kMaxOffset ||
4214 // BaseReg can be FP, which is not necessarily aligned to 16-bytes. In
4215 // that case, BaseRegOffsetBytes will not be aligned to 16 bytes, which
4216 // is required for the offset of ST2G.
4217 BaseRegOffsetBytes % 16 != 0) {
4218 Register ScratchReg = MRI->createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
4219 emitFrameOffset(MBB&: *MBB, MBBI: InsertI, DL, DestReg: ScratchReg, SrcReg: BaseReg,
4220 Offset: StackOffset::getFixed(Fixed: BaseRegOffsetBytes), TII);
4221 BaseReg = ScratchReg;
4222 BaseRegOffsetBytes = 0;
4223 }
4224
4225 MachineInstr *LastI = nullptr;
4226 while (Size) {
4227 int64_t InstrSize = (Size > 16) ? 32 : 16;
4228 unsigned Opcode =
4229 InstrSize == 16
4230 ? (ZeroData ? AArch64::STZGi : AArch64::STGi)
4231 : (ZeroData ? AArch64::STZ2Gi : AArch64::ST2Gi);
4232 assert(BaseRegOffsetBytes % 16 == 0);
4233 MachineInstr *I = BuildMI(BB&: *MBB, I: InsertI, MIMD: DL, MCID: TII->get(Opcode))
4234 .addReg(RegNo: AArch64::SP)
4235 .addReg(RegNo: BaseReg)
4236 .addImm(Val: BaseRegOffsetBytes / 16)
4237 .setMemRefs(CombinedMemRefs);
4238 // A store to [BaseReg, #0] should go last for an opportunity to fold the
4239 // final SP adjustment in the epilogue.
4240 if (BaseRegOffsetBytes == 0)
4241 LastI = I;
4242 BaseRegOffsetBytes += InstrSize;
4243 Size -= InstrSize;
4244 }
4245
4246 if (LastI)
4247 MBB->splice(Where: InsertI, Other: MBB, From: LastI);
4248}
4249
4250void TagStoreEdit::emitLoop(MachineBasicBlock::iterator InsertI) {
4251 const AArch64InstrInfo *TII =
4252 MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
4253
4254 Register BaseReg = FrameRegUpdate
4255 ? FrameReg
4256 : MRI->createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
4257 Register SizeReg = MRI->createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
4258
4259 emitFrameOffset(MBB&: *MBB, MBBI: InsertI, DL, DestReg: BaseReg, SrcReg: FrameReg, Offset: FrameRegOffset, TII);
4260
4261 int64_t LoopSize = Size;
4262 // If the loop size is not a multiple of 32, split off one 16-byte store at
4263 // the end to fold BaseReg update into.
4264 if (FrameRegUpdate && *FrameRegUpdate)
4265 LoopSize -= LoopSize % 32;
4266 MachineInstr *LoopI = BuildMI(BB&: *MBB, I: InsertI, MIMD: DL,
4267 MCID: TII->get(Opcode: ZeroData ? AArch64::STZGloop_wback
4268 : AArch64::STGloop_wback))
4269 .addDef(RegNo: SizeReg)
4270 .addDef(RegNo: BaseReg)
4271 .addImm(Val: LoopSize)
4272 .addReg(RegNo: BaseReg)
4273 .setMemRefs(CombinedMemRefs);
4274 if (FrameRegUpdate)
4275 LoopI->setFlags(FrameRegUpdateFlags);
4276
4277 int64_t ExtraBaseRegUpdate =
4278 FrameRegUpdate ? (*FrameRegUpdate - FrameRegOffset.getFixed() - Size) : 0;
4279 if (LoopSize < Size) {
4280 assert(FrameRegUpdate);
4281 assert(Size - LoopSize == 16);
4282 // Tag 16 more bytes at BaseReg and update BaseReg.
4283 BuildMI(BB&: *MBB, I: InsertI, MIMD: DL,
4284 MCID: TII->get(Opcode: ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex))
4285 .addDef(RegNo: BaseReg)
4286 .addReg(RegNo: BaseReg)
4287 .addReg(RegNo: BaseReg)
4288 .addImm(Val: 1 + ExtraBaseRegUpdate / 16)
4289 .setMemRefs(CombinedMemRefs)
4290 .setMIFlags(FrameRegUpdateFlags);
4291 } else if (ExtraBaseRegUpdate) {
4292 // Update BaseReg.
4293 BuildMI(
4294 BB&: *MBB, I: InsertI, MIMD: DL,
4295 MCID: TII->get(Opcode: ExtraBaseRegUpdate > 0 ? AArch64::ADDXri : AArch64::SUBXri))
4296 .addDef(RegNo: BaseReg)
4297 .addReg(RegNo: BaseReg)
4298 .addImm(Val: std::abs(i: ExtraBaseRegUpdate))
4299 .addImm(Val: 0)
4300 .setMIFlags(FrameRegUpdateFlags);
4301 }
4302}
4303
4304// Check if *II is a register update that can be merged into STGloop that ends
4305// at (Reg + Size). RemainingOffset is the required adjustment to Reg after the
4306// end of the loop.
4307bool canMergeRegUpdate(MachineBasicBlock::iterator II, unsigned Reg,
4308 int64_t Size, int64_t *TotalOffset) {
4309 MachineInstr &MI = *II;
4310 if ((MI.getOpcode() == AArch64::ADDXri ||
4311 MI.getOpcode() == AArch64::SUBXri) &&
4312 MI.getOperand(i: 0).getReg() == Reg && MI.getOperand(i: 1).getReg() == Reg) {
4313 unsigned Shift = AArch64_AM::getShiftValue(Imm: MI.getOperand(i: 3).getImm());
4314 int64_t Offset = MI.getOperand(i: 2).getImm() << Shift;
4315 if (MI.getOpcode() == AArch64::SUBXri)
4316 Offset = -Offset;
4317 int64_t AbsPostOffset = std::abs(i: Offset - Size);
4318 const int64_t kMaxOffset =
4319 0xFFF; // Max encoding for unshifted ADDXri / SUBXri
4320 if (AbsPostOffset <= kMaxOffset && AbsPostOffset % 16 == 0) {
4321 *TotalOffset = Offset;
4322 return true;
4323 }
4324 }
4325 return false;
4326}
4327
4328void mergeMemRefs(const SmallVectorImpl<TagStoreInstr> &TSE,
4329 SmallVectorImpl<MachineMemOperand *> &MemRefs) {
4330 MemRefs.clear();
4331 for (auto &TS : TSE) {
4332 MachineInstr *MI = TS.MI;
4333 // An instruction without memory operands may access anything. Be
4334 // conservative and return an empty list.
4335 if (MI->memoperands_empty()) {
4336 MemRefs.clear();
4337 return;
4338 }
4339 MemRefs.append(in_start: MI->memoperands_begin(), in_end: MI->memoperands_end());
4340 }
4341}
4342
4343void TagStoreEdit::emitCode(MachineBasicBlock::iterator &InsertI,
4344 const AArch64FrameLowering *TFI,
4345 bool TryMergeSPUpdate) {
4346 if (TagStores.empty())
4347 return;
4348 TagStoreInstr &FirstTagStore = TagStores[0];
4349 TagStoreInstr &LastTagStore = TagStores[TagStores.size() - 1];
4350 Size = LastTagStore.Offset - FirstTagStore.Offset + LastTagStore.Size;
4351 DL = TagStores[0].MI->getDebugLoc();
4352
4353 Register Reg;
4354 FrameRegOffset = TFI->resolveFrameOffsetReference(
4355 MF: *MF, ObjectOffset: FirstTagStore.Offset, isFixed: false /*isFixed*/, isSVE: false /*isSVE*/, FrameReg&: Reg,
4356 /*PreferFP=*/false, /*ForSimm=*/true);
4357 FrameReg = Reg;
4358 FrameRegUpdate = std::nullopt;
4359
4360 mergeMemRefs(TSE: TagStores, MemRefs&: CombinedMemRefs);
4361
4362 LLVM_DEBUG({
4363 dbgs() << "Replacing adjacent STG instructions:\n";
4364 for (const auto &Instr : TagStores) {
4365 dbgs() << " " << *Instr.MI;
4366 }
4367 });
4368
4369 // Size threshold where a loop becomes shorter than a linear sequence of
4370 // tagging instructions.
4371 const int kSetTagLoopThreshold = 176;
4372 if (Size < kSetTagLoopThreshold) {
4373 if (TagStores.size() < 2)
4374 return;
4375 emitUnrolled(InsertI);
4376 } else {
4377 MachineInstr *UpdateInstr = nullptr;
4378 int64_t TotalOffset = 0;
4379 if (TryMergeSPUpdate) {
4380 // See if we can merge base register update into the STGloop.
4381 // This is done in AArch64LoadStoreOptimizer for "normal" stores,
4382 // but STGloop is way too unusual for that, and also it only
4383 // realistically happens in function epilogue. Also, STGloop is expanded
4384 // before that pass.
4385 if (InsertI != MBB->end() &&
4386 canMergeRegUpdate(II: InsertI, Reg: FrameReg, Size: FrameRegOffset.getFixed() + Size,
4387 TotalOffset: &TotalOffset)) {
4388 UpdateInstr = &*InsertI++;
4389 LLVM_DEBUG(dbgs() << "Folding SP update into loop:\n "
4390 << *UpdateInstr);
4391 }
4392 }
4393
4394 if (!UpdateInstr && TagStores.size() < 2)
4395 return;
4396
4397 if (UpdateInstr) {
4398 FrameRegUpdate = TotalOffset;
4399 FrameRegUpdateFlags = UpdateInstr->getFlags();
4400 }
4401 emitLoop(InsertI);
4402 if (UpdateInstr)
4403 UpdateInstr->eraseFromParent();
4404 }
4405
4406 for (auto &TS : TagStores)
4407 TS.MI->eraseFromParent();
4408}
4409
4410bool isMergeableStackTaggingInstruction(MachineInstr &MI, int64_t &Offset,
4411 int64_t &Size, bool &ZeroData) {
4412 MachineFunction &MF = *MI.getParent()->getParent();
4413 const MachineFrameInfo &MFI = MF.getFrameInfo();
4414
4415 unsigned Opcode = MI.getOpcode();
4416 ZeroData = (Opcode == AArch64::STZGloop || Opcode == AArch64::STZGi ||
4417 Opcode == AArch64::STZ2Gi);
4418
4419 if (Opcode == AArch64::STGloop || Opcode == AArch64::STZGloop) {
4420 if (!MI.getOperand(i: 0).isDead() || !MI.getOperand(i: 1).isDead())
4421 return false;
4422 if (!MI.getOperand(i: 2).isImm() || !MI.getOperand(i: 3).isFI())
4423 return false;
4424 Offset = MFI.getObjectOffset(ObjectIdx: MI.getOperand(i: 3).getIndex());
4425 Size = MI.getOperand(i: 2).getImm();
4426 return true;
4427 }
4428
4429 if (Opcode == AArch64::STGi || Opcode == AArch64::STZGi)
4430 Size = 16;
4431 else if (Opcode == AArch64::ST2Gi || Opcode == AArch64::STZ2Gi)
4432 Size = 32;
4433 else
4434 return false;
4435
4436 if (MI.getOperand(i: 0).getReg() != AArch64::SP || !MI.getOperand(i: 1).isFI())
4437 return false;
4438
4439 Offset = MFI.getObjectOffset(ObjectIdx: MI.getOperand(i: 1).getIndex()) +
4440 16 * MI.getOperand(i: 2).getImm();
4441 return true;
4442}
4443
4444// Detect a run of memory tagging instructions for adjacent stack frame slots,
4445// and replace them with a shorter instruction sequence:
4446// * replace STG + STG with ST2G
4447// * replace STGloop + STGloop with STGloop
4448// This code needs to run when stack slot offsets are already known, but before
4449// FrameIndex operands in STG instructions are eliminated.
4450MachineBasicBlock::iterator tryMergeAdjacentSTG(MachineBasicBlock::iterator II,
4451 const AArch64FrameLowering *TFI,
4452 RegScavenger *RS) {
4453 bool FirstZeroData;
4454 int64_t Size, Offset;
4455 MachineInstr &MI = *II;
4456 MachineBasicBlock *MBB = MI.getParent();
4457 MachineBasicBlock::iterator NextI = ++II;
4458 if (&MI == &MBB->instr_back())
4459 return II;
4460 if (!isMergeableStackTaggingInstruction(MI, Offset, Size, ZeroData&: FirstZeroData))
4461 return II;
4462
4463 SmallVector<TagStoreInstr, 4> Instrs;
4464 Instrs.emplace_back(Args: &MI, Args&: Offset, Args&: Size);
4465
4466 constexpr int kScanLimit = 10;
4467 int Count = 0;
4468 for (MachineBasicBlock::iterator E = MBB->end();
4469 NextI != E && Count < kScanLimit; ++NextI) {
4470 MachineInstr &MI = *NextI;
4471 bool ZeroData;
4472 int64_t Size, Offset;
4473 // Collect instructions that update memory tags with a FrameIndex operand
4474 // and (when applicable) constant size, and whose output registers are dead
4475 // (the latter is almost always the case in practice). Since these
4476 // instructions effectively have no inputs or outputs, we are free to skip
4477 // any non-aliasing instructions in between without tracking used registers.
4478 if (isMergeableStackTaggingInstruction(MI, Offset, Size, ZeroData)) {
4479 if (ZeroData != FirstZeroData)
4480 break;
4481 Instrs.emplace_back(Args: &MI, Args&: Offset, Args&: Size);
4482 continue;
4483 }
4484
4485 // Only count non-transient, non-tagging instructions toward the scan
4486 // limit.
4487 if (!MI.isTransient())
4488 ++Count;
4489
4490 // Just in case, stop before the epilogue code starts.
4491 if (MI.getFlag(Flag: MachineInstr::FrameSetup) ||
4492 MI.getFlag(Flag: MachineInstr::FrameDestroy))
4493 break;
4494
4495 // Reject anything that may alias the collected instructions.
4496 if (MI.mayLoadOrStore() || MI.hasUnmodeledSideEffects())
4497 break;
4498 }
4499
4500 // New code will be inserted after the last tagging instruction we've found.
4501 MachineBasicBlock::iterator InsertI = Instrs.back().MI;
4502
4503 // All the gathered stack tag instructions are merged and placed after
4504 // last tag store in the list. The check should be made if the nzcv
4505 // flag is live at the point where we are trying to insert. Otherwise
4506 // the nzcv flag might get clobbered if any stg loops are present.
4507
4508 // FIXME : This approach of bailing out from merge is conservative in
4509 // some ways like even if stg loops are not present after merge the
4510 // insert list, this liveness check is done (which is not needed).
4511 LivePhysRegs LiveRegs(*(MBB->getParent()->getSubtarget().getRegisterInfo()));
4512 LiveRegs.addLiveOuts(MBB: *MBB);
4513 for (auto I = MBB->rbegin();; ++I) {
4514 MachineInstr &MI = *I;
4515 if (MI == InsertI)
4516 break;
4517 LiveRegs.stepBackward(MI: *I);
4518 }
4519 InsertI++;
4520 if (LiveRegs.contains(Reg: AArch64::NZCV))
4521 return InsertI;
4522
4523 llvm::stable_sort(Range&: Instrs,
4524 C: [](const TagStoreInstr &Left, const TagStoreInstr &Right) {
4525 return Left.Offset < Right.Offset;
4526 });
4527
4528 // Make sure that we don't have any overlapping stores.
4529 int64_t CurOffset = Instrs[0].Offset;
4530 for (auto &Instr : Instrs) {
4531 if (CurOffset > Instr.Offset)
4532 return NextI;
4533 CurOffset = Instr.Offset + Instr.Size;
4534 }
4535
4536 // Find contiguous runs of tagged memory and emit shorter instruction
4537 // sequencies for them when possible.
4538 TagStoreEdit TSE(MBB, FirstZeroData);
4539 std::optional<int64_t> EndOffset;
4540 for (auto &Instr : Instrs) {
4541 if (EndOffset && *EndOffset != Instr.Offset) {
4542 // Found a gap.
4543 TSE.emitCode(InsertI, TFI, /*TryMergeSPUpdate = */ false);
4544 TSE.clear();
4545 }
4546
4547 TSE.addInstruction(I: Instr);
4548 EndOffset = Instr.Offset + Instr.Size;
4549 }
4550
4551 const MachineFunction *MF = MBB->getParent();
4552 // Multiple FP/SP updates in a loop cannot be described by CFI instructions.
4553 TSE.emitCode(
4554 InsertI, TFI, /*TryMergeSPUpdate = */
4555 !MF->getInfo<AArch64FunctionInfo>()->needsAsyncDwarfUnwindInfo(MF: *MF));
4556
4557 return InsertI;
4558}
4559} // namespace
4560
4561MachineBasicBlock::iterator emitVGSaveRestore(MachineBasicBlock::iterator II,
4562 const AArch64FrameLowering *TFI) {
4563 MachineInstr &MI = *II;
4564 MachineBasicBlock *MBB = MI.getParent();
4565 MachineFunction *MF = MBB->getParent();
4566
4567 if (MI.getOpcode() != AArch64::VGSavePseudo &&
4568 MI.getOpcode() != AArch64::VGRestorePseudo)
4569 return II;
4570
4571 SMEAttrs FuncAttrs(MF->getFunction());
4572 bool LocallyStreaming =
4573 FuncAttrs.hasStreamingBody() && !FuncAttrs.hasStreamingInterface();
4574 const AArch64FunctionInfo *AFI = MF->getInfo<AArch64FunctionInfo>();
4575 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
4576 const AArch64InstrInfo *TII =
4577 MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
4578
4579 int64_t VGFrameIdx =
4580 LocallyStreaming ? AFI->getStreamingVGIdx() : AFI->getVGIdx();
4581 assert(VGFrameIdx != std::numeric_limits<int>::max() &&
4582 "Expected FrameIdx for VG");
4583
4584 unsigned CFIIndex;
4585 if (MI.getOpcode() == AArch64::VGSavePseudo) {
4586 const MachineFrameInfo &MFI = MF->getFrameInfo();
4587 int64_t Offset =
4588 MFI.getObjectOffset(ObjectIdx: VGFrameIdx) - TFI->getOffsetOfLocalArea();
4589 CFIIndex = MF->addFrameInst(Inst: MCCFIInstruction::createOffset(
4590 L: nullptr, Register: TRI->getDwarfRegNum(RegNum: AArch64::VG, isEH: true), Offset));
4591 } else
4592 CFIIndex = MF->addFrameInst(Inst: MCCFIInstruction::createRestore(
4593 L: nullptr, Register: TRI->getDwarfRegNum(RegNum: AArch64::VG, isEH: true)));
4594
4595 MachineInstr *UnwindInst = BuildMI(BB&: *MBB, I: II, MIMD: II->getDebugLoc(),
4596 MCID: TII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
4597 .addCFIIndex(CFIIndex);
4598
4599 MI.eraseFromParent();
4600 return UnwindInst->getIterator();
4601}
4602
4603void AArch64FrameLowering::processFunctionBeforeFrameIndicesReplaced(
4604 MachineFunction &MF, RegScavenger *RS = nullptr) const {
4605 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
4606 for (auto &BB : MF)
4607 for (MachineBasicBlock::iterator II = BB.begin(); II != BB.end();) {
4608 if (AFI->hasStreamingModeChanges())
4609 II = emitVGSaveRestore(II, TFI: this);
4610 if (StackTaggingMergeSetTag)
4611 II = tryMergeAdjacentSTG(II, TFI: this, RS);
4612 }
4613}
4614
4615/// For Win64 AArch64 EH, the offset to the Unwind object is from the SP
4616/// before the update. This is easily retrieved as it is exactly the offset
4617/// that is set in processFunctionBeforeFrameFinalized.
4618StackOffset AArch64FrameLowering::getFrameIndexReferencePreferSP(
4619 const MachineFunction &MF, int FI, Register &FrameReg,
4620 bool IgnoreSPUpdates) const {
4621 const MachineFrameInfo &MFI = MF.getFrameInfo();
4622 if (IgnoreSPUpdates) {
4623 LLVM_DEBUG(dbgs() << "Offset from the SP for " << FI << " is "
4624 << MFI.getObjectOffset(FI) << "\n");
4625 FrameReg = AArch64::SP;
4626 return StackOffset::getFixed(Fixed: MFI.getObjectOffset(ObjectIdx: FI));
4627 }
4628
4629 // Go to common code if we cannot provide sp + offset.
4630 if (MFI.hasVarSizedObjects() ||
4631 MF.getInfo<AArch64FunctionInfo>()->getStackSizeSVE() ||
4632 MF.getSubtarget().getRegisterInfo()->hasStackRealignment(MF))
4633 return getFrameIndexReference(MF, FI, FrameReg);
4634
4635 FrameReg = AArch64::SP;
4636 return getStackOffset(MF, ObjectOffset: MFI.getObjectOffset(ObjectIdx: FI));
4637}
4638
4639/// The parent frame offset (aka dispFrame) is only used on X86_64 to retrieve
4640/// the parent's frame pointer
4641unsigned AArch64FrameLowering::getWinEHParentFrameOffset(
4642 const MachineFunction &MF) const {
4643 return 0;
4644}
4645
4646/// Funclets only need to account for space for the callee saved registers,
4647/// as the locals are accounted for in the parent's stack frame.
4648unsigned AArch64FrameLowering::getWinEHFuncletFrameSize(
4649 const MachineFunction &MF) const {
4650 // This is the size of the pushed CSRs.
4651 unsigned CSSize =
4652 MF.getInfo<AArch64FunctionInfo>()->getCalleeSavedStackSize();
4653 // This is the amount of stack a funclet needs to allocate.
4654 return alignTo(Size: CSSize + MF.getFrameInfo().getMaxCallFrameSize(),
4655 A: getStackAlign());
4656}
4657
4658namespace {
4659struct FrameObject {
4660 bool IsValid = false;
4661 // Index of the object in MFI.
4662 int ObjectIndex = 0;
4663 // Group ID this object belongs to.
4664 int GroupIndex = -1;
4665 // This object should be placed first (closest to SP).
4666 bool ObjectFirst = false;
4667 // This object's group (which always contains the object with
4668 // ObjectFirst==true) should be placed first.
4669 bool GroupFirst = false;
4670
4671 // Used to distinguish between FP and GPR accesses. The values are decided so
4672 // that they sort FPR < Hazard < GPR and they can be or'd together.
4673 unsigned Accesses = 0;
4674 enum { AccessFPR = 1, AccessHazard = 2, AccessGPR = 4 };
4675};
4676
4677class GroupBuilder {
4678 SmallVector<int, 8> CurrentMembers;
4679 int NextGroupIndex = 0;
4680 std::vector<FrameObject> &Objects;
4681
4682public:
4683 GroupBuilder(std::vector<FrameObject> &Objects) : Objects(Objects) {}
4684 void AddMember(int Index) { CurrentMembers.push_back(Elt: Index); }
4685 void EndCurrentGroup() {
4686 if (CurrentMembers.size() > 1) {
4687 // Create a new group with the current member list. This might remove them
4688 // from their pre-existing groups. That's OK, dealing with overlapping
4689 // groups is too hard and unlikely to make a difference.
4690 LLVM_DEBUG(dbgs() << "group:");
4691 for (int Index : CurrentMembers) {
4692 Objects[Index].GroupIndex = NextGroupIndex;
4693 LLVM_DEBUG(dbgs() << " " << Index);
4694 }
4695 LLVM_DEBUG(dbgs() << "\n");
4696 NextGroupIndex++;
4697 }
4698 CurrentMembers.clear();
4699 }
4700};
4701
4702bool FrameObjectCompare(const FrameObject &A, const FrameObject &B) {
4703 // Objects at a lower index are closer to FP; objects at a higher index are
4704 // closer to SP.
4705 //
4706 // For consistency in our comparison, all invalid objects are placed
4707 // at the end. This also allows us to stop walking when we hit the
4708 // first invalid item after it's all sorted.
4709 //
4710 // If we want to include a stack hazard region, order FPR accesses < the
4711 // hazard object < GPRs accesses in order to create a separation between the
4712 // two. For the Accesses field 1 = FPR, 2 = Hazard Object, 4 = GPR.
4713 //
4714 // Otherwise the "first" object goes first (closest to SP), followed by the
4715 // members of the "first" group.
4716 //
4717 // The rest are sorted by the group index to keep the groups together.
4718 // Higher numbered groups are more likely to be around longer (i.e. untagged
4719 // in the function epilogue and not at some earlier point). Place them closer
4720 // to SP.
4721 //
4722 // If all else equal, sort by the object index to keep the objects in the
4723 // original order.
4724 return std::make_tuple(args: !A.IsValid, args: A.Accesses, args: A.ObjectFirst, args: A.GroupFirst,
4725 args: A.GroupIndex, args: A.ObjectIndex) <
4726 std::make_tuple(args: !B.IsValid, args: B.Accesses, args: B.ObjectFirst, args: B.GroupFirst,
4727 args: B.GroupIndex, args: B.ObjectIndex);
4728}
4729} // namespace
4730
4731void AArch64FrameLowering::orderFrameObjects(
4732 const MachineFunction &MF, SmallVectorImpl<int> &ObjectsToAllocate) const {
4733 if (!OrderFrameObjects || ObjectsToAllocate.empty())
4734 return;
4735
4736 const AArch64FunctionInfo &AFI = *MF.getInfo<AArch64FunctionInfo>();
4737 const MachineFrameInfo &MFI = MF.getFrameInfo();
4738 std::vector<FrameObject> FrameObjects(MFI.getObjectIndexEnd());
4739 for (auto &Obj : ObjectsToAllocate) {
4740 FrameObjects[Obj].IsValid = true;
4741 FrameObjects[Obj].ObjectIndex = Obj;
4742 }
4743
4744 // Identify FPR vs GPR slots for hazards, and stack slots that are tagged at
4745 // the same time.
4746 GroupBuilder GB(FrameObjects);
4747 for (auto &MBB : MF) {
4748 for (auto &MI : MBB) {
4749 if (MI.isDebugInstr())
4750 continue;
4751
4752 if (AFI.hasStackHazardSlotIndex()) {
4753 std::optional<int> FI = getLdStFrameID(MI, MFI);
4754 if (FI && *FI >= 0 && *FI < (int)FrameObjects.size()) {
4755 if (MFI.getStackID(ObjectIdx: *FI) == TargetStackID::ScalableVector ||
4756 AArch64InstrInfo::isFpOrNEON(MI))
4757 FrameObjects[*FI].Accesses |= FrameObject::AccessFPR;
4758 else
4759 FrameObjects[*FI].Accesses |= FrameObject::AccessGPR;
4760 }
4761 }
4762
4763 int OpIndex;
4764 switch (MI.getOpcode()) {
4765 case AArch64::STGloop:
4766 case AArch64::STZGloop:
4767 OpIndex = 3;
4768 break;
4769 case AArch64::STGi:
4770 case AArch64::STZGi:
4771 case AArch64::ST2Gi:
4772 case AArch64::STZ2Gi:
4773 OpIndex = 1;
4774 break;
4775 default:
4776 OpIndex = -1;
4777 }
4778
4779 int TaggedFI = -1;
4780 if (OpIndex >= 0) {
4781 const MachineOperand &MO = MI.getOperand(i: OpIndex);
4782 if (MO.isFI()) {
4783 int FI = MO.getIndex();
4784 if (FI >= 0 && FI < MFI.getObjectIndexEnd() &&
4785 FrameObjects[FI].IsValid)
4786 TaggedFI = FI;
4787 }
4788 }
4789
4790 // If this is a stack tagging instruction for a slot that is not part of a
4791 // group yet, either start a new group or add it to the current one.
4792 if (TaggedFI >= 0)
4793 GB.AddMember(Index: TaggedFI);
4794 else
4795 GB.EndCurrentGroup();
4796 }
4797 // Groups should never span multiple basic blocks.
4798 GB.EndCurrentGroup();
4799 }
4800
4801 if (AFI.hasStackHazardSlotIndex()) {
4802 FrameObjects[AFI.getStackHazardSlotIndex()].Accesses =
4803 FrameObject::AccessHazard;
4804 // If a stack object is unknown or both GPR and FPR, sort it into GPR.
4805 for (auto &Obj : FrameObjects)
4806 if (!Obj.Accesses ||
4807 Obj.Accesses == (FrameObject::AccessGPR | FrameObject::AccessFPR))
4808 Obj.Accesses = FrameObject::AccessGPR;
4809 }
4810
4811 // If the function's tagged base pointer is pinned to a stack slot, we want to
4812 // put that slot first when possible. This will likely place it at SP + 0,
4813 // and save one instruction when generating the base pointer because IRG does
4814 // not allow an immediate offset.
4815 std::optional<int> TBPI = AFI.getTaggedBasePointerIndex();
4816 if (TBPI) {
4817 FrameObjects[*TBPI].ObjectFirst = true;
4818 FrameObjects[*TBPI].GroupFirst = true;
4819 int FirstGroupIndex = FrameObjects[*TBPI].GroupIndex;
4820 if (FirstGroupIndex >= 0)
4821 for (FrameObject &Object : FrameObjects)
4822 if (Object.GroupIndex == FirstGroupIndex)
4823 Object.GroupFirst = true;
4824 }
4825
4826 llvm::stable_sort(Range&: FrameObjects, C: FrameObjectCompare);
4827
4828 int i = 0;
4829 for (auto &Obj : FrameObjects) {
4830 // All invalid items are sorted at the end, so it's safe to stop.
4831 if (!Obj.IsValid)
4832 break;
4833 ObjectsToAllocate[i++] = Obj.ObjectIndex;
4834 }
4835
4836 LLVM_DEBUG({
4837 dbgs() << "Final frame order:\n";
4838 for (auto &Obj : FrameObjects) {
4839 if (!Obj.IsValid)
4840 break;
4841 dbgs() << " " << Obj.ObjectIndex << ": group " << Obj.GroupIndex;
4842 if (Obj.ObjectFirst)
4843 dbgs() << ", first";
4844 if (Obj.GroupFirst)
4845 dbgs() << ", group-first";
4846 dbgs() << "\n";
4847 }
4848 });
4849}
4850
4851/// Emit a loop to decrement SP until it is equal to TargetReg, with probes at
4852/// least every ProbeSize bytes. Returns an iterator of the first instruction
4853/// after the loop. The difference between SP and TargetReg must be an exact
4854/// multiple of ProbeSize.
4855MachineBasicBlock::iterator
4856AArch64FrameLowering::inlineStackProbeLoopExactMultiple(
4857 MachineBasicBlock::iterator MBBI, int64_t ProbeSize,
4858 Register TargetReg) const {
4859 MachineBasicBlock &MBB = *MBBI->getParent();
4860 MachineFunction &MF = *MBB.getParent();
4861 const AArch64InstrInfo *TII =
4862 MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
4863 DebugLoc DL = MBB.findDebugLoc(MBBI);
4864
4865 MachineFunction::iterator MBBInsertPoint = std::next(x: MBB.getIterator());
4866 MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
4867 MF.insert(MBBI: MBBInsertPoint, MBB: LoopMBB);
4868 MachineBasicBlock *ExitMBB = MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
4869 MF.insert(MBBI: MBBInsertPoint, MBB: ExitMBB);
4870
4871 // SUB SP, SP, #ProbeSize (or equivalent if ProbeSize is not encodable
4872 // in SUB).
4873 emitFrameOffset(MBB&: *LoopMBB, MBBI: LoopMBB->end(), DL, DestReg: AArch64::SP, SrcReg: AArch64::SP,
4874 Offset: StackOffset::getFixed(Fixed: -ProbeSize), TII,
4875 MachineInstr::FrameSetup);
4876 // STR XZR, [SP]
4877 BuildMI(BB&: *LoopMBB, I: LoopMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::STRXui))
4878 .addReg(RegNo: AArch64::XZR)
4879 .addReg(RegNo: AArch64::SP)
4880 .addImm(Val: 0)
4881 .setMIFlags(MachineInstr::FrameSetup);
4882 // CMP SP, TargetReg
4883 BuildMI(BB&: *LoopMBB, I: LoopMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::SUBSXrx64),
4884 DestReg: AArch64::XZR)
4885 .addReg(RegNo: AArch64::SP)
4886 .addReg(RegNo: TargetReg)
4887 .addImm(Val: AArch64_AM::getArithExtendImm(ET: AArch64_AM::UXTX, Imm: 0))
4888 .setMIFlags(MachineInstr::FrameSetup);
4889 // B.CC Loop
4890 BuildMI(BB&: *LoopMBB, I: LoopMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::Bcc))
4891 .addImm(Val: AArch64CC::NE)
4892 .addMBB(MBB: LoopMBB)
4893 .setMIFlags(MachineInstr::FrameSetup);
4894
4895 LoopMBB->addSuccessor(Succ: ExitMBB);
4896 LoopMBB->addSuccessor(Succ: LoopMBB);
4897 // Synthesize the exit MBB.
4898 ExitMBB->splice(Where: ExitMBB->end(), Other: &MBB, From: MBBI, To: MBB.end());
4899 ExitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
4900 MBB.addSuccessor(Succ: LoopMBB);
4901 // Update liveins.
4902 fullyRecomputeLiveIns(MBBs: {ExitMBB, LoopMBB});
4903
4904 return ExitMBB->begin();
4905}
4906
4907void AArch64FrameLowering::inlineStackProbeFixed(
4908 MachineBasicBlock::iterator MBBI, Register ScratchReg, int64_t FrameSize,
4909 StackOffset CFAOffset) const {
4910 MachineBasicBlock *MBB = MBBI->getParent();
4911 MachineFunction &MF = *MBB->getParent();
4912 const AArch64InstrInfo *TII =
4913 MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
4914 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
4915 bool EmitAsyncCFI = AFI->needsAsyncDwarfUnwindInfo(MF);
4916 bool HasFP = hasFP(MF);
4917
4918 DebugLoc DL;
4919 int64_t ProbeSize = MF.getInfo<AArch64FunctionInfo>()->getStackProbeSize();
4920 int64_t NumBlocks = FrameSize / ProbeSize;
4921 int64_t ResidualSize = FrameSize % ProbeSize;
4922
4923 LLVM_DEBUG(dbgs() << "Stack probing: total " << FrameSize << " bytes, "
4924 << NumBlocks << " blocks of " << ProbeSize
4925 << " bytes, plus " << ResidualSize << " bytes\n");
4926
4927 // Decrement SP by NumBlock * ProbeSize bytes, with either unrolled or
4928 // ordinary loop.
4929 if (NumBlocks <= AArch64::StackProbeMaxLoopUnroll) {
4930 for (int i = 0; i < NumBlocks; ++i) {
4931 // SUB SP, SP, #ProbeSize (or equivalent if ProbeSize is not
4932 // encodable in a SUB).
4933 emitFrameOffset(MBB&: *MBB, MBBI, DL, DestReg: AArch64::SP, SrcReg: AArch64::SP,
4934 Offset: StackOffset::getFixed(Fixed: -ProbeSize), TII,
4935 MachineInstr::FrameSetup, SetNZCV: false, NeedsWinCFI: false, HasWinCFI: nullptr,
4936 EmitCFAOffset: EmitAsyncCFI && !HasFP, InitialOffset: CFAOffset);
4937 CFAOffset += StackOffset::getFixed(Fixed: ProbeSize);
4938 // STR XZR, [SP]
4939 BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::STRXui))
4940 .addReg(RegNo: AArch64::XZR)
4941 .addReg(RegNo: AArch64::SP)
4942 .addImm(Val: 0)
4943 .setMIFlags(MachineInstr::FrameSetup);
4944 }
4945 } else if (NumBlocks != 0) {
4946 // SUB ScratchReg, SP, #FrameSize (or equivalent if FrameSize is not
4947 // encodable in ADD). ScrathReg may temporarily become the CFA register.
4948 emitFrameOffset(MBB&: *MBB, MBBI, DL, DestReg: ScratchReg, SrcReg: AArch64::SP,
4949 Offset: StackOffset::getFixed(Fixed: -ProbeSize * NumBlocks), TII,
4950 MachineInstr::FrameSetup, SetNZCV: false, NeedsWinCFI: false, HasWinCFI: nullptr,
4951 EmitCFAOffset: EmitAsyncCFI && !HasFP, InitialOffset: CFAOffset);
4952 CFAOffset += StackOffset::getFixed(Fixed: ProbeSize * NumBlocks);
4953 MBBI = inlineStackProbeLoopExactMultiple(MBBI, ProbeSize, TargetReg: ScratchReg);
4954 MBB = MBBI->getParent();
4955 if (EmitAsyncCFI && !HasFP) {
4956 // Set the CFA register back to SP.
4957 const AArch64RegisterInfo &RegInfo =
4958 *MF.getSubtarget<AArch64Subtarget>().getRegisterInfo();
4959 unsigned Reg = RegInfo.getDwarfRegNum(RegNum: AArch64::SP, isEH: true);
4960 unsigned CFIIndex =
4961 MF.addFrameInst(Inst: MCCFIInstruction::createDefCfaRegister(L: nullptr, Register: Reg));
4962 BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
4963 .addCFIIndex(CFIIndex)
4964 .setMIFlags(MachineInstr::FrameSetup);
4965 }
4966 }
4967
4968 if (ResidualSize != 0) {
4969 // SUB SP, SP, #ResidualSize (or equivalent if ResidualSize is not encodable
4970 // in SUB).
4971 emitFrameOffset(MBB&: *MBB, MBBI, DL, DestReg: AArch64::SP, SrcReg: AArch64::SP,
4972 Offset: StackOffset::getFixed(Fixed: -ResidualSize), TII,
4973 MachineInstr::FrameSetup, SetNZCV: false, NeedsWinCFI: false, HasWinCFI: nullptr,
4974 EmitCFAOffset: EmitAsyncCFI && !HasFP, InitialOffset: CFAOffset);
4975 if (ResidualSize > AArch64::StackProbeMaxUnprobedStack) {
4976 // STR XZR, [SP]
4977 BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::STRXui))
4978 .addReg(RegNo: AArch64::XZR)
4979 .addReg(RegNo: AArch64::SP)
4980 .addImm(Val: 0)
4981 .setMIFlags(MachineInstr::FrameSetup);
4982 }
4983 }
4984}
4985
4986void AArch64FrameLowering::inlineStackProbe(MachineFunction &MF,
4987 MachineBasicBlock &MBB) const {
4988 // Get the instructions that need to be replaced. We emit at most two of
4989 // these. Remember them in order to avoid complications coming from the need
4990 // to traverse the block while potentially creating more blocks.
4991 SmallVector<MachineInstr *, 4> ToReplace;
4992 for (MachineInstr &MI : MBB)
4993 if (MI.getOpcode() == AArch64::PROBED_STACKALLOC ||
4994 MI.getOpcode() == AArch64::PROBED_STACKALLOC_VAR)
4995 ToReplace.push_back(Elt: &MI);
4996
4997 for (MachineInstr *MI : ToReplace) {
4998 if (MI->getOpcode() == AArch64::PROBED_STACKALLOC) {
4999 Register ScratchReg = MI->getOperand(i: 0).getReg();
5000 int64_t FrameSize = MI->getOperand(i: 1).getImm();
5001 StackOffset CFAOffset = StackOffset::get(Fixed: MI->getOperand(i: 2).getImm(),
5002 Scalable: MI->getOperand(i: 3).getImm());
5003 inlineStackProbeFixed(MBBI: MI->getIterator(), ScratchReg, FrameSize,
5004 CFAOffset);
5005 } else {
5006 assert(MI->getOpcode() == AArch64::PROBED_STACKALLOC_VAR &&
5007 "Stack probe pseudo-instruction expected");
5008 const AArch64InstrInfo *TII =
5009 MI->getMF()->getSubtarget<AArch64Subtarget>().getInstrInfo();
5010 Register TargetReg = MI->getOperand(i: 0).getReg();
5011 (void)TII->probedStackAlloc(MBBI: MI->getIterator(), TargetReg, FrameSetup: true);
5012 }
5013 MI->eraseFromParent();
5014 }
5015}
5016
5017struct StackAccess {
5018 enum AccessType {
5019 NotAccessed = 0, // Stack object not accessed by load/store instructions.
5020 GPR = 1 << 0, // A general purpose register.
5021 PPR = 1 << 1, // A predicate register.
5022 FPR = 1 << 2, // A floating point/Neon/SVE register.
5023 };
5024
5025 int Idx;
5026 StackOffset Offset;
5027 int64_t Size;
5028 unsigned AccessTypes;
5029
5030 StackAccess() : Idx(0), Offset(), Size(0), AccessTypes(NotAccessed) {}
5031
5032 bool operator<(const StackAccess &Rhs) const {
5033 return std::make_tuple(args: start(), args: Idx) <
5034 std::make_tuple(args: Rhs.start(), args: Rhs.Idx);
5035 }
5036
5037 bool isCPU() const {
5038 // Predicate register load and store instructions execute on the CPU.
5039 return AccessTypes & (AccessType::GPR | AccessType::PPR);
5040 }
5041 bool isSME() const { return AccessTypes & AccessType::FPR; }
5042 bool isMixed() const { return isCPU() && isSME(); }
5043
5044 int64_t start() const { return Offset.getFixed() + Offset.getScalable(); }
5045 int64_t end() const { return start() + Size; }
5046
5047 std::string getTypeString() const {
5048 switch (AccessTypes) {
5049 case AccessType::FPR:
5050 return "FPR";
5051 case AccessType::PPR:
5052 return "PPR";
5053 case AccessType::GPR:
5054 return "GPR";
5055 case AccessType::NotAccessed:
5056 return "NA";
5057 default:
5058 return "Mixed";
5059 }
5060 }
5061
5062 void print(raw_ostream &OS) const {
5063 OS << getTypeString() << " stack object at [SP"
5064 << (Offset.getFixed() < 0 ? "" : "+") << Offset.getFixed();
5065 if (Offset.getScalable())
5066 OS << (Offset.getScalable() < 0 ? "" : "+") << Offset.getScalable()
5067 << " * vscale";
5068 OS << "]";
5069 }
5070};
5071
5072static inline raw_ostream &operator<<(raw_ostream &OS, const StackAccess &SA) {
5073 SA.print(OS);
5074 return OS;
5075}
5076
5077void AArch64FrameLowering::emitRemarks(
5078 const MachineFunction &MF, MachineOptimizationRemarkEmitter *ORE) const {
5079
5080 SMEAttrs Attrs(MF.getFunction());
5081 if (Attrs.hasNonStreamingInterfaceAndBody())
5082 return;
5083
5084 const uint64_t HazardSize =
5085 (StackHazardSize) ? StackHazardSize : StackHazardRemarkSize;
5086
5087 if (HazardSize == 0)
5088 return;
5089
5090 const MachineFrameInfo &MFI = MF.getFrameInfo();
5091 // Bail if function has no stack objects.
5092 if (!MFI.hasStackObjects())
5093 return;
5094
5095 std::vector<StackAccess> StackAccesses(MFI.getNumObjects());
5096
5097 size_t NumFPLdSt = 0;
5098 size_t NumNonFPLdSt = 0;
5099
5100 // Collect stack accesses via Load/Store instructions.
5101 for (const MachineBasicBlock &MBB : MF) {
5102 for (const MachineInstr &MI : MBB) {
5103 if (!MI.mayLoadOrStore() || MI.getNumMemOperands() < 1)
5104 continue;
5105 for (MachineMemOperand *MMO : MI.memoperands()) {
5106 std::optional<int> FI = getMMOFrameID(MMO, MFI);
5107 if (FI && !MFI.isDeadObjectIndex(ObjectIdx: *FI)) {
5108 int FrameIdx = *FI;
5109
5110 size_t ArrIdx = FrameIdx + MFI.getNumFixedObjects();
5111 if (StackAccesses[ArrIdx].AccessTypes == StackAccess::NotAccessed) {
5112 StackAccesses[ArrIdx].Idx = FrameIdx;
5113 StackAccesses[ArrIdx].Offset =
5114 getFrameIndexReferenceFromSP(MF, FI: FrameIdx);
5115 StackAccesses[ArrIdx].Size = MFI.getObjectSize(ObjectIdx: FrameIdx);
5116 }
5117
5118 unsigned RegTy = StackAccess::AccessType::GPR;
5119 if (MFI.getStackID(ObjectIdx: FrameIdx) == TargetStackID::ScalableVector) {
5120 if (AArch64::PPRRegClass.contains(Reg: MI.getOperand(i: 0).getReg()))
5121 RegTy = StackAccess::PPR;
5122 else
5123 RegTy = StackAccess::FPR;
5124 } else if (AArch64InstrInfo::isFpOrNEON(MI)) {
5125 RegTy = StackAccess::FPR;
5126 }
5127
5128 StackAccesses[ArrIdx].AccessTypes |= RegTy;
5129
5130 if (RegTy == StackAccess::FPR)
5131 ++NumFPLdSt;
5132 else
5133 ++NumNonFPLdSt;
5134 }
5135 }
5136 }
5137 }
5138
5139 if (NumFPLdSt == 0 || NumNonFPLdSt == 0)
5140 return;
5141
5142 llvm::sort(C&: StackAccesses);
5143 StackAccesses.erase(first: llvm::remove_if(Range&: StackAccesses,
5144 P: [](const StackAccess &S) {
5145 return S.AccessTypes ==
5146 StackAccess::NotAccessed;
5147 }),
5148 last: StackAccesses.end());
5149
5150 SmallVector<const StackAccess *> MixedObjects;
5151 SmallVector<std::pair<const StackAccess *, const StackAccess *>> HazardPairs;
5152
5153 if (StackAccesses.front().isMixed())
5154 MixedObjects.push_back(Elt: &StackAccesses.front());
5155
5156 for (auto It = StackAccesses.begin(), End = std::prev(x: StackAccesses.end());
5157 It != End; ++It) {
5158 const auto &First = *It;
5159 const auto &Second = *(It + 1);
5160
5161 if (Second.isMixed())
5162 MixedObjects.push_back(Elt: &Second);
5163
5164 if ((First.isSME() && Second.isCPU()) ||
5165 (First.isCPU() && Second.isSME())) {
5166 uint64_t Distance = static_cast<uint64_t>(Second.start() - First.end());
5167 if (Distance < HazardSize)
5168 HazardPairs.emplace_back(Args: &First, Args: &Second);
5169 }
5170 }
5171
5172 auto EmitRemark = [&](llvm::StringRef Str) {
5173 ORE->emit(RemarkBuilder: [&]() {
5174 auto R = MachineOptimizationRemarkAnalysis(
5175 "sme", "StackHazard", MF.getFunction().getSubprogram(), &MF.front());
5176 return R << formatv(Fmt: "stack hazard in '{0}': ", Vals: MF.getName()).str() << Str;
5177 });
5178 };
5179
5180 for (const auto &P : HazardPairs)
5181 EmitRemark(formatv(Fmt: "{0} is too close to {1}", Vals: *P.first, Vals: *P.second).str());
5182
5183 for (const auto *Obj : MixedObjects)
5184 EmitRemark(
5185 formatv(Fmt: "{0} accessed by both GP and FP instructions", Vals: *Obj).str());
5186}
5187