1//===-- SystemZFrameLowering.cpp - Frame lowering for SystemZ -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "SystemZFrameLowering.h"
10#include "SystemZCallingConv.h"
11#include "SystemZInstrInfo.h"
12#include "SystemZMachineFunctionInfo.h"
13#include "SystemZRegisterInfo.h"
14#include "SystemZSubtarget.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/CodeGen/LivePhysRegs.h"
17#include "llvm/CodeGen/MachineInstrBuilder.h"
18#include "llvm/CodeGen/MachineModuleInfo.h"
19#include "llvm/CodeGen/MachineRegisterInfo.h"
20#include "llvm/CodeGen/RegisterScavenging.h"
21#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
22#include "llvm/IR/CallingConv.h"
23#include "llvm/IR/Function.h"
24#include "llvm/Target/TargetMachine.h"
25
26using namespace llvm;
27
28namespace {
29// The ABI-defined register save slots, relative to the CFA (i.e.
30// incoming stack pointer + SystemZMC::ELFCallFrameSize).
31static const TargetFrameLowering::SpillSlot ELFSpillOffsetTable[] = {
32 { .Reg: SystemZ::R2D, .Offset: 0x10 },
33 { .Reg: SystemZ::R3D, .Offset: 0x18 },
34 { .Reg: SystemZ::R4D, .Offset: 0x20 },
35 { .Reg: SystemZ::R5D, .Offset: 0x28 },
36 { .Reg: SystemZ::R6D, .Offset: 0x30 },
37 { .Reg: SystemZ::R7D, .Offset: 0x38 },
38 { .Reg: SystemZ::R8D, .Offset: 0x40 },
39 { .Reg: SystemZ::R9D, .Offset: 0x48 },
40 { .Reg: SystemZ::R10D, .Offset: 0x50 },
41 { .Reg: SystemZ::R11D, .Offset: 0x58 },
42 { .Reg: SystemZ::R12D, .Offset: 0x60 },
43 { .Reg: SystemZ::R13D, .Offset: 0x68 },
44 { .Reg: SystemZ::R14D, .Offset: 0x70 },
45 { .Reg: SystemZ::R15D, .Offset: 0x78 },
46 { .Reg: SystemZ::F0D, .Offset: 0x80 },
47 { .Reg: SystemZ::F2D, .Offset: 0x88 },
48 { .Reg: SystemZ::F4D, .Offset: 0x90 },
49 { .Reg: SystemZ::F6D, .Offset: 0x98 }
50};
51
52static const TargetFrameLowering::SpillSlot XPLINKSpillOffsetTable[] = {
53 {.Reg: SystemZ::R4D, .Offset: 0x00}, {.Reg: SystemZ::R5D, .Offset: 0x08}, {.Reg: SystemZ::R6D, .Offset: 0x10},
54 {.Reg: SystemZ::R7D, .Offset: 0x18}, {.Reg: SystemZ::R8D, .Offset: 0x20}, {.Reg: SystemZ::R9D, .Offset: 0x28},
55 {.Reg: SystemZ::R10D, .Offset: 0x30}, {.Reg: SystemZ::R11D, .Offset: 0x38}, {.Reg: SystemZ::R12D, .Offset: 0x40},
56 {.Reg: SystemZ::R13D, .Offset: 0x48}, {.Reg: SystemZ::R14D, .Offset: 0x50}, {.Reg: SystemZ::R15D, .Offset: 0x58}};
57} // end anonymous namespace
58
59SystemZFrameLowering::SystemZFrameLowering(StackDirection D, Align StackAl,
60 int LAO, Align TransAl,
61 bool StackReal, unsigned PointerSize)
62 : TargetFrameLowering(D, StackAl, LAO, TransAl, StackReal),
63 PointerSize(PointerSize) {}
64
65std::unique_ptr<SystemZFrameLowering>
66SystemZFrameLowering::create(const SystemZSubtarget &STI) {
67 unsigned PtrSz = 8;
68 if (STI.isTargetXPLINK64())
69 return std::make_unique<SystemZXPLINKFrameLowering>(args&: PtrSz);
70 return std::make_unique<SystemZELFFrameLowering>(args&: PtrSz);
71}
72
73namespace {
74struct SZFrameSortingObj {
75 bool IsValid = false; // True if we care about this Object.
76 uint32_t ObjectIndex = 0; // Index of Object into MFI list.
77 uint64_t ObjectSize = 0; // Size of Object in bytes.
78 uint32_t D12Count = 0; // 12-bit displacement only.
79 uint32_t DPairCount = 0; // 12 or 20 bit displacement.
80};
81typedef std::vector<SZFrameSortingObj> SZFrameObjVec;
82} // namespace
83
84// TODO: Move to base class.
85void SystemZELFFrameLowering::orderFrameObjects(
86 const MachineFunction &MF, SmallVectorImpl<int> &ObjectsToAllocate) const {
87 const MachineFrameInfo &MFI = MF.getFrameInfo();
88 auto *TII = MF.getSubtarget<SystemZSubtarget>().getInstrInfo();
89
90 // Make a vector of sorting objects to track all MFI objects and mark those
91 // to be sorted as valid.
92 if (ObjectsToAllocate.size() <= 1)
93 return;
94 SZFrameObjVec SortingObjects(MFI.getObjectIndexEnd());
95 for (auto &Obj : ObjectsToAllocate) {
96 SortingObjects[Obj].IsValid = true;
97 SortingObjects[Obj].ObjectIndex = Obj;
98 SortingObjects[Obj].ObjectSize = MFI.getObjectSize(ObjectIdx: Obj);
99 }
100
101 // Examine uses for each object and record short (12-bit) and "pair"
102 // displacement types.
103 for (auto &MBB : MF)
104 for (auto &MI : MBB) {
105 if (MI.isDebugInstr())
106 continue;
107 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
108 const MachineOperand &MO = MI.getOperand(i: I);
109 if (!MO.isFI())
110 continue;
111 int Index = MO.getIndex();
112 if (Index >= 0 && Index < MFI.getObjectIndexEnd() &&
113 SortingObjects[Index].IsValid) {
114 if (TII->hasDisplacementPairInsn(Opcode: MI.getOpcode()))
115 SortingObjects[Index].DPairCount++;
116 else if (!(MI.getDesc().TSFlags & SystemZII::Has20BitOffset))
117 SortingObjects[Index].D12Count++;
118 }
119 }
120 }
121
122 // Sort all objects for short/paired displacements, which should be
123 // sufficient as it seems like all frame objects typically are within the
124 // long displacement range. Sorting works by computing the "density" as
125 // Count / ObjectSize. The comparisons of two such fractions are refactored
126 // by multiplying both sides with A.ObjectSize * B.ObjectSize, in order to
127 // eliminate the (fp) divisions. A higher density object needs to go after
128 // in the list in order for it to end up lower on the stack.
129 auto CmpD12 = [](const SZFrameSortingObj &A, const SZFrameSortingObj &B) {
130 // Put all invalid and variable sized objects at the end.
131 if (!A.IsValid || !B.IsValid)
132 return A.IsValid;
133 if (!A.ObjectSize || !B.ObjectSize)
134 return A.ObjectSize > 0;
135 uint64_t ADensityCmp = A.D12Count * B.ObjectSize;
136 uint64_t BDensityCmp = B.D12Count * A.ObjectSize;
137 if (ADensityCmp != BDensityCmp)
138 return ADensityCmp < BDensityCmp;
139 return A.DPairCount * B.ObjectSize < B.DPairCount * A.ObjectSize;
140 };
141 llvm::stable_sort(Range&: SortingObjects, C: CmpD12);
142
143 // Now modify the original list to represent the final order that
144 // we want.
145 unsigned Idx = 0;
146 for (auto &Obj : SortingObjects) {
147 // All invalid items are sorted at the end, so it's safe to stop.
148 if (!Obj.IsValid)
149 break;
150 ObjectsToAllocate[Idx++] = Obj.ObjectIndex;
151 }
152}
153
154bool SystemZFrameLowering::hasReservedCallFrame(
155 const MachineFunction &MF) const {
156 // The ELF ABI requires us to allocate 160 bytes of stack space for the
157 // callee, with any outgoing stack arguments being placed above that. It
158 // seems better to make that area a permanent feature of the frame even if
159 // we're using a frame pointer. Similarly, 64-bit XPLINK requires 96 bytes
160 // of stack space for the register save area.
161 return true;
162}
163
164void SystemZFrameLowering::emitIncrement(MachineBasicBlock &MBB,
165 MachineBasicBlock::iterator &MBBI,
166 const DebugLoc &DL, Register Reg,
167 int64_t NumBytes,
168 const TargetInstrInfo *TII) const {
169 while (NumBytes) {
170 unsigned Opcode;
171 int64_t ThisVal = NumBytes;
172 if (isInt<16>(x: NumBytes))
173 Opcode = SystemZ::AGHI;
174 else {
175 Opcode = SystemZ::AGFI;
176 // Make sure we maintain stack alignment.
177 int64_t MinVal = -uint64_t(1) << 31;
178 int64_t MaxVal = (int64_t(1) << 31) - getStackAlignment();
179 if (ThisVal < MinVal)
180 ThisVal = MinVal;
181 else if (ThisVal > MaxVal)
182 ThisVal = MaxVal;
183 }
184 MachineInstr *MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode), DestReg: Reg)
185 .addReg(RegNo: Reg)
186 .addImm(Val: ThisVal);
187 // The CC implicit def is dead.
188 MI->getOperand(i: 3).setIsDead();
189 NumBytes -= ThisVal;
190 }
191}
192
193bool SystemZELFFrameLowering::assignCalleeSavedSpillSlots(
194 MachineFunction &MF, const TargetRegisterInfo *TRI,
195 std::vector<CalleeSavedInfo> &CSI) const {
196 SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
197 MachineFrameInfo &MFFrame = MF.getFrameInfo();
198 bool IsVarArg = MF.getFunction().isVarArg();
199 if (CSI.empty())
200 return true; // Early exit if no callee saved registers are modified!
201
202 unsigned LowGPR = 0;
203 unsigned HighGPR = SystemZ::R15D;
204 int StartSPOffset = SystemZMC::ELFCallFrameSize;
205 for (auto &CS : CSI) {
206 MCRegister Reg = CS.getReg();
207 int Offset = getRegSpillOffset(MF, Reg);
208 if (Offset) {
209 if (SystemZ::GR64BitRegClass.contains(Reg) && StartSPOffset > Offset) {
210 LowGPR = Reg;
211 StartSPOffset = Offset;
212 }
213 Offset -= SystemZMC::ELFCallFrameSize;
214 int FrameIdx =
215 MFFrame.CreateFixedSpillStackObject(Size: getPointerSize(), SPOffset: Offset);
216 CS.setFrameIdx(FrameIdx);
217 } else
218 CS.setFrameIdx(INT32_MAX);
219 }
220
221 // Save the range of call-saved registers, for use by the
222 // prologue/epilogue inserters.
223 ZFI->setRestoreGPRRegs(Low: LowGPR, High: HighGPR, Offs: StartSPOffset);
224 if (IsVarArg) {
225 // Also save the GPR varargs, if any. R6D is call-saved, so would
226 // already be included, but we also need to handle the call-clobbered
227 // argument registers.
228 Register FirstGPR = ZFI->getVarArgsFirstGPR();
229 if (FirstGPR < SystemZ::ELFNumArgGPRs) {
230 unsigned Reg = SystemZ::ELFArgGPRs[FirstGPR];
231 int Offset = getRegSpillOffset(MF, Reg);
232 if (StartSPOffset > Offset) {
233 LowGPR = Reg; StartSPOffset = Offset;
234 }
235 }
236 }
237 ZFI->setSpillGPRRegs(Low: LowGPR, High: HighGPR, Offs: StartSPOffset);
238
239 // Create fixed stack objects for the remaining registers.
240 int CurrOffset = -SystemZMC::ELFCallFrameSize;
241 if (usePackedStack(MF))
242 CurrOffset += StartSPOffset;
243
244 for (auto &CS : CSI) {
245 if (CS.getFrameIdx() != INT32_MAX)
246 continue;
247 MCRegister Reg = CS.getReg();
248 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
249 unsigned Size = TRI->getSpillSize(RC: *RC);
250 CurrOffset -= Size;
251 assert(CurrOffset % 8 == 0 &&
252 "8-byte alignment required for for all register save slots");
253 int FrameIdx = MFFrame.CreateFixedSpillStackObject(Size, SPOffset: CurrOffset);
254 CS.setFrameIdx(FrameIdx);
255 }
256
257 return true;
258}
259
260void SystemZELFFrameLowering::determineCalleeSaves(MachineFunction &MF,
261 BitVector &SavedRegs,
262 RegScavenger *RS) const {
263 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
264
265 MachineFrameInfo &MFFrame = MF.getFrameInfo();
266 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
267 bool HasFP = hasFP(MF);
268 SystemZMachineFunctionInfo *MFI = MF.getInfo<SystemZMachineFunctionInfo>();
269 bool IsVarArg = MF.getFunction().isVarArg();
270
271 // va_start stores incoming FPR varargs in the normal way, but delegates
272 // the saving of incoming GPR varargs to spillCalleeSavedRegisters().
273 // Record these pending uses, which typically include the call-saved
274 // argument register R6D.
275 if (IsVarArg)
276 for (unsigned I = MFI->getVarArgsFirstGPR(); I < SystemZ::ELFNumArgGPRs; ++I)
277 SavedRegs.set(SystemZ::ELFArgGPRs[I]);
278
279 // If there are any landing pads, entering them will modify r6/r7.
280 if (!MF.getLandingPads().empty()) {
281 SavedRegs.set(SystemZ::R6D);
282 SavedRegs.set(SystemZ::R7D);
283 }
284
285 // If the function requires a frame pointer, record that the hard
286 // frame pointer will be clobbered.
287 if (HasFP)
288 SavedRegs.set(SystemZ::R11D);
289
290 // If the function calls other functions, record that the return
291 // address register will be clobbered.
292 if (MFFrame.hasCalls())
293 SavedRegs.set(SystemZ::R14D);
294
295 // If we are saving GPRs other than the stack pointer, we might as well
296 // save and restore the stack pointer at the same time, via STMG and LMG.
297 // This allows the deallocation to be done by the LMG, rather than needing
298 // a separate %r15 addition.
299 const MCPhysReg *CSRegs = TRI->getCalleeSavedRegs(MF: &MF);
300 for (unsigned I = 0; CSRegs[I]; ++I) {
301 unsigned Reg = CSRegs[I];
302 if (SystemZ::GR64BitRegClass.contains(Reg) && SavedRegs.test(Idx: Reg)) {
303 SavedRegs.set(SystemZ::R15D);
304 break;
305 }
306 }
307}
308
309SystemZELFFrameLowering::SystemZELFFrameLowering(unsigned PointerSize)
310 : SystemZFrameLowering(TargetFrameLowering::StackGrowsDown, Align(8), 0,
311 Align(8), /* StackRealignable */ false, PointerSize),
312 RegSpillOffsets(0) {
313
314 // Due to the SystemZ ABI, the DWARF CFA (Canonical Frame Address) is not
315 // equal to the incoming stack pointer, but to incoming stack pointer plus
316 // 160. Instead of using a Local Area Offset, the Register save area will
317 // be occupied by fixed frame objects, and all offsets are actually
318 // relative to CFA.
319
320 // Create a mapping from register number to save slot offset.
321 // These offsets are relative to the start of the register save area.
322 RegSpillOffsets.grow(N: SystemZ::NUM_TARGET_REGS);
323 for (const auto &Entry : ELFSpillOffsetTable)
324 RegSpillOffsets[Entry.Reg] = Entry.Offset;
325}
326
327// Add GPR64 to the save instruction being built by MIB, which is in basic
328// block MBB. IsImplicit says whether this is an explicit operand to the
329// instruction, or an implicit one that comes between the explicit start
330// and end registers.
331static void addSavedGPR(MachineBasicBlock &MBB, MachineInstrBuilder &MIB,
332 unsigned GPR64, bool IsImplicit) {
333 const TargetRegisterInfo *RI =
334 MBB.getParent()->getSubtarget().getRegisterInfo();
335 Register GPR32 = RI->getSubReg(Reg: GPR64, Idx: SystemZ::subreg_l32);
336 bool IsLive = MBB.isLiveIn(Reg: GPR64) || MBB.isLiveIn(Reg: GPR32);
337 if (!IsLive || !IsImplicit) {
338 MIB.addReg(RegNo: GPR64, Flags: getImplRegState(B: IsImplicit) | getKillRegState(B: !IsLive));
339 if (!IsLive)
340 MBB.addLiveIn(PhysReg: GPR64);
341 }
342}
343
344bool SystemZELFFrameLowering::spillCalleeSavedRegisters(
345 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
346 ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
347 if (CSI.empty())
348 return false;
349
350 MachineFunction &MF = *MBB.getParent();
351 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
352 SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
353 bool IsVarArg = MF.getFunction().isVarArg();
354 DebugLoc DL;
355
356 // Save GPRs
357 SystemZ::GPRRegs SpillGPRs = ZFI->getSpillGPRRegs();
358 if (SpillGPRs.LowGPR) {
359 assert(SpillGPRs.LowGPR != SpillGPRs.HighGPR &&
360 "Should be saving %r15 and something else");
361
362 // Build an STMG instruction.
363 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STMG));
364
365 // Add the explicit register operands.
366 addSavedGPR(MBB, MIB, GPR64: SpillGPRs.LowGPR, IsImplicit: false);
367 addSavedGPR(MBB, MIB, GPR64: SpillGPRs.HighGPR, IsImplicit: false);
368
369 // Add the address.
370 MIB.addReg(RegNo: SystemZ::R15D).addImm(Val: SpillGPRs.GPROffset);
371
372 // Make sure all call-saved GPRs are included as operands and are
373 // marked as live on entry.
374 for (const CalleeSavedInfo &I : CSI) {
375 MCRegister Reg = I.getReg();
376 if (SystemZ::GR64BitRegClass.contains(Reg))
377 addSavedGPR(MBB, MIB, GPR64: Reg, IsImplicit: true);
378 }
379
380 // ...likewise GPR varargs.
381 if (IsVarArg)
382 for (unsigned I = ZFI->getVarArgsFirstGPR(); I < SystemZ::ELFNumArgGPRs; ++I)
383 addSavedGPR(MBB, MIB, GPR64: SystemZ::ELFArgGPRs[I], IsImplicit: true);
384 }
385
386 // Save FPRs/VRs in the normal TargetInstrInfo way.
387 for (const CalleeSavedInfo &I : CSI) {
388 MCRegister Reg = I.getReg();
389 if (SystemZ::FP64BitRegClass.contains(Reg)) {
390 MBB.addLiveIn(PhysReg: Reg);
391 TII->storeRegToStackSlot(MBB, MI: MBBI, SrcReg: Reg, isKill: true, FrameIndex: I.getFrameIdx(),
392 RC: &SystemZ::FP64BitRegClass, VReg: Register());
393 }
394 if (SystemZ::VR128BitRegClass.contains(Reg)) {
395 MBB.addLiveIn(PhysReg: Reg);
396 TII->storeRegToStackSlot(MBB, MI: MBBI, SrcReg: Reg, isKill: true, FrameIndex: I.getFrameIdx(),
397 RC: &SystemZ::VR128BitRegClass, VReg: Register());
398 }
399 }
400
401 return true;
402}
403
404bool SystemZELFFrameLowering::restoreCalleeSavedRegisters(
405 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
406 MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
407 if (CSI.empty())
408 return false;
409
410 MachineFunction &MF = *MBB.getParent();
411 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
412 SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
413 bool HasFP = hasFP(MF);
414 DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
415
416 // Restore FPRs/VRs in the normal TargetInstrInfo way.
417 for (const CalleeSavedInfo &I : CSI) {
418 MCRegister Reg = I.getReg();
419 if (SystemZ::FP64BitRegClass.contains(Reg))
420 TII->loadRegFromStackSlot(MBB, MI: MBBI, DestReg: Reg, FrameIndex: I.getFrameIdx(),
421 RC: &SystemZ::FP64BitRegClass, VReg: Register());
422 if (SystemZ::VR128BitRegClass.contains(Reg))
423 TII->loadRegFromStackSlot(MBB, MI: MBBI, DestReg: Reg, FrameIndex: I.getFrameIdx(),
424 RC: &SystemZ::VR128BitRegClass, VReg: Register());
425 }
426
427 // Restore call-saved GPRs (but not call-clobbered varargs, which at
428 // this point might hold return values).
429 SystemZ::GPRRegs RestoreGPRs = ZFI->getRestoreGPRRegs();
430 if (RestoreGPRs.LowGPR) {
431 // If we saved any of %r2-%r5 as varargs, we should also be saving
432 // and restoring %r6. If we're saving %r6 or above, we should be
433 // restoring it too.
434 assert(RestoreGPRs.LowGPR != RestoreGPRs.HighGPR &&
435 "Should be loading %r15 and something else");
436
437 // Build an LMG instruction.
438 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LMG));
439
440 // Add the explicit register operands.
441 MIB.addReg(RegNo: RestoreGPRs.LowGPR, Flags: RegState::Define);
442 MIB.addReg(RegNo: RestoreGPRs.HighGPR, Flags: RegState::Define);
443
444 // Add the address.
445 MIB.addReg(RegNo: HasFP ? SystemZ::R11D : SystemZ::R15D);
446 MIB.addImm(Val: RestoreGPRs.GPROffset);
447
448 // Do a second scan adding regs as being defined by instruction
449 for (const CalleeSavedInfo &I : CSI) {
450 MCRegister Reg = I.getReg();
451 if (Reg != RestoreGPRs.LowGPR && Reg != RestoreGPRs.HighGPR &&
452 SystemZ::GR64BitRegClass.contains(Reg))
453 MIB.addReg(RegNo: Reg, Flags: RegState::ImplicitDefine);
454 }
455 }
456
457 return true;
458}
459
460void SystemZELFFrameLowering::processFunctionBeforeFrameFinalized(
461 MachineFunction &MF, RegScavenger *RS) const {
462 MachineFrameInfo &MFFrame = MF.getFrameInfo();
463 SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
464 MachineRegisterInfo *MRI = &MF.getRegInfo();
465 bool BackChain = MF.getSubtarget<SystemZSubtarget>().hasBackChain();
466
467 if (!usePackedStack(MF) || BackChain)
468 // Create the incoming register save area.
469 getOrCreateFramePointerSaveIndex(MF);
470
471 // Get the size of our stack frame to be allocated ...
472 uint64_t StackSize = (MFFrame.estimateStackSize(MF) +
473 SystemZMC::ELFCallFrameSize);
474 // ... and the maximum offset we may need to reach into the
475 // caller's frame to access the save area or stack arguments.
476 int64_t MaxArgOffset = 0;
477 for (int I = MFFrame.getObjectIndexBegin(); I != 0; ++I)
478 if (MFFrame.getObjectOffset(ObjectIdx: I) >= 0) {
479 int64_t ArgOffset = MFFrame.getObjectOffset(ObjectIdx: I) +
480 MFFrame.getObjectSize(ObjectIdx: I);
481 MaxArgOffset = std::max(a: MaxArgOffset, b: ArgOffset);
482 }
483
484 uint64_t MaxReach = StackSize + MaxArgOffset;
485 if (!isUInt<12>(x: MaxReach)) {
486 // We may need register scavenging slots if some parts of the frame
487 // are outside the reach of an unsigned 12-bit displacement.
488 // Create 2 for the case where both addresses in an MVC are
489 // out of range.
490 RS->addScavengingFrameIndex(
491 FI: MFFrame.CreateSpillStackObject(Size: getPointerSize(), Alignment: Align(8)));
492 RS->addScavengingFrameIndex(
493 FI: MFFrame.CreateSpillStackObject(Size: getPointerSize(), Alignment: Align(8)));
494 }
495
496 // If R6 is used as an argument register it is still callee saved. If it in
497 // this case is not clobbered (and restored) it should never be marked as
498 // killed.
499 if (MF.front().isLiveIn(Reg: SystemZ::R6D) &&
500 ZFI->getRestoreGPRRegs().LowGPR != SystemZ::R6D)
501 for (auto &MO : MRI->use_nodbg_operands(Reg: SystemZ::R6D))
502 MO.setIsKill(false);
503}
504
505// Add CFI for the new CFA offset.
506static void buildCFAOffs(MachineBasicBlock &MBB,
507 MachineBasicBlock::iterator MBBI,
508 const DebugLoc &DL, int Offset,
509 const SystemZInstrInfo *ZII) {
510 unsigned CFIIndex = MBB.getParent()->addFrameInst(
511 Inst: MCCFIInstruction::cfiDefCfaOffset(L: nullptr, Offset: -Offset));
512 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: ZII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
513 .addCFIIndex(CFIIndex);
514}
515
516// Add CFI for the new frame location.
517static void buildDefCFAReg(MachineBasicBlock &MBB,
518 MachineBasicBlock::iterator MBBI,
519 const DebugLoc &DL, unsigned Reg,
520 const SystemZInstrInfo *ZII) {
521 MachineFunction &MF = *MBB.getParent();
522 const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
523 unsigned RegNum = MRI->getDwarfRegNum(Reg, isEH: true);
524 unsigned CFIIndex = MF.addFrameInst(
525 Inst: MCCFIInstruction::createDefCfaRegister(L: nullptr, Register: RegNum));
526 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: ZII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
527 .addCFIIndex(CFIIndex);
528}
529
530void SystemZELFFrameLowering::emitPrologue(MachineFunction &MF,
531 MachineBasicBlock &MBB) const {
532 assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported");
533 const SystemZSubtarget &STI = MF.getSubtarget<SystemZSubtarget>();
534 const SystemZTargetLowering &TLI = *STI.getTargetLowering();
535 MachineFrameInfo &MFFrame = MF.getFrameInfo();
536 auto *ZII = STI.getInstrInfo();
537 SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
538 MachineBasicBlock::iterator MBBI = MBB.begin();
539 const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
540 const std::vector<CalleeSavedInfo> &CSI = MFFrame.getCalleeSavedInfo();
541 bool HasFP = hasFP(MF);
542
543 // In GHC calling convention C stack space, including the ABI-defined
544 // 160-byte base area, is (de)allocated by GHC itself. This stack space may
545 // be used by LLVM as spill slots for the tail recursive GHC functions. Thus
546 // do not allocate stack space here, too.
547 if (MF.getFunction().getCallingConv() == CallingConv::GHC) {
548 if (MFFrame.getStackSize() > 2048 * sizeof(long)) {
549 report_fatal_error(
550 reason: "Pre allocated stack space for GHC function is too small");
551 }
552 if (HasFP) {
553 report_fatal_error(
554 reason: "In GHC calling convention a frame pointer is not supported");
555 }
556 MFFrame.setStackSize(MFFrame.getStackSize() + SystemZMC::ELFCallFrameSize);
557 return;
558 }
559
560 // Debug location must be unknown since the first debug location is used
561 // to determine the end of the prologue.
562 DebugLoc DL;
563 // Add mcount instrumentation if necessary.
564 if (MF.getFunction()
565 .getFnAttribute(Kind: "systemz-instrument-function-entry")
566 .getValueAsString() == "mcount") {
567
568 // Store return address 8 bytes above stack pointer.
569 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::STG))
570 .addReg(RegNo: SystemZ::R14D)
571 .addReg(RegNo: SystemZ::R15D)
572 .addImm(Val: 8)
573 .addReg(RegNo: 0);
574
575 // Call mcount (Regmask from CC AnyReg since mcount preserves all normal
576 // argument registers).
577 const uint32_t *Mask = MF.getSubtarget<SystemZSubtarget>()
578 .getSpecialRegisters()
579 ->getCallPreservedMask(MF, CC: CallingConv::AnyReg);
580 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::CallBRASL))
581 .addExternalSymbol(FnName: "mcount")
582 .addRegMask(Mask);
583
584 // Reload return address from 8 bytes above stack pointer.
585 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::LG))
586 .addReg(RegNo: SystemZ::R14D, Flags: RegState::Define)
587 .addReg(RegNo: SystemZ::R15D)
588 .addImm(Val: 8)
589 .addReg(RegNo: 0);
590 }
591
592 // The current offset of the stack pointer from the CFA.
593 int64_t SPOffsetFromCFA = -SystemZMC::ELFCFAOffsetFromInitialSP;
594
595 if (ZFI->getSpillGPRRegs().LowGPR) {
596 // Skip over the GPR saves.
597 if (MBBI != MBB.end() && MBBI->getOpcode() == SystemZ::STMG)
598 ++MBBI;
599 else
600 llvm_unreachable("Couldn't skip over GPR saves");
601
602 // Add CFI for the GPR saves.
603 for (auto &Save : CSI) {
604 MCRegister Reg = Save.getReg();
605 if (SystemZ::GR64BitRegClass.contains(Reg)) {
606 int FI = Save.getFrameIdx();
607 int64_t Offset = MFFrame.getObjectOffset(ObjectIdx: FI);
608 unsigned CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::createOffset(
609 L: nullptr, Register: MRI->getDwarfRegNum(Reg, isEH: true), Offset));
610 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: ZII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
611 .addCFIIndex(CFIIndex);
612 }
613 }
614 }
615
616 uint64_t StackSize = MFFrame.getStackSize();
617 // We need to allocate the ABI-defined 160-byte base area whenever
618 // we allocate stack space for our own use and whenever we call another
619 // function.
620 bool HasStackObject = false;
621 for (unsigned i = 0, e = MFFrame.getObjectIndexEnd(); i != e; ++i)
622 if (!MFFrame.isDeadObjectIndex(ObjectIdx: i)) {
623 HasStackObject = true;
624 break;
625 }
626 if (HasStackObject || MFFrame.hasCalls())
627 StackSize += SystemZMC::ELFCallFrameSize;
628 // Don't allocate the incoming reg save area.
629 StackSize = StackSize > SystemZMC::ELFCallFrameSize
630 ? StackSize - SystemZMC::ELFCallFrameSize
631 : 0;
632 MFFrame.setStackSize(StackSize);
633
634 if (StackSize) {
635 // Allocate StackSize bytes.
636 int64_t Delta = -int64_t(StackSize);
637 const unsigned ProbeSize = TLI.getStackProbeSize(MF);
638 bool FreeProbe = (ZFI->getSpillGPRRegs().GPROffset &&
639 (ZFI->getSpillGPRRegs().GPROffset + StackSize) < ProbeSize);
640 if (!FreeProbe &&
641 MF.getSubtarget().getTargetLowering()->hasInlineStackProbe(MF)) {
642 // Stack probing may involve looping, but splitting the prologue block
643 // is not possible at this point since it would invalidate the
644 // SaveBlocks / RestoreBlocks sets of PEI in the single block function
645 // case. Build a pseudo to be handled later by inlineStackProbe().
646 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::PROBED_STACKALLOC))
647 .addImm(Val: StackSize);
648 }
649 else {
650 bool StoreBackchain = MF.getSubtarget<SystemZSubtarget>().hasBackChain();
651 // If we need backchain, save current stack pointer. R1 is free at
652 // this point.
653 if (StoreBackchain)
654 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::LGR))
655 .addReg(RegNo: SystemZ::R1D, Flags: RegState::Define).addReg(RegNo: SystemZ::R15D);
656 emitIncrement(MBB, MBBI, DL, Reg: SystemZ::R15D, NumBytes: Delta, TII: ZII);
657 buildCFAOffs(MBB, MBBI, DL, Offset: SPOffsetFromCFA + Delta, ZII);
658 if (StoreBackchain)
659 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::STG))
660 .addReg(RegNo: SystemZ::R1D, Flags: RegState::Kill).addReg(RegNo: SystemZ::R15D)
661 .addImm(Val: getBackchainOffset(MF)).addReg(RegNo: 0);
662 }
663 SPOffsetFromCFA += Delta;
664 }
665
666 if (HasFP) {
667 // Copy the base of the frame to R11.
668 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::LGR), DestReg: SystemZ::R11D)
669 .addReg(RegNo: SystemZ::R15D);
670
671 // Add CFI for the new frame location.
672 buildDefCFAReg(MBB, MBBI, DL, Reg: SystemZ::R11D, ZII);
673
674 // Mark the FramePtr as live at the beginning of every block except
675 // the entry block. (We'll have marked R11 as live on entry when
676 // saving the GPRs.)
677 for (MachineBasicBlock &MBBJ : llvm::drop_begin(RangeOrContainer&: MF))
678 MBBJ.addLiveIn(PhysReg: SystemZ::R11D);
679 }
680
681 // Skip over the FPR/VR saves.
682 SmallVector<unsigned, 8> CFIIndexes;
683 for (auto &Save : CSI) {
684 MCRegister Reg = Save.getReg();
685 if (SystemZ::FP64BitRegClass.contains(Reg)) {
686 if (MBBI != MBB.end() &&
687 (MBBI->getOpcode() == SystemZ::STD ||
688 MBBI->getOpcode() == SystemZ::STDY))
689 ++MBBI;
690 else
691 llvm_unreachable("Couldn't skip over FPR save");
692 } else if (SystemZ::VR128BitRegClass.contains(Reg)) {
693 if (MBBI != MBB.end() &&
694 MBBI->getOpcode() == SystemZ::VST)
695 ++MBBI;
696 else
697 llvm_unreachable("Couldn't skip over VR save");
698 } else
699 continue;
700
701 // Add CFI for the this save.
702 unsigned DwarfReg = MRI->getDwarfRegNum(Reg, isEH: true);
703 Register IgnoredFrameReg;
704 int64_t Offset =
705 getFrameIndexReference(MF, FI: Save.getFrameIdx(), FrameReg&: IgnoredFrameReg)
706 .getFixed();
707
708 unsigned CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::createOffset(
709 L: nullptr, Register: DwarfReg, Offset: SPOffsetFromCFA + Offset));
710 CFIIndexes.push_back(Elt: CFIIndex);
711 }
712 // Complete the CFI for the FPR/VR saves, modelling them as taking effect
713 // after the last save.
714 for (auto CFIIndex : CFIIndexes) {
715 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: ZII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
716 .addCFIIndex(CFIIndex);
717 }
718}
719
720void SystemZELFFrameLowering::emitEpilogue(MachineFunction &MF,
721 MachineBasicBlock &MBB) const {
722 MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
723 auto *ZII =
724 static_cast<const SystemZInstrInfo *>(MF.getSubtarget().getInstrInfo());
725 SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
726 MachineFrameInfo &MFFrame = MF.getFrameInfo();
727
728 // See SystemZELFFrameLowering::emitPrologue
729 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
730 return;
731
732 // Skip the return instruction.
733 assert(MBBI->isReturn() && "Can only insert epilogue into returning blocks");
734
735 uint64_t StackSize = MFFrame.getStackSize();
736 if (ZFI->getRestoreGPRRegs().LowGPR) {
737 --MBBI;
738 unsigned Opcode = MBBI->getOpcode();
739 if (Opcode != SystemZ::LMG)
740 llvm_unreachable("Expected to see callee-save register restore code");
741
742 unsigned AddrOpNo = 2;
743 DebugLoc DL = MBBI->getDebugLoc();
744 uint64_t Offset = StackSize + MBBI->getOperand(i: AddrOpNo + 1).getImm();
745 unsigned NewOpcode = ZII->getOpcodeForOffset(Opcode, Offset);
746
747 // If the offset is too large, use the largest stack-aligned offset
748 // and add the rest to the base register (the stack or frame pointer).
749 if (!NewOpcode) {
750 uint64_t NumBytes = Offset - 0x7fff8;
751 emitIncrement(MBB, MBBI, DL, Reg: MBBI->getOperand(i: AddrOpNo).getReg(),
752 NumBytes, TII: ZII);
753 Offset -= NumBytes;
754 NewOpcode = ZII->getOpcodeForOffset(Opcode, Offset);
755 assert(NewOpcode && "No restore instruction available");
756 }
757
758 MBBI->setDesc(ZII->get(Opcode: NewOpcode));
759 MBBI->getOperand(i: AddrOpNo + 1).ChangeToImmediate(ImmVal: Offset);
760 } else if (StackSize) {
761 DebugLoc DL = MBBI->getDebugLoc();
762 emitIncrement(MBB, MBBI, DL, Reg: SystemZ::R15D, NumBytes: StackSize, TII: ZII);
763 }
764}
765
766void SystemZELFFrameLowering::inlineStackProbe(
767 MachineFunction &MF, MachineBasicBlock &PrologMBB) const {
768 auto *ZII =
769 static_cast<const SystemZInstrInfo *>(MF.getSubtarget().getInstrInfo());
770 const SystemZSubtarget &STI = MF.getSubtarget<SystemZSubtarget>();
771 const SystemZTargetLowering &TLI = *STI.getTargetLowering();
772
773 MachineInstr *StackAllocMI = nullptr;
774 for (MachineInstr &MI : PrologMBB)
775 if (MI.getOpcode() == SystemZ::PROBED_STACKALLOC) {
776 StackAllocMI = &MI;
777 break;
778 }
779 if (StackAllocMI == nullptr)
780 return;
781 uint64_t StackSize = StackAllocMI->getOperand(i: 0).getImm();
782 const unsigned ProbeSize = TLI.getStackProbeSize(MF);
783 uint64_t NumFullBlocks = StackSize / ProbeSize;
784 uint64_t Residual = StackSize % ProbeSize;
785 int64_t SPOffsetFromCFA = -SystemZMC::ELFCFAOffsetFromInitialSP;
786 MachineBasicBlock *MBB = &PrologMBB;
787 MachineBasicBlock::iterator MBBI = StackAllocMI;
788 const DebugLoc DL = StackAllocMI->getDebugLoc();
789
790 // Allocate a block of Size bytes on the stack and probe it.
791 auto allocateAndProbe = [&](MachineBasicBlock &InsMBB,
792 MachineBasicBlock::iterator InsPt, unsigned Size,
793 bool EmitCFI) -> void {
794 emitIncrement(MBB&: InsMBB, MBBI&: InsPt, DL, Reg: SystemZ::R15D, NumBytes: -int64_t(Size), TII: ZII);
795 if (EmitCFI) {
796 SPOffsetFromCFA -= Size;
797 buildCFAOffs(MBB&: InsMBB, MBBI: InsPt, DL, Offset: SPOffsetFromCFA, ZII);
798 }
799 // Probe by means of a volatile compare.
800 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(),
801 F: MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad, Size: 8, BaseAlignment: Align(1));
802 BuildMI(BB&: InsMBB, I: InsPt, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::CG))
803 .addReg(RegNo: SystemZ::R0D, Flags: RegState::Undef)
804 .addReg(RegNo: SystemZ::R15D)
805 .addImm(Val: Size - 8)
806 .addReg(RegNo: 0)
807 .setOperandDead(4)
808 .addMemOperand(MMO);
809 };
810
811 bool StoreBackchain = MF.getSubtarget<SystemZSubtarget>().hasBackChain();
812 if (StoreBackchain)
813 BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::LGR))
814 .addReg(RegNo: SystemZ::R1D, Flags: RegState::Define).addReg(RegNo: SystemZ::R15D);
815
816 MachineBasicBlock *DoneMBB = nullptr;
817 MachineBasicBlock *LoopMBB = nullptr;
818 if (NumFullBlocks < 3) {
819 // Emit unrolled probe statements.
820 for (unsigned int i = 0; i < NumFullBlocks; i++)
821 allocateAndProbe(*MBB, MBBI, ProbeSize, true/*EmitCFI*/);
822 } else {
823 // Emit a loop probing the pages.
824 uint64_t LoopAlloc = ProbeSize * NumFullBlocks;
825 SPOffsetFromCFA -= LoopAlloc;
826
827 // Use R0D to hold the exit value.
828 BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::LGR), DestReg: SystemZ::R0D)
829 .addReg(RegNo: SystemZ::R15D);
830 buildDefCFAReg(MBB&: *MBB, MBBI, DL, Reg: SystemZ::R0D, ZII);
831 emitIncrement(MBB&: *MBB, MBBI, DL, Reg: SystemZ::R0D, NumBytes: -int64_t(LoopAlloc), TII: ZII);
832 buildCFAOffs(MBB&: *MBB, MBBI, DL, Offset: -int64_t(SystemZMC::ELFCallFrameSize + LoopAlloc),
833 ZII);
834
835 DoneMBB = SystemZ::splitBlockBefore(MI: MBBI, MBB);
836 LoopMBB = SystemZ::emitBlockAfter(MBB);
837 MBB->addSuccessor(Succ: LoopMBB);
838 LoopMBB->addSuccessor(Succ: LoopMBB);
839 LoopMBB->addSuccessor(Succ: DoneMBB);
840
841 MBB = LoopMBB;
842 allocateAndProbe(*MBB, MBB->end(), ProbeSize, false/*EmitCFI*/);
843 BuildMI(BB&: *MBB, I: MBB->end(), MIMD: DL, MCID: ZII->get(Opcode: SystemZ::CLGR))
844 .addReg(RegNo: SystemZ::R15D).addReg(RegNo: SystemZ::R0D);
845 BuildMI(BB&: *MBB, I: MBB->end(), MIMD: DL, MCID: ZII->get(Opcode: SystemZ::BRC))
846 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_GT).addMBB(MBB);
847
848 MBB = DoneMBB;
849 MBBI = DoneMBB->begin();
850 buildDefCFAReg(MBB&: *MBB, MBBI, DL, Reg: SystemZ::R15D, ZII);
851 }
852
853 if (Residual)
854 allocateAndProbe(*MBB, MBBI, Residual, true/*EmitCFI*/);
855
856 if (StoreBackchain)
857 BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::STG))
858 .addReg(RegNo: SystemZ::R1D, Flags: RegState::Kill).addReg(RegNo: SystemZ::R15D)
859 .addImm(Val: getBackchainOffset(MF)).addReg(RegNo: 0);
860
861 StackAllocMI->eraseFromParent();
862 if (DoneMBB != nullptr) {
863 // Compute the live-in lists for the new blocks.
864 fullyRecomputeLiveIns(MBBs: {DoneMBB, LoopMBB});
865 }
866}
867
868bool SystemZELFFrameLowering::hasFPImpl(const MachineFunction &MF) const {
869 return (MF.disableFramePointerElim() ||
870 MF.getFrameInfo().hasVarSizedObjects());
871}
872
873StackOffset SystemZELFFrameLowering::getFrameIndexReference(
874 const MachineFunction &MF, int FI, Register &FrameReg) const {
875 // Our incoming SP is actually SystemZMC::ELFCallFrameSize below the CFA, so
876 // add that difference here.
877 StackOffset Offset =
878 TargetFrameLowering::getFrameIndexReference(MF, FI, FrameReg);
879 return Offset + StackOffset::getFixed(Fixed: SystemZMC::ELFCallFrameSize);
880}
881
882unsigned SystemZELFFrameLowering::getRegSpillOffset(MachineFunction &MF,
883 Register Reg) const {
884 bool IsVarArg = MF.getFunction().isVarArg();
885 const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
886 bool BackChain = Subtarget.hasBackChain();
887 bool SoftFloat = Subtarget.hasSoftFloat();
888 unsigned Offset = RegSpillOffsets[Reg];
889 if (usePackedStack(MF) && !(IsVarArg && !SoftFloat)) {
890 if (SystemZ::GR64BitRegClass.contains(Reg))
891 // Put all GPRs at the top of the Register save area with packed
892 // stack. Make room for the backchain if needed.
893 Offset += BackChain ? 24 : 32;
894 else
895 Offset = 0;
896 }
897 return Offset;
898}
899
900int SystemZELFFrameLowering::getOrCreateFramePointerSaveIndex(
901 MachineFunction &MF) const {
902 SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
903 int FI = ZFI->getFramePointerSaveIndex();
904 if (!FI) {
905 MachineFrameInfo &MFFrame = MF.getFrameInfo();
906 int Offset = getBackchainOffset(MF) - SystemZMC::ELFCallFrameSize;
907 FI = MFFrame.CreateFixedObject(Size: getPointerSize(), SPOffset: Offset, IsImmutable: false);
908 ZFI->setFramePointerSaveIndex(FI);
909 }
910 return FI;
911}
912
913bool SystemZELFFrameLowering::usePackedStack(MachineFunction &MF) const {
914 bool HasPackedStackAttr = MF.getFunction().hasFnAttribute(Kind: "packed-stack");
915 const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
916 bool BackChain = Subtarget.hasBackChain();
917 bool SoftFloat = Subtarget.hasSoftFloat();
918 if (HasPackedStackAttr && BackChain && !SoftFloat)
919 report_fatal_error(reason: "packed-stack + backchain + hard-float is unsupported.");
920 bool CallConv = MF.getFunction().getCallingConv() != CallingConv::GHC;
921 return HasPackedStackAttr && CallConv;
922}
923
924SystemZXPLINKFrameLowering::SystemZXPLINKFrameLowering(unsigned PointerSize)
925 : SystemZFrameLowering(TargetFrameLowering::StackGrowsDown, Align(32), 0,
926 Align(32), /* StackRealignable */ false,
927 PointerSize),
928 RegSpillOffsets(-1) {
929
930 // Create a mapping from register number to save slot offset.
931 // These offsets are relative to the start of the local are area.
932 RegSpillOffsets.grow(N: SystemZ::NUM_TARGET_REGS);
933 for (const auto &Entry : XPLINKSpillOffsetTable)
934 RegSpillOffsets[Entry.Reg] = Entry.Offset;
935}
936
937int SystemZXPLINKFrameLowering::getOrCreateFramePointerSaveIndex(
938 MachineFunction &MF) const {
939 SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
940 int FI = ZFI->getFramePointerSaveIndex();
941 if (!FI) {
942 MachineFrameInfo &MFFrame = MF.getFrameInfo();
943 FI = MFFrame.CreateFixedObject(Size: getPointerSize(), SPOffset: 0, IsImmutable: false);
944 MFFrame.setStackID(ObjectIdx: FI, ID: TargetStackID::NoAlloc);
945 ZFI->setFramePointerSaveIndex(FI);
946 }
947 return FI;
948}
949
950// Checks if the function is a potential candidate for being a XPLeaf routine.
951static bool isXPLeafCandidate(const MachineFunction &MF) {
952 const MachineFrameInfo &MFFrame = MF.getFrameInfo();
953 const MachineRegisterInfo &MRI = MF.getRegInfo();
954 const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
955 auto *Regs =
956 static_cast<SystemZXPLINK64Registers *>(Subtarget.getSpecialRegisters());
957
958 // If function calls other functions including alloca, then it is not a XPLeaf
959 // routine.
960 if (MFFrame.hasCalls())
961 return false;
962
963 // If the function has var Sized Objects, then it is not a XPLeaf routine.
964 if (MFFrame.hasVarSizedObjects())
965 return false;
966
967 // If the function adjusts the stack, then it is not a XPLeaf routine.
968 if (MFFrame.adjustsStack())
969 return false;
970
971 // If function modifies the stack pointer register, then it is not a XPLeaf
972 // routine.
973 if (MRI.isPhysRegModified(PhysReg: Regs->getStackPointerRegister()))
974 return false;
975
976 // If function modifies the ADA register, then it is not a XPLeaf routine.
977 if (MRI.isPhysRegModified(PhysReg: Regs->getAddressOfCalleeRegister()))
978 return false;
979
980 // If function modifies the return address register, then it is not a XPLeaf
981 // routine.
982 if (MRI.isPhysRegModified(PhysReg: Regs->getReturnFunctionAddressRegister()))
983 return false;
984
985 // If the backchain pointer should be stored, then it is not a XPLeaf routine.
986 if (MF.getSubtarget<SystemZSubtarget>().hasBackChain())
987 return false;
988
989 // If function acquires its own stack frame, then it is not a XPLeaf routine.
990 // At the time this function is called, only slots for local variables are
991 // allocated, so this is a very rough estimate.
992 if (MFFrame.estimateStackSize(MF) > 0)
993 return false;
994
995 return true;
996}
997
998bool SystemZXPLINKFrameLowering::assignCalleeSavedSpillSlots(
999 MachineFunction &MF, const TargetRegisterInfo *TRI,
1000 std::vector<CalleeSavedInfo> &CSI) const {
1001 MachineFrameInfo &MFFrame = MF.getFrameInfo();
1002 SystemZMachineFunctionInfo *MFI = MF.getInfo<SystemZMachineFunctionInfo>();
1003 const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
1004 auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
1005 auto &GRRegClass = SystemZ::GR64BitRegClass;
1006
1007 // At this point, the result of isXPLeafCandidate() is not accurate because
1008 // the size of the save area has not yet been determined. If
1009 // isXPLeafCandidate() indicates a potential leaf function, and there are no
1010 // callee-save registers, then it is indeed a leaf function, and we can early
1011 // exit.
1012 // TODO: It is possible for leaf functions to use callee-saved registers.
1013 // It can use the 0-2k range between R4 and the caller's stack frame without
1014 // acquiring its own stack frame.
1015 bool IsLeaf = CSI.empty() && isXPLeafCandidate(MF);
1016 if (IsLeaf)
1017 return true;
1018
1019 // For non-leaf functions:
1020 // - the address of callee (entry point) register R6 must be saved
1021 CSI.push_back(x: CalleeSavedInfo(Regs.getAddressOfCalleeRegister()));
1022 CSI.back().setRestored(false);
1023
1024 // The return address register R7 must be saved and restored.
1025 CSI.push_back(x: CalleeSavedInfo(Regs.getReturnFunctionAddressRegister()));
1026
1027 // If the function needs a frame pointer, or if the backchain pointer should
1028 // be stored, then save the stack pointer register R4.
1029 if (hasFP(MF) || Subtarget.hasBackChain())
1030 CSI.push_back(x: CalleeSavedInfo(Regs.getStackPointerRegister()));
1031
1032 // If this function has an associated personality function then the
1033 // environment register R5 must be saved in the DSA.
1034 if (!MF.getLandingPads().empty()) {
1035 CSI.push_back(x: CalleeSavedInfo(Regs.getADARegister()));
1036 CSI.back().setRestored(false);
1037 }
1038
1039 // Scan the call-saved GPRs and find the bounds of the register spill area.
1040 Register LowRestoreGPR = 0;
1041 int LowRestoreOffset = INT32_MAX;
1042 Register LowSpillGPR = 0;
1043 int LowSpillOffset = INT32_MAX;
1044 Register HighGPR = 0;
1045 int HighOffset = -1;
1046
1047 // Query index of the saved frame pointer.
1048 int FPSI = MFI->getFramePointerSaveIndex();
1049
1050 for (auto &CS : CSI) {
1051 MCRegister Reg = CS.getReg();
1052 int Offset = RegSpillOffsets[Reg];
1053 if (Offset >= 0) {
1054 if (GRRegClass.contains(Reg)) {
1055 if (LowSpillOffset > Offset) {
1056 LowSpillOffset = Offset;
1057 LowSpillGPR = Reg;
1058 }
1059 if (CS.isRestored() && LowRestoreOffset > Offset) {
1060 LowRestoreOffset = Offset;
1061 LowRestoreGPR = Reg;
1062 }
1063
1064 if (Offset > HighOffset) {
1065 HighOffset = Offset;
1066 HighGPR = Reg;
1067 }
1068 // Non-volatile GPRs are saved in the dedicated register save area at
1069 // the bottom of the stack and are not truly part of the "normal" stack
1070 // frame. Mark the frame index as NoAlloc to indicate it as such.
1071 unsigned RegSize = getPointerSize();
1072 int FrameIdx =
1073 (FPSI && Offset == 0)
1074 ? FPSI
1075 : MFFrame.CreateFixedSpillStackObject(Size: RegSize, SPOffset: Offset);
1076 CS.setFrameIdx(FrameIdx);
1077 MFFrame.setStackID(ObjectIdx: FrameIdx, ID: TargetStackID::NoAlloc);
1078 }
1079 } else {
1080 MCRegister Reg = CS.getReg();
1081 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
1082 Align Alignment = TRI->getSpillAlign(RC: *RC);
1083 unsigned Size = TRI->getSpillSize(RC: *RC);
1084 Alignment = std::min(a: Alignment, b: getStackAlign());
1085 int FrameIdx = MFFrame.CreateStackObject(Size, Alignment, isSpillSlot: true);
1086 CS.setFrameIdx(FrameIdx);
1087 }
1088 }
1089
1090 // Save the range of call-saved registers, for use by the
1091 // prologue/epilogue inserters.
1092 if (LowRestoreGPR)
1093 MFI->setRestoreGPRRegs(Low: LowRestoreGPR, High: HighGPR, Offs: LowRestoreOffset);
1094
1095 // Save the range of call-saved registers, for use by the epilogue inserter.
1096 assert(LowSpillGPR && "Expected registers to spill");
1097 MFI->setSpillGPRRegs(Low: LowSpillGPR, High: HighGPR, Offs: LowSpillOffset);
1098
1099 return true;
1100}
1101
1102void SystemZXPLINKFrameLowering::determineCalleeSaves(MachineFunction &MF,
1103 BitVector &SavedRegs,
1104 RegScavenger *RS) const {
1105 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
1106
1107 bool HasFP = hasFP(MF);
1108 const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
1109 auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
1110
1111 // If the function requires a frame pointer, record that the hard
1112 // frame pointer will be clobbered.
1113 if (HasFP)
1114 SavedRegs.set(Regs.getFramePointerRegister());
1115}
1116
1117bool SystemZXPLINKFrameLowering::spillCalleeSavedRegisters(
1118 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
1119 ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
1120 if (CSI.empty())
1121 return true;
1122
1123 MachineFunction &MF = *MBB.getParent();
1124 SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
1125 const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
1126 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1127 auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
1128 SystemZ::GPRRegs SpillGPRs = ZFI->getSpillGPRRegs();
1129 DebugLoc DL;
1130
1131 // Save GPRs
1132 if (SpillGPRs.LowGPR) {
1133 assert(SpillGPRs.LowGPR != SpillGPRs.HighGPR &&
1134 "Should be saving multiple registers");
1135
1136 // Build an STM/STMG instruction.
1137 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STMG));
1138
1139 // Add the explicit register operands.
1140 addSavedGPR(MBB, MIB, GPR64: SpillGPRs.LowGPR, IsImplicit: false);
1141 addSavedGPR(MBB, MIB, GPR64: SpillGPRs.HighGPR, IsImplicit: false);
1142
1143 // Add the address r4
1144 MIB.addReg(RegNo: Regs.getStackPointerRegister());
1145
1146 // Add the partial offset
1147 // We cannot add the actual offset as, at the stack is not finalized
1148 MIB.addImm(Val: SpillGPRs.GPROffset);
1149
1150 // Make sure all call-saved GPRs are included as operands and are
1151 // marked as live on entry.
1152 auto &GRRegClass = SystemZ::GR64BitRegClass;
1153 for (const CalleeSavedInfo &I : CSI) {
1154 MCRegister Reg = I.getReg();
1155 if (GRRegClass.contains(Reg))
1156 addSavedGPR(MBB, MIB, GPR64: Reg, IsImplicit: true);
1157 }
1158 }
1159
1160 // Spill FPRs to the stack in the normal TargetInstrInfo way
1161 // Registers in CSI are in inverted order so registers with large
1162 // numbers will be assigned to high address.
1163 // Reverse the order at spilling and restoring so instructions on
1164 // registers with small numbers are emitted first.
1165 for (const CalleeSavedInfo &I : llvm::reverse(C&: CSI)) {
1166 MCRegister Reg = I.getReg();
1167 if (SystemZ::FP64BitRegClass.contains(Reg)) {
1168 MBB.addLiveIn(PhysReg: Reg);
1169 TII->storeRegToStackSlot(MBB, MI: MBBI, SrcReg: Reg, isKill: true, FrameIndex: I.getFrameIdx(),
1170 RC: &SystemZ::FP64BitRegClass, VReg: Register());
1171 }
1172 if (SystemZ::VR128BitRegClass.contains(Reg)) {
1173 MBB.addLiveIn(PhysReg: Reg);
1174 TII->storeRegToStackSlot(MBB, MI: MBBI, SrcReg: Reg, isKill: true, FrameIndex: I.getFrameIdx(),
1175 RC: &SystemZ::VR128BitRegClass, VReg: Register());
1176 }
1177 }
1178
1179 return true;
1180}
1181
1182bool SystemZXPLINKFrameLowering::restoreCalleeSavedRegisters(
1183 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
1184 MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
1185
1186 if (CSI.empty())
1187 return false;
1188
1189 MachineFunction &MF = *MBB.getParent();
1190 SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
1191 const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
1192 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1193 auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
1194
1195 DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
1196
1197 // Restore FPRs in the normal TargetInstrInfo way.
1198 for (const CalleeSavedInfo &I : llvm::reverse(C&: CSI)) {
1199 MCRegister Reg = I.getReg();
1200 if (SystemZ::FP64BitRegClass.contains(Reg))
1201 TII->loadRegFromStackSlot(MBB, MI: MBBI, DestReg: Reg, FrameIndex: I.getFrameIdx(),
1202 RC: &SystemZ::FP64BitRegClass, VReg: Register());
1203 if (SystemZ::VR128BitRegClass.contains(Reg))
1204 TII->loadRegFromStackSlot(MBB, MI: MBBI, DestReg: Reg, FrameIndex: I.getFrameIdx(),
1205 RC: &SystemZ::VR128BitRegClass, VReg: Register());
1206 }
1207
1208 // Restore call-saved GPRs (but not call-clobbered varargs, which at
1209 // this point might hold return values).
1210 SystemZ::GPRRegs RestoreGPRs = ZFI->getRestoreGPRRegs();
1211 if (RestoreGPRs.LowGPR) {
1212 assert(isInt<20>(Regs.getStackPointerBias() + RestoreGPRs.GPROffset));
1213 if (RestoreGPRs.LowGPR == RestoreGPRs.HighGPR)
1214 // Build an LG/L instruction.
1215 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG), DestReg: RestoreGPRs.LowGPR)
1216 .addReg(RegNo: Regs.getStackPointerRegister())
1217 .addImm(Val: Regs.getStackPointerBias() + RestoreGPRs.GPROffset)
1218 .addReg(RegNo: 0);
1219 else {
1220 // Build an LMG/LM instruction.
1221 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LMG));
1222
1223 // Add the explicit register operands.
1224 MIB.addReg(RegNo: RestoreGPRs.LowGPR, Flags: RegState::Define);
1225 MIB.addReg(RegNo: RestoreGPRs.HighGPR, Flags: RegState::Define);
1226
1227 // Add the address.
1228 MIB.addReg(RegNo: Regs.getStackPointerRegister());
1229 MIB.addImm(Val: Regs.getStackPointerBias() + RestoreGPRs.GPROffset);
1230
1231 // Do a second scan adding regs as being defined by instruction
1232 for (const CalleeSavedInfo &I : CSI) {
1233 MCRegister Reg = I.getReg();
1234 if (Reg > RestoreGPRs.LowGPR && Reg < RestoreGPRs.HighGPR)
1235 MIB.addReg(RegNo: Reg, Flags: RegState::ImplicitDefine);
1236 }
1237 }
1238 }
1239
1240 return true;
1241}
1242
1243void SystemZXPLINKFrameLowering::emitPrologue(MachineFunction &MF,
1244 MachineBasicBlock &MBB) const {
1245 assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported");
1246 unsigned InstCount = MBB.size();
1247 const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
1248 SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
1249 MachineBasicBlock::iterator MBBI = MBB.begin();
1250 auto *ZII = Subtarget.getInstrInfo();
1251 auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
1252 MachineFrameInfo &MFFrame = MF.getFrameInfo();
1253 MachineInstr *StoreInstr = nullptr;
1254
1255 determineFrameLayout(MF);
1256
1257 bool HasFP = hasFP(MF);
1258 // Debug location must be unknown since the first debug location is used
1259 // to determine the end of the prologue.
1260 DebugLoc DL;
1261 uint64_t Offset = 0;
1262
1263 const uint64_t StackSize = MFFrame.getStackSize();
1264
1265 if (ZFI->getSpillGPRRegs().LowGPR) {
1266 // Skip over the GPR saves.
1267 if ((MBBI != MBB.end()) && ((MBBI->getOpcode() == SystemZ::STMG))) {
1268 const int Operand = 3;
1269 // Now we can set the offset for the operation, since now the Stack
1270 // has been finalized.
1271 Offset = Regs.getStackPointerBias() + MBBI->getOperand(i: Operand).getImm();
1272 // Maximum displacement for STMG instruction.
1273 if (isInt<20>(x: Offset - StackSize))
1274 Offset -= StackSize;
1275 else
1276 StoreInstr = &*MBBI;
1277 MBBI->getOperand(i: Operand).setImm(Offset);
1278 ++MBBI;
1279 } else
1280 llvm_unreachable("Couldn't skip over GPR saves");
1281 }
1282
1283 if (StackSize) {
1284 MachineBasicBlock::iterator InsertPt = StoreInstr ? StoreInstr : MBBI;
1285 // Allocate StackSize bytes.
1286 int64_t Delta = -int64_t(StackSize);
1287
1288 // In case the STM(G) instruction also stores SP (R4), but the displacement
1289 // is too large, the SP register is manipulated first before storing,
1290 // resulting in the wrong value stored and retrieved later. In this case, we
1291 // need to temporarily save the value of SP, and store it later to memory.
1292 if (StoreInstr && HasFP) {
1293 // Insert LR r0,r4 before STMG instruction.
1294 BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::LGR))
1295 .addReg(RegNo: SystemZ::R0D, Flags: RegState::Define)
1296 .addReg(RegNo: SystemZ::R4D);
1297 // Insert ST r0,xxx(,r4) after STMG instruction.
1298 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::STG))
1299 .addReg(RegNo: SystemZ::R0D, Flags: RegState::Kill)
1300 .addReg(RegNo: SystemZ::R4D)
1301 .addImm(Val: Offset)
1302 .addReg(RegNo: 0);
1303 }
1304
1305 emitIncrement(MBB, MBBI&: InsertPt, DL, Reg: Regs.getStackPointerRegister(), NumBytes: Delta,
1306 TII: ZII);
1307
1308 // If the requested stack size is larger than the guard page, then we need
1309 // to check if we need to call the stack extender. This requires adding a
1310 // conditional branch, but splitting the prologue block is not possible at
1311 // this point since it would invalidate the SaveBlocks / RestoreBlocks sets
1312 // of PEI in the single block function case. Build a pseudo to be handled
1313 // later by inlineStackProbe().
1314 const uint64_t GuardPageSize = 1024 * 1024;
1315 if (StackSize > GuardPageSize) {
1316 assert(StoreInstr && "Wrong insertion point");
1317 BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::XPLINK_STACKALLOC));
1318 }
1319 }
1320
1321 if (HasFP) {
1322 // Copy the base of the frame to Frame Pointer Register.
1323 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::LGR),
1324 DestReg: Regs.getFramePointerRegister())
1325 .addReg(RegNo: Regs.getStackPointerRegister());
1326
1327 // Mark the FramePtr as live at the beginning of every block except
1328 // the entry block. (We'll have marked R8 as live on entry when
1329 // saving the GPRs.)
1330 for (MachineBasicBlock &B : llvm::drop_begin(RangeOrContainer&: MF))
1331 B.addLiveIn(PhysReg: Regs.getFramePointerRegister());
1332 }
1333
1334 // Save GPRs used for varargs, if any.
1335 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1336 bool IsVarArg = MF.getFunction().isVarArg();
1337
1338 if (IsVarArg) {
1339 // FixedRegs is the number of used registers, accounting for shadow
1340 // registers.
1341 unsigned FixedRegs = ZFI->getVarArgsFirstGPR() + ZFI->getVarArgsFirstFPR();
1342 auto &GPRs = SystemZ::XPLINK64ArgGPRs;
1343 for (unsigned I = FixedRegs; I < SystemZ::XPLINK64NumArgGPRs; I++) {
1344 uint64_t StartOffset = MFFrame.getOffsetAdjustment() +
1345 MFFrame.getStackSize() + Regs.getCallFrameSize() +
1346 getOffsetOfLocalArea() + I * getPointerSize();
1347 unsigned Reg = GPRs[I];
1348 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1349 .addReg(RegNo: Reg)
1350 .addReg(RegNo: Regs.getStackPointerRegister())
1351 .addImm(Val: StartOffset)
1352 .addReg(RegNo: 0);
1353 if (!MBB.isLiveIn(Reg))
1354 MBB.addLiveIn(PhysReg: Reg);
1355 }
1356 }
1357
1358 // Check if any new instructions were inserted. If not, it means no there is
1359 // no prologue and thus no need for a fence. The fence is required because
1360 // moving instructions inside the prologue might violate some of the rules
1361 // required to hold for prologues, for example the maximum lengths of the
1362 // prologue code. See all rules at
1363 // https://www.ibm.com/docs/en/zos/3.1.0?topic=SSLTBW_3.1.0/com.ibm.zos.v3r1.ceev100/cee1v2319.html
1364 if (InstCount < MBB.size()) {
1365 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::FENCE));
1366 }
1367}
1368
1369void SystemZXPLINKFrameLowering::emitEpilogue(MachineFunction &MF,
1370 MachineBasicBlock &MBB) const {
1371 const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
1372 MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
1373 SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
1374 MachineFrameInfo &MFFrame = MF.getFrameInfo();
1375 auto *ZII = Subtarget.getInstrInfo();
1376 auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
1377
1378 // Skip the return instruction.
1379 assert(MBBI->isReturn() && "Can only insert epilogue into returning blocks");
1380
1381 uint64_t StackSize = MFFrame.getStackSize();
1382 if (StackSize) {
1383 unsigned SPReg = Regs.getStackPointerRegister();
1384 if (ZFI->getRestoreGPRRegs().LowGPR != SPReg) {
1385 DebugLoc DL = MBBI->getDebugLoc();
1386 emitIncrement(MBB, MBBI, DL, Reg: SPReg, NumBytes: StackSize, TII: ZII);
1387 }
1388 }
1389}
1390
1391// Emit a compare of the stack pointer against the stack floor, and a call to
1392// the LE stack extender if needed.
1393void SystemZXPLINKFrameLowering::inlineStackProbe(
1394 MachineFunction &MF, MachineBasicBlock &PrologMBB) const {
1395 auto *ZII =
1396 static_cast<const SystemZInstrInfo *>(MF.getSubtarget().getInstrInfo());
1397
1398 MachineInstr *StackAllocMI = nullptr;
1399 for (MachineInstr &MI : PrologMBB)
1400 if (MI.getOpcode() == SystemZ::XPLINK_STACKALLOC) {
1401 StackAllocMI = &MI;
1402 break;
1403 }
1404 if (StackAllocMI == nullptr)
1405 return;
1406
1407 bool NeedSaveSP = hasFP(MF);
1408 bool NeedSaveArg = PrologMBB.isLiveIn(Reg: SystemZ::R3D);
1409 const int64_t SaveSlotR3 = 2192;
1410
1411 MachineBasicBlock &MBB = PrologMBB;
1412 const DebugLoc DL = StackAllocMI->getDebugLoc();
1413
1414 // The 2nd half of block MBB after split.
1415 MachineBasicBlock *NextMBB;
1416
1417 // Add new basic block for the call to the stack overflow function.
1418 MachineBasicBlock *StackExtMBB =
1419 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
1420 MF.push_back(MBB: StackExtMBB);
1421
1422 // LG r3,72(,r3)
1423 BuildMI(BB: StackExtMBB, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::LG), DestReg: SystemZ::R3D)
1424 .addReg(RegNo: SystemZ::R3D)
1425 .addImm(Val: 72)
1426 .addReg(RegNo: 0);
1427 // BASR r3,r3
1428 BuildMI(BB: StackExtMBB, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::CallBASR_STACKEXT))
1429 .addReg(RegNo: SystemZ::R3D);
1430 if (NeedSaveArg) {
1431 if (!NeedSaveSP) {
1432 // LGR r0,r3
1433 BuildMI(BB&: MBB, I: StackAllocMI, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::LGR))
1434 .addReg(RegNo: SystemZ::R0D, Flags: RegState::Define)
1435 .addReg(RegNo: SystemZ::R3D);
1436 } else {
1437 // In this case, the incoming value of r4 is saved in r0 so the
1438 // latter register is unavailable. Store r3 in its corresponding
1439 // slot in the parameter list instead. Do this at the start of
1440 // the prolog before r4 is manipulated by anything else.
1441 // STG r3, 2192(r4)
1442 BuildMI(BB&: MBB, I: MBB.begin(), MIMD: DL, MCID: ZII->get(Opcode: SystemZ::STG))
1443 .addReg(RegNo: SystemZ::R3D)
1444 .addReg(RegNo: SystemZ::R4D)
1445 .addImm(Val: SaveSlotR3)
1446 .addReg(RegNo: 0);
1447 }
1448 }
1449 // LLGT r3,1208
1450 BuildMI(BB&: MBB, I: StackAllocMI, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::LLGT), DestReg: SystemZ::R3D)
1451 .addReg(RegNo: 0)
1452 .addImm(Val: 1208)
1453 .addReg(RegNo: 0);
1454 // CG r4,64(,r3)
1455 BuildMI(BB&: MBB, I: StackAllocMI, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::CG))
1456 .addReg(RegNo: SystemZ::R4D)
1457 .addReg(RegNo: SystemZ::R3D)
1458 .addImm(Val: 64)
1459 .addReg(RegNo: 0);
1460 // JLL b'0100',F'37'
1461 BuildMI(BB&: MBB, I: StackAllocMI, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::BRC))
1462 .addImm(Val: SystemZ::CCMASK_ICMP)
1463 .addImm(Val: SystemZ::CCMASK_CMP_LT)
1464 .addMBB(MBB: StackExtMBB);
1465
1466 NextMBB = SystemZ::splitBlockBefore(MI: StackAllocMI, MBB: &MBB);
1467 MBB.addSuccessor(Succ: NextMBB);
1468 MBB.addSuccessor(Succ: StackExtMBB);
1469 if (NeedSaveArg) {
1470 if (!NeedSaveSP) {
1471 // LGR r3, r0
1472 BuildMI(BB&: *NextMBB, I: StackAllocMI, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::LGR))
1473 .addReg(RegNo: SystemZ::R3D, Flags: RegState::Define)
1474 .addReg(RegNo: SystemZ::R0D, Flags: RegState::Kill);
1475 } else {
1476 // In this case, the incoming value of r4 is saved in r0 so the
1477 // latter register is unavailable. We stored r3 in its corresponding
1478 // slot in the parameter list instead and we now restore it from there.
1479 // LGR r3, r0
1480 BuildMI(BB&: *NextMBB, I: StackAllocMI, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::LGR))
1481 .addReg(RegNo: SystemZ::R3D, Flags: RegState::Define)
1482 .addReg(RegNo: SystemZ::R0D);
1483 // LG r3, 2192(r3)
1484 BuildMI(BB&: *NextMBB, I: StackAllocMI, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::LG))
1485 .addReg(RegNo: SystemZ::R3D, Flags: RegState::Define)
1486 .addReg(RegNo: SystemZ::R3D)
1487 .addImm(Val: SaveSlotR3)
1488 .addReg(RegNo: 0);
1489 }
1490 }
1491
1492 // Add jump back from stack extension BB.
1493 BuildMI(BB: StackExtMBB, MIMD: DL, MCID: ZII->get(Opcode: SystemZ::J)).addMBB(MBB: NextMBB);
1494 StackExtMBB->addSuccessor(Succ: NextMBB);
1495
1496 StackAllocMI->eraseFromParent();
1497
1498 // Compute the live-in lists for the new blocks.
1499 fullyRecomputeLiveIns(MBBs: {StackExtMBB, NextMBB});
1500}
1501
1502bool SystemZXPLINKFrameLowering::hasFPImpl(const MachineFunction &MF) const {
1503 return (MF.getFrameInfo().hasVarSizedObjects());
1504}
1505
1506void SystemZXPLINKFrameLowering::processFunctionBeforeFrameFinalized(
1507 MachineFunction &MF, RegScavenger *RS) const {
1508 MachineFrameInfo &MFFrame = MF.getFrameInfo();
1509 const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
1510 auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
1511
1512 // Setup stack frame offset
1513 MFFrame.setOffsetAdjustment(Regs.getStackPointerBias());
1514
1515 // Nothing to do for leaf functions.
1516 uint64_t StackSize = MFFrame.estimateStackSize(MF);
1517 if (StackSize == 0 && MFFrame.getCalleeSavedInfo().empty())
1518 return;
1519
1520 // Although the XPLINK specifications for AMODE64 state that minimum size
1521 // of the param area is minimum 32 bytes and no rounding is otherwise
1522 // specified, we round this area in 64 bytes increments to be compatible
1523 // with existing compilers.
1524 MFFrame.setMaxCallFrameSize(
1525 std::max(a: 64U, b: (unsigned)alignTo(Value: MFFrame.getMaxCallFrameSize(), Align: 64)));
1526
1527 // Add frame values with positive object offsets. Since the displacement from
1528 // the SP/FP is calculated by ObjectOffset + StackSize + Bias, object offsets
1529 // with positive values are in the caller's stack frame. We need to include
1530 // that since it is accessed by displacement to SP/FP.
1531 int64_t LargestArgOffset = 0;
1532 for (int I = MFFrame.getObjectIndexBegin(); I != 0; ++I) {
1533 if (MFFrame.getObjectOffset(ObjectIdx: I) >= 0) {
1534 int64_t ObjOffset = MFFrame.getObjectOffset(ObjectIdx: I) + MFFrame.getObjectSize(ObjectIdx: I);
1535 LargestArgOffset = std::max(a: ObjOffset, b: LargestArgOffset);
1536 }
1537 }
1538
1539 uint64_t MaxReach = (StackSize + Regs.getCallFrameSize() +
1540 Regs.getStackPointerBias() + LargestArgOffset);
1541
1542 if (!isUInt<12>(x: MaxReach)) {
1543 // We may need register scavenging slots if some parts of the frame
1544 // are outside the reach of an unsigned 12-bit displacement.
1545 RS->addScavengingFrameIndex(FI: MFFrame.CreateSpillStackObject(Size: 8, Alignment: Align(8)));
1546 RS->addScavengingFrameIndex(FI: MFFrame.CreateSpillStackObject(Size: 8, Alignment: Align(8)));
1547 }
1548}
1549
1550// Determines the size of the frame, and creates the deferred spill objects.
1551void SystemZXPLINKFrameLowering::determineFrameLayout(
1552 MachineFunction &MF) const {
1553 MachineFrameInfo &MFFrame = MF.getFrameInfo();
1554 const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
1555 auto *Regs =
1556 static_cast<SystemZXPLINK64Registers *>(Subtarget.getSpecialRegisters());
1557
1558 uint64_t StackSize = MFFrame.getStackSize();
1559 // A function which saves callee-saved registers needs a register save area of
1560 // its own, even if it has no other stack objects. Otherwise the registers are
1561 // stored relative to the unchanged stack pointer, i.e. into the save area of
1562 // the caller's DSA.
1563 if (StackSize == 0 && MFFrame.getCalleeSavedInfo().empty())
1564 return;
1565
1566 // Add the size of the register save area and the reserved area to the size.
1567 StackSize += Regs->getCallFrameSize();
1568 MFFrame.setStackSize(StackSize);
1569
1570 // We now know the stack size. Update the stack objects for the register save
1571 // area now. This has no impact on the stack frame layout, as this is already
1572 // computed. However, it makes sure that all callee saved registers have a
1573 // valid offset assigned.
1574 for (int FrameIdx = MFFrame.getObjectIndexBegin(); FrameIdx != 0;
1575 ++FrameIdx) {
1576 if (MFFrame.getStackID(ObjectIdx: FrameIdx) == TargetStackID::NoAlloc) {
1577 int64_t SPOffset = MFFrame.getObjectOffset(ObjectIdx: FrameIdx);
1578 SPOffset -= StackSize;
1579 MFFrame.setObjectOffset(ObjectIdx: FrameIdx, SPOffset);
1580 }
1581 }
1582}
1583