1//===-- X86FrameLowering.cpp - X86 Frame Information ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the X86 implementation of TargetFrameLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "X86FrameLowering.h"
14#include "MCTargetDesc/X86MCTargetDesc.h"
15#include "X86.h"
16#include "X86InstrBuilder.h"
17#include "X86InstrInfo.h"
18#include "X86MachineFunctionInfo.h"
19#include "X86Subtarget.h"
20#include "X86TargetMachine.h"
21#include "llvm/ADT/Statistic.h"
22#include "llvm/CodeGen/LivePhysRegs.h"
23#include "llvm/CodeGen/MachineFrameInfo.h"
24#include "llvm/CodeGen/MachineFunction.h"
25#include "llvm/CodeGen/MachineInstrBuilder.h"
26#include "llvm/CodeGen/MachineModuleInfo.h"
27#include "llvm/CodeGen/MachineRegisterInfo.h"
28#include "llvm/CodeGen/RegisterScavenging.h"
29#include "llvm/CodeGen/WinEHFuncInfo.h"
30#include "llvm/IR/DataLayout.h"
31#include "llvm/IR/EHPersonalities.h"
32#include "llvm/IR/Function.h"
33#include "llvm/IR/Module.h"
34#include "llvm/MC/MCAsmInfo.h"
35#include "llvm/MC/MCObjectFileInfo.h"
36#include "llvm/MC/MCSymbol.h"
37#include "llvm/Support/LEB128.h"
38#include "llvm/Target/TargetOptions.h"
39#include <cstdlib>
40
41#define DEBUG_TYPE "x86-fl"
42
43STATISTIC(NumFrameLoopProbe, "Number of loop stack probes used in prologue");
44STATISTIC(NumFrameExtraProbe,
45 "Number of extra stack probes generated in prologue");
46STATISTIC(NumFunctionUsingPush2Pop2, "Number of functions using push2/pop2");
47
48using namespace llvm;
49
50bool llvm::requireWinX64UnwindV3(const MachineFunction &MF) {
51 const Function &Fn = MF.getFunction();
52
53 // Whole module is in V3 mode.
54 if (Fn.getParent()->getWinX64EHUnwindMode() == WinX64EHUnwindMode::V3)
55 return true;
56
57 // Otherwise promote a function that may use EGPR (R16-R31), which V1/V2
58 // unwind codes cannot encode. The per-function "+egpr" feature is the signal,
59 // so an auto-dispatch APX clone gets V3 while the baseline clone stays on the
60 // module default. We conservatively promote any egpr function rather than
61 // checking for an actual EGPR save, keeping this a cheap query. (PUSH2/POP2
62 // does not need V3: V1/V2 describe a PUSH2 as two SEH_PushReg codes.)
63 return Fn.needsUnwindTableEntry() &&
64 MF.getSubtarget<X86Subtarget>().hasEGPR();
65}
66
67static const TargetRegisterClass *
68getCalleeSavedSpillRC(MCRegister Reg, const X86Subtarget &STI,
69 const TargetRegisterInfo &TRI) {
70 if (X86::VK16RegClass.contains(Reg))
71 return STI.hasBWI() ? &X86::VK64RegClass : &X86::VK16RegClass;
72 return TRI.getMinimalPhysRegClass(Reg);
73}
74
75X86FrameLowering::X86FrameLowering(const X86Subtarget &STI,
76 MaybeAlign StackAlignOverride)
77 : TargetFrameLowering(StackGrowsDown, StackAlignOverride.valueOrOne(),
78 STI.is64Bit() ? -8 : -4),
79 STI(STI), TII(*STI.getInstrInfo()), TRI(STI.getRegisterInfo()) {
80 // Cache a bunch of frame-related predicates for this subtarget.
81 SlotSize = TRI->getSlotSize();
82 assert(SlotSize == 4 || SlotSize == 8);
83 Is64Bit = STI.is64Bit();
84 IsLP64 = STI.isTarget64BitLP64();
85 // standard x86_64 uses 64-bit frame/stack pointers, x32 - 32-bit.
86 Uses64BitFramePtr = STI.isTarget64BitLP64();
87 StackPtr = TRI->getStackRegister();
88}
89
90bool X86FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
91 return !MF.getFrameInfo().hasVarSizedObjects() &&
92 !MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences() &&
93 !MF.getInfo<X86MachineFunctionInfo>()->hasPreallocatedCall();
94}
95
96/// canSimplifyCallFramePseudos - If there is a reserved call frame, the
97/// call frame pseudos can be simplified. Having a FP, as in the default
98/// implementation, is not sufficient here since we can't always use it.
99/// Use a more nuanced condition.
100bool X86FrameLowering::canSimplifyCallFramePseudos(
101 const MachineFunction &MF) const {
102 return hasReservedCallFrame(MF) ||
103 MF.getInfo<X86MachineFunctionInfo>()->hasPreallocatedCall() ||
104 (hasFP(MF) && !TRI->hasStackRealignment(MF)) ||
105 TRI->hasBasePointer(MF);
106}
107
108// needsFrameIndexResolution - Do we need to perform FI resolution for
109// this function. Normally, this is required only when the function
110// has any stack objects. However, FI resolution actually has another job,
111// not apparent from the title - it resolves callframesetup/destroy
112// that were not simplified earlier.
113// So, this is required for x86 functions that have push sequences even
114// when there are no stack objects.
115bool X86FrameLowering::needsFrameIndexResolution(
116 const MachineFunction &MF) const {
117 return MF.getFrameInfo().hasStackObjects() ||
118 MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences();
119}
120
121/// hasFPImpl - Return true if the specified function should have a dedicated
122/// frame pointer register. This is true if the function has variable sized
123/// allocas or if frame pointer elimination is disabled.
124bool X86FrameLowering::hasFPImpl(const MachineFunction &MF) const {
125 const MachineFrameInfo &MFI = MF.getFrameInfo();
126 return (MF.getTarget().Options.DisableFramePointerElim(MF) ||
127 TRI->hasStackRealignment(MF) || MFI.hasVarSizedObjects() ||
128 MFI.isFrameAddressTaken() || MFI.hasOpaqueSPAdjustment() ||
129 MF.getInfo<X86MachineFunctionInfo>()->getForceFramePointer() ||
130 MF.getInfo<X86MachineFunctionInfo>()->hasPreallocatedCall() ||
131 MF.callsUnwindInit() || MF.hasEHFunclets() || MF.callsEHReturn() ||
132 MFI.hasStackMap() || MFI.hasPatchPoint() ||
133 (isWin64Prologue(MF) && MFI.hasCopyImplyingStackAdjustment()));
134}
135
136static unsigned getSUBriOpcode(bool IsLP64) {
137 return IsLP64 ? X86::SUB64ri32 : X86::SUB32ri;
138}
139
140static unsigned getADDriOpcode(bool IsLP64) {
141 return IsLP64 ? X86::ADD64ri32 : X86::ADD32ri;
142}
143
144static unsigned getSUBrrOpcode(bool IsLP64) {
145 return IsLP64 ? X86::SUB64rr : X86::SUB32rr;
146}
147
148static unsigned getADDrrOpcode(bool IsLP64) {
149 return IsLP64 ? X86::ADD64rr : X86::ADD32rr;
150}
151
152static unsigned getANDriOpcode(bool IsLP64, int64_t Imm) {
153 return IsLP64 ? X86::AND64ri32 : X86::AND32ri;
154}
155
156static unsigned getLEArOpcode(bool IsLP64) {
157 return IsLP64 ? X86::LEA64r : X86::LEA32r;
158}
159
160// Push-Pop Acceleration (PPX) hint is used to indicate that the POP reads the
161// value written by the PUSH from the stack. The processor tracks these marked
162// instructions internally and fast-forwards register data between matching PUSH
163// and POP instructions, without going through memory or through the training
164// loop of the Fast Store Forwarding Predictor (FSFP). Instead, a more efficient
165// memory-renaming optimization can be used.
166//
167// The PPX hint is purely a performance hint. Instructions with this hint have
168// the same functional semantics as those without. PPX hints set by the
169// compiler that violate the balancing rule may turn off the PPX optimization,
170// but they will not affect program semantics.
171//
172// Hence, PPX is used for balanced spill/reloads (Exceptions and setjmp/longjmp
173// are not considered).
174//
175// PUSH2 and POP2 are instructions for (respectively) pushing/popping 2
176// GPRs at a time to/from the stack.
177static unsigned getPUSHOpcode(const X86Subtarget &ST) {
178 return ST.is64Bit() ? (ST.hasPPX() ? X86::PUSHP64r : X86::PUSH64r)
179 : X86::PUSH32r;
180}
181static unsigned getPOPOpcode(const X86Subtarget &ST) {
182 return ST.is64Bit() ? (ST.hasPPX() ? X86::POPP64r : X86::POP64r)
183 : X86::POP32r;
184}
185static unsigned getPUSH2Opcode(const X86Subtarget &ST) {
186 return ST.hasPPX() ? X86::PUSH2P : X86::PUSH2;
187}
188static unsigned getPOP2Opcode(const X86Subtarget &ST) {
189 return ST.hasPPX() ? X86::POP2P : X86::POP2;
190}
191
192static bool isEAXLiveIn(MachineBasicBlock &MBB) {
193 for (MachineBasicBlock::RegisterMaskPair RegMask : MBB.liveins()) {
194 MCRegister Reg = RegMask.PhysReg;
195
196 if (Reg == X86::RAX || Reg == X86::EAX || Reg == X86::AX ||
197 Reg == X86::AH || Reg == X86::AL)
198 return true;
199 }
200
201 return false;
202}
203
204/// Check if the flags need to be preserved before the terminators.
205/// This would be the case, if the eflags is live-in of the region
206/// composed by the terminators or live-out of that region, without
207/// being defined by a terminator.
208static bool
209flagsNeedToBePreservedBeforeTheTerminators(const MachineBasicBlock &MBB) {
210 for (const MachineInstr &MI : MBB.terminators()) {
211 bool BreakNext = false;
212 for (const MachineOperand &MO : MI.operands()) {
213 if (!MO.isReg())
214 continue;
215 Register Reg = MO.getReg();
216 if (Reg != X86::EFLAGS)
217 continue;
218
219 // This terminator needs an eflags that is not defined
220 // by a previous another terminator:
221 // EFLAGS is live-in of the region composed by the terminators.
222 if (!MO.isDef())
223 return true;
224 // This terminator defines the eflags, i.e., we don't need to preserve it.
225 // However, we still need to check this specific terminator does not
226 // read a live-in value.
227 BreakNext = true;
228 }
229 // We found a definition of the eflags, no need to preserve them.
230 if (BreakNext)
231 return false;
232 }
233
234 // None of the terminators use or define the eflags.
235 // Check if they are live-out, that would imply we need to preserve them.
236 for (const MachineBasicBlock *Succ : MBB.successors())
237 if (Succ->isLiveIn(Reg: X86::EFLAGS))
238 return true;
239
240 return false;
241}
242
243constexpr uint64_t MaxSPChunk = (1ULL << 31) - 1;
244
245/// emitSPUpdate - Emit a series of instructions to increment / decrement the
246/// stack pointer by a constant value.
247void X86FrameLowering::emitSPUpdate(MachineBasicBlock &MBB,
248 MachineBasicBlock::iterator &MBBI,
249 const DebugLoc &DL, int64_t NumBytes,
250 bool InEpilogue) const {
251 bool isSub = NumBytes < 0;
252 uint64_t Offset = isSub ? -NumBytes : NumBytes;
253 MachineInstr::MIFlag Flag =
254 isSub ? MachineInstr::FrameSetup : MachineInstr::FrameDestroy;
255
256 if (!Uses64BitFramePtr && !isUInt<32>(x: Offset)) {
257 // We're being asked to adjust a 32-bit stack pointer by 4 GiB or more.
258 // This might be unreachable code, so don't complain now; just trap if
259 // it's reached at runtime.
260 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::TRAP));
261 return;
262 }
263
264 MachineFunction &MF = *MBB.getParent();
265 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
266 const X86TargetLowering &TLI = *STI.getTargetLowering();
267 const bool EmitInlineStackProbe = TLI.hasInlineStackProbe(MF);
268
269 // It's ok to not take into account large chunks when probing, as the
270 // allocation is split in smaller chunks anyway.
271 if (EmitInlineStackProbe && !InEpilogue) {
272
273 // This pseudo-instruction is going to be expanded, potentially using a
274 // loop, by inlineStackProbe().
275 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::STACKALLOC_W_PROBING)).addImm(Val: Offset);
276 return;
277 } else if (Offset > MaxSPChunk) {
278 // Rather than emit a long series of instructions for large offsets,
279 // load the offset into a register and do one sub/add
280 unsigned Reg = 0;
281 unsigned Rax = (unsigned)(Uses64BitFramePtr ? X86::RAX : X86::EAX);
282
283 if (isSub && !isEAXLiveIn(MBB))
284 Reg = Rax;
285 else
286 Reg = getX86SubSuperRegister(Reg: TRI->findDeadCallerSavedReg(MBB, MBBI),
287 Size: Uses64BitFramePtr ? 64 : 32);
288
289 unsigned AddSubRROpc = isSub ? getSUBrrOpcode(IsLP64: Uses64BitFramePtr)
290 : getADDrrOpcode(IsLP64: Uses64BitFramePtr);
291 if (Reg) {
292 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
293 MCID: TII.get(Opcode: X86::getMOVriOpcode(Use64BitReg: Uses64BitFramePtr, Imm: Offset)), DestReg: Reg)
294 .addImm(Val: Offset)
295 .setMIFlag(Flag);
296 MachineInstr *MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AddSubRROpc), DestReg: StackPtr)
297 .addReg(RegNo: StackPtr)
298 .addReg(RegNo: Reg);
299 MI->getOperand(i: 3).setIsDead(); // The EFLAGS implicit def is dead.
300 return;
301 } else if (Offset > 8 * MaxSPChunk) {
302 // If we would need more than 8 add or sub instructions (a >16GB stack
303 // frame), it's worth spilling RAX to materialize this immediate.
304 // pushq %rax
305 // movabsq +-$Offset+-SlotSize, %rax
306 // addq %rsp, %rax
307 // xchg %rax, (%rsp)
308 // movq (%rsp), %rsp
309 assert(Uses64BitFramePtr && "can't have 32-bit 16GB stack frame");
310 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH64r))
311 .addReg(RegNo: Rax, Flags: RegState::Kill)
312 .setMIFlag(Flag);
313 // Subtract is not commutative, so negate the offset and always use add.
314 // Subtract 8 less and add 8 more to account for the PUSH we just did.
315 if (isSub)
316 Offset = -(Offset - SlotSize);
317 else
318 Offset = Offset + SlotSize;
319 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
320 MCID: TII.get(Opcode: X86::getMOVriOpcode(Use64BitReg: Uses64BitFramePtr, Imm: Offset)), DestReg: Rax)
321 .addImm(Val: Offset)
322 .setMIFlag(Flag);
323 MachineInstr *MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::ADD64rr), DestReg: Rax)
324 .addReg(RegNo: Rax)
325 .addReg(RegNo: StackPtr);
326 MI->getOperand(i: 3).setIsDead(); // The EFLAGS implicit def is dead.
327 // Exchange the new SP in RAX with the top of the stack.
328 addRegOffset(
329 MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::XCHG64rm), DestReg: Rax).addReg(RegNo: Rax),
330 Reg: StackPtr, isKill: false, Offset: 0);
331 // Load new SP from the top of the stack into RSP.
332 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64rm), DestReg: StackPtr),
333 Reg: StackPtr, isKill: false, Offset: 0);
334 return;
335 }
336 }
337
338 while (Offset) {
339 if (Offset == SlotSize) {
340 // Use push / pop for slot sized adjustments as a size optimization. We
341 // need to find a dead register when using pop.
342 unsigned Reg = isSub ? (unsigned)(Is64Bit ? X86::RAX : X86::EAX)
343 : TRI->findDeadCallerSavedReg(MBB, MBBI);
344 if (Reg) {
345 unsigned Opc = isSub ? (Is64Bit ? X86::PUSH64r : X86::PUSH32r)
346 : (Is64Bit ? X86::POP64r : X86::POP32r);
347 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opc))
348 .addReg(RegNo: Reg, Flags: getDefRegState(B: !isSub) | getUndefRegState(B: isSub))
349 .setMIFlag(Flag);
350 return;
351 }
352 }
353
354 uint64_t ThisVal = std::min(a: Offset, b: MaxSPChunk);
355
356 BuildStackAdjustment(MBB, MBBI, DL, Offset: isSub ? -ThisVal : ThisVal, InEpilogue)
357 .setMIFlag(Flag);
358
359 Offset -= ThisVal;
360 }
361}
362
363MachineInstrBuilder X86FrameLowering::BuildStackAdjustment(
364 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
365 const DebugLoc &DL, int64_t Offset, bool InEpilogue) const {
366 assert(Offset != 0 && "zero offset stack adjustment requested");
367
368 // On Atom, using LEA to adjust SP is preferred, but using it in the epilogue
369 // is tricky.
370 bool UseLEA;
371 if (!InEpilogue) {
372 // Check if inserting the prologue at the beginning
373 // of MBB would require to use LEA operations.
374 // We need to use LEA operations if EFLAGS is live in, because
375 // it means an instruction will read it before it gets defined.
376 UseLEA = STI.useLeaForSP() || MBB.isLiveIn(Reg: X86::EFLAGS);
377 } else {
378 // If we can use LEA for SP but we shouldn't, check that none
379 // of the terminators uses the eflags. Otherwise we will insert
380 // a ADD that will redefine the eflags and break the condition.
381 // Alternatively, we could move the ADD, but this may not be possible
382 // and is an optimization anyway.
383 UseLEA = canUseLEAForSPInEpilogue(MF: *MBB.getParent());
384 if (UseLEA && !STI.useLeaForSP())
385 UseLEA = flagsNeedToBePreservedBeforeTheTerminators(MBB);
386 // If that assert breaks, that means we do not do the right thing
387 // in canUseAsEpilogue.
388 assert((UseLEA || !flagsNeedToBePreservedBeforeTheTerminators(MBB)) &&
389 "We shouldn't have allowed this insertion point");
390 }
391
392 MachineInstrBuilder MI;
393 // Use an NF (no-flags) variant as a smaller replacement for LEA when EFLAGS
394 // must be preserved (i.e. only when we would otherwise emit LEA). If EFLAGS
395 // is dead we prefer the plain SUB/ADD, which is shorter than the EVEX-encoded
396 // NF form. The NF stack-adjust opcodes below are 64-bit (SUB64ri32_NF/
397 // ADD64ri32_NF), so don't use them for the x32 ABI where the stack pointer is
398 // 32-bit. NF cannot reach a Win64 epilogue (which never uses LEA for the SP
399 // adjustment unless it has a frame pointer, and that path doesn't go through
400 // here), so the Windows epilogue unwinder never sees an undisassemblable NF
401 // add/sub.
402 bool UseNF = UseLEA && STI.hasNF() && Uses64BitFramePtr;
403 bool IsSub = Offset < 0;
404 uint64_t AbsOffset = IsSub ? -Offset : Offset;
405 if (UseNF) {
406 const unsigned Opc = IsSub ? X86::SUB64ri32_NF : X86::ADD64ri32_NF;
407 MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: StackPtr)
408 .addReg(RegNo: StackPtr)
409 .addImm(Val: AbsOffset);
410 // NF instructions define no EFLAGS, so there is nothing to mark dead.
411 } else if (UseLEA) {
412 MI = addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
413 MCID: TII.get(Opcode: getLEArOpcode(IsLP64: Uses64BitFramePtr)),
414 DestReg: StackPtr),
415 Reg: StackPtr, isKill: false, Offset);
416 } else {
417 const unsigned Opc = IsSub ? getSUBriOpcode(IsLP64: Uses64BitFramePtr)
418 : getADDriOpcode(IsLP64: Uses64BitFramePtr);
419 MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: StackPtr)
420 .addReg(RegNo: StackPtr)
421 .addImm(Val: AbsOffset);
422 MI->getOperand(i: 3).setIsDead(); // The EFLAGS implicit def is dead.
423 }
424 return MI;
425}
426
427template <typename FoundT, typename CalcT>
428int64_t X86FrameLowering::mergeSPUpdates(MachineBasicBlock &MBB,
429 MachineBasicBlock::iterator &MBBI,
430 FoundT FoundStackAdjust,
431 CalcT CalcNewOffset,
432 bool doMergeWithPrevious) const {
433 if ((doMergeWithPrevious && MBBI == MBB.begin()) ||
434 (!doMergeWithPrevious && MBBI == MBB.end()))
435 return CalcNewOffset(0);
436
437 MachineBasicBlock::iterator PI = doMergeWithPrevious ? std::prev(x: MBBI) : MBBI;
438
439 PI = skipDebugInstructionsBackward(It: PI, Begin: MBB.begin());
440 // It is assumed that ADD/SUB/LEA instruction is succeded by one CFI
441 // instruction, and that there are no DBG_VALUE or other instructions between
442 // ADD/SUB/LEA and its corresponding CFI instruction.
443 /* TODO: Add support for the case where there are multiple CFI instructions
444 below the ADD/SUB/LEA, e.g.:
445 ...
446 add
447 cfi_def_cfa_offset
448 cfi_offset
449 ...
450 */
451 if (doMergeWithPrevious && PI != MBB.begin() && PI->isCFIInstruction())
452 PI = std::prev(x: PI);
453
454 int64_t Offset = 0;
455 for (;;) {
456 unsigned Opc = PI->getOpcode();
457
458 if ((Opc == X86::ADD64ri32 || Opc == X86::ADD32ri ||
459 Opc == X86::ADD64ri32_NF) &&
460 PI->getOperand(i: 0).getReg() == StackPtr) {
461 assert(PI->getOperand(1).getReg() == StackPtr);
462 Offset = PI->getOperand(i: 2).getImm();
463 } else if ((Opc == X86::LEA32r || Opc == X86::LEA64_32r) &&
464 PI->getOperand(i: 0).getReg() == StackPtr &&
465 PI->getOperand(i: 1).getReg() == StackPtr &&
466 PI->getOperand(i: 2).getImm() == 1 &&
467 PI->getOperand(i: 3).getReg() == X86::NoRegister &&
468 PI->getOperand(i: 5).getReg() == X86::NoRegister) {
469 // For LEAs we have: def = lea SP, FI, noreg, Offset, noreg.
470 Offset = PI->getOperand(i: 4).getImm();
471 } else if ((Opc == X86::SUB64ri32 || Opc == X86::SUB32ri ||
472 Opc == X86::SUB64ri32_NF) &&
473 PI->getOperand(i: 0).getReg() == StackPtr) {
474 assert(PI->getOperand(1).getReg() == StackPtr);
475 Offset = -PI->getOperand(i: 2).getImm();
476 } else
477 return CalcNewOffset(0);
478
479 FoundStackAdjust(PI, Offset);
480 if ((uint64_t)std::abs(i: (int64_t)CalcNewOffset(Offset)) < MaxSPChunk)
481 break;
482
483 if (doMergeWithPrevious ? (PI == MBB.begin()) : (PI == MBB.end()))
484 return CalcNewOffset(0);
485
486 PI = doMergeWithPrevious ? std::prev(x: PI) : std::next(x: PI);
487 }
488
489 PI = MBB.erase(I: PI);
490 if (PI != MBB.end() && PI->isCFIInstruction()) {
491 auto CIs = MBB.getParent()->getFrameInstructions();
492 MCCFIInstruction CI = CIs[PI->getOperand(i: 0).getCFIIndex()];
493 if (CI.getOperation() == MCCFIInstruction::OpDefCfaOffset ||
494 CI.getOperation() == MCCFIInstruction::OpAdjustCfaOffset)
495 PI = MBB.erase(I: PI);
496 }
497 if (!doMergeWithPrevious)
498 MBBI = skipDebugInstructionsForward(It: PI, End: MBB.end());
499
500 return CalcNewOffset(Offset);
501}
502
503int64_t X86FrameLowering::mergeSPAdd(MachineBasicBlock &MBB,
504 MachineBasicBlock::iterator &MBBI,
505 int64_t AddOffset,
506 bool doMergeWithPrevious) const {
507 return mergeSPUpdates(
508 MBB, MBBI, CalcNewOffset: [AddOffset](int64_t Offset) { return AddOffset + Offset; },
509 doMergeWithPrevious);
510}
511
512void X86FrameLowering::BuildCFI(MachineBasicBlock &MBB,
513 MachineBasicBlock::iterator MBBI,
514 const DebugLoc &DL,
515 const MCCFIInstruction &CFIInst,
516 MachineInstr::MIFlag Flag) const {
517 MachineFunction &MF = *MBB.getParent();
518 unsigned CFIIndex = MF.addFrameInst(Inst: CFIInst);
519
520 if (CFIInst.getOperation() == MCCFIInstruction::OpAdjustCfaOffset)
521 MF.getInfo<X86MachineFunctionInfo>()->setHasCFIAdjustCfa(true);
522
523 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::CFI_INSTRUCTION))
524 .addCFIIndex(CFIIndex)
525 .setMIFlag(Flag);
526}
527
528/// Emits Dwarf Info specifying offsets of callee saved registers and
529/// frame pointer. This is called only when basic block sections are enabled.
530void X86FrameLowering::emitCalleeSavedFrameMovesFullCFA(
531 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
532 MachineFunction &MF = *MBB.getParent();
533 if (!hasFP(MF)) {
534 emitCalleeSavedFrameMoves(MBB, MBBI, DL: DebugLoc{}, IsPrologue: true);
535 return;
536 }
537 const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
538 const Register FramePtr = TRI->getFrameRegister(MF);
539 const Register MachineFramePtr =
540 STI.isTarget64BitILP32() ? Register(getX86SubSuperRegister(Reg: FramePtr, Size: 64))
541 : FramePtr;
542 unsigned DwarfReg = MRI->getDwarfRegNum(Reg: MachineFramePtr, isEH: true);
543 // Offset = space for return address + size of the frame pointer itself.
544 int64_t Offset = (Is64Bit ? 8 : 4) + (Uses64BitFramePtr ? 8 : 4);
545 BuildCFI(MBB, MBBI, DL: DebugLoc{},
546 CFIInst: MCCFIInstruction::createOffset(L: nullptr, Register: DwarfReg, Offset: -Offset));
547 emitCalleeSavedFrameMoves(MBB, MBBI, DL: DebugLoc{}, IsPrologue: true);
548}
549
550void X86FrameLowering::emitCalleeSavedFrameMoves(
551 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
552 const DebugLoc &DL, bool IsPrologue) const {
553 MachineFunction &MF = *MBB.getParent();
554 MachineFrameInfo &MFI = MF.getFrameInfo();
555 const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
556 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
557
558 // Add callee saved registers to move list.
559 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
560
561 // Calculate offsets.
562 for (const CalleeSavedInfo &I : CSI) {
563 int64_t Offset = MFI.getObjectOffset(ObjectIdx: I.getFrameIdx());
564 MCRegister Reg = I.getReg();
565 unsigned DwarfReg = MRI->getDwarfRegNum(Reg, isEH: true);
566
567 if (IsPrologue) {
568 if (X86FI->getStackPtrSaveMI()) {
569 // +2*SlotSize because there is return address and ebp at the bottom
570 // of the stack.
571 // | retaddr |
572 // | ebp |
573 // | |<--ebp
574 Offset += 2 * SlotSize;
575 SmallString<64> CfaExpr;
576 CfaExpr.push_back(Elt: dwarf::DW_CFA_expression);
577 uint8_t buffer[16];
578 CfaExpr.append(in_start: buffer, in_end: buffer + encodeULEB128(Value: DwarfReg, p: buffer));
579 CfaExpr.push_back(Elt: 2);
580 Register FramePtr = TRI->getFrameRegister(MF);
581 const Register MachineFramePtr =
582 STI.isTarget64BitILP32()
583 ? Register(getX86SubSuperRegister(Reg: FramePtr, Size: 64))
584 : FramePtr;
585 unsigned DwarfFramePtr = MRI->getDwarfRegNum(Reg: MachineFramePtr, isEH: true);
586 CfaExpr.push_back(Elt: (uint8_t)(dwarf::DW_OP_breg0 + DwarfFramePtr));
587 CfaExpr.append(in_start: buffer, in_end: buffer + encodeSLEB128(Value: Offset, p: buffer));
588 BuildCFI(MBB, MBBI, DL,
589 CFIInst: MCCFIInstruction::createEscape(L: nullptr, Vals: CfaExpr.str()),
590 Flag: MachineInstr::FrameSetup);
591 } else {
592 BuildCFI(MBB, MBBI, DL,
593 CFIInst: MCCFIInstruction::createOffset(L: nullptr, Register: DwarfReg, Offset));
594 }
595 } else {
596 BuildCFI(MBB, MBBI, DL,
597 CFIInst: MCCFIInstruction::createRestore(L: nullptr, Register: DwarfReg));
598 }
599 }
600 if (auto *MI = X86FI->getStackPtrSaveMI()) {
601 int FI = MI->getOperand(i: 1).getIndex();
602 int64_t Offset = MFI.getObjectOffset(ObjectIdx: FI) + 2 * SlotSize;
603 SmallString<64> CfaExpr;
604 Register FramePtr = TRI->getFrameRegister(MF);
605 const Register MachineFramePtr =
606 STI.isTarget64BitILP32()
607 ? Register(getX86SubSuperRegister(Reg: FramePtr, Size: 64))
608 : FramePtr;
609 unsigned DwarfFramePtr = MRI->getDwarfRegNum(Reg: MachineFramePtr, isEH: true);
610 CfaExpr.push_back(Elt: (uint8_t)(dwarf::DW_OP_breg0 + DwarfFramePtr));
611 uint8_t buffer[16];
612 CfaExpr.append(in_start: buffer, in_end: buffer + encodeSLEB128(Value: Offset, p: buffer));
613 CfaExpr.push_back(Elt: dwarf::DW_OP_deref);
614
615 SmallString<64> DefCfaExpr;
616 DefCfaExpr.push_back(Elt: dwarf::DW_CFA_def_cfa_expression);
617 DefCfaExpr.append(in_start: buffer, in_end: buffer + encodeSLEB128(Value: CfaExpr.size(), p: buffer));
618 DefCfaExpr.append(RHS: CfaExpr.str());
619 // DW_CFA_def_cfa_expression: DW_OP_breg5 offset, DW_OP_deref
620 BuildCFI(MBB, MBBI, DL,
621 CFIInst: MCCFIInstruction::createEscape(L: nullptr, Vals: DefCfaExpr.str()),
622 Flag: MachineInstr::FrameSetup);
623 }
624}
625
626void X86FrameLowering::emitZeroCallUsedRegs(BitVector RegsToZero,
627 MachineBasicBlock &MBB,
628 RegScavenger *) const {
629 const MachineFunction &MF = *MBB.getParent();
630
631 // Insertion point.
632 MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
633
634 // Fake a debug loc.
635 DebugLoc DL;
636 if (MBBI != MBB.end())
637 DL = MBBI->getDebugLoc();
638
639 // Zero out FP stack if referenced. Do this outside of the loop below so that
640 // it's done only once.
641 for (MCRegister Reg : RegsToZero.set_bits()) {
642 if (!X86::RFP80RegClass.contains(Reg))
643 continue;
644
645 // Do not push zeros over x87 return values. X86FloatingPoint records
646 // returned values as implicit ST0/ST1 uses on the return instruction.
647 unsigned NumFPRegs = 8;
648 if (MBBI->hasRegisterImplicitUseOperand(Reg: X86::ST0))
649 --NumFPRegs;
650 if (MBBI->hasRegisterImplicitUseOperand(Reg: X86::ST1))
651 --NumFPRegs;
652
653 for (unsigned i = 0; i != NumFPRegs; ++i)
654 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::LD_F0));
655
656 for (unsigned i = 0; i != NumFPRegs; ++i)
657 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::ST_FPrr)).addReg(RegNo: X86::ST0);
658 break;
659 }
660
661 // For GPRs, we only care to clear out the 32-bit register.
662 BitVector GPRsToZero(TRI->getNumRegs());
663 for (MCRegister Reg : RegsToZero.set_bits())
664 if (TRI->isGeneralPurposeRegister(MF, Reg)) {
665 GPRsToZero.set(getX86SubSuperRegister(Reg, Size: 32));
666 RegsToZero.reset(Idx: Reg);
667 }
668
669 // Zero out the GPRs first.
670 for (MCRegister Reg : GPRsToZero.set_bits())
671 TII.buildClearRegister(Reg, MBB, Iter: MBBI, DL);
672
673 // Zero out the remaining registers.
674 for (MCRegister Reg : RegsToZero.set_bits())
675 TII.buildClearRegister(Reg, MBB, Iter: MBBI, DL);
676}
677
678void X86FrameLowering::emitStackProbe(
679 MachineFunction &MF, MachineBasicBlock &MBB,
680 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog,
681 std::optional<MachineFunction::DebugInstrOperandPair> InstrNum) const {
682 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
683 if (STI.isTargetWindowsCoreCLR()) {
684 if (InProlog) {
685 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::STACKALLOC_W_PROBING))
686 .addImm(Val: 0 /* no explicit stack size */);
687 } else {
688 emitStackProbeInline(MF, MBB, MBBI, DL, InProlog: false);
689 }
690 } else {
691 emitStackProbeCall(MF, MBB, MBBI, DL, InProlog, InstrNum);
692 }
693}
694
695bool X86FrameLowering::stackProbeFunctionModifiesSP() const {
696 return STI.isOSWindows() && !STI.isTargetWin64();
697}
698
699void X86FrameLowering::inlineStackProbe(MachineFunction &MF,
700 MachineBasicBlock &PrologMBB) const {
701 auto Where = llvm::find_if(Range&: PrologMBB, P: [](MachineInstr &MI) {
702 return MI.getOpcode() == X86::STACKALLOC_W_PROBING;
703 });
704 if (Where != PrologMBB.end()) {
705 DebugLoc DL = PrologMBB.findDebugLoc(MBBI: Where);
706 emitStackProbeInline(MF, MBB&: PrologMBB, MBBI: Where, DL, InProlog: true);
707 Where->eraseFromParent();
708 }
709}
710
711void X86FrameLowering::emitStackProbeInline(MachineFunction &MF,
712 MachineBasicBlock &MBB,
713 MachineBasicBlock::iterator MBBI,
714 const DebugLoc &DL,
715 bool InProlog) const {
716 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
717 if (STI.isTargetWindowsCoreCLR() && STI.is64Bit())
718 emitStackProbeInlineWindowsCoreCLR64(MF, MBB, MBBI, DL, InProlog);
719 else
720 emitStackProbeInlineGeneric(MF, MBB, MBBI, DL, InProlog);
721}
722
723void X86FrameLowering::emitStackProbeInlineGeneric(
724 MachineFunction &MF, MachineBasicBlock &MBB,
725 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog) const {
726 MachineInstr &AllocWithProbe = *MBBI;
727 uint64_t Offset = AllocWithProbe.getOperand(i: 0).getImm();
728
729 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
730 const X86TargetLowering &TLI = *STI.getTargetLowering();
731 assert(!(STI.is64Bit() && STI.isTargetWindowsCoreCLR()) &&
732 "different expansion expected for CoreCLR 64 bit");
733
734 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
735 uint64_t ProbeChunk = StackProbeSize * 8;
736
737 uint64_t MaxAlign =
738 TRI->hasStackRealignment(MF) ? calculateMaxStackAlign(MF) : 0;
739
740 // Synthesize a loop or unroll it, depending on the number of iterations.
741 // BuildStackAlignAND ensures that only MaxAlign % StackProbeSize bits left
742 // between the unaligned rsp and current rsp.
743 if (Offset > ProbeChunk) {
744 emitStackProbeInlineGenericLoop(MF, MBB, MBBI, DL, Offset,
745 Align: MaxAlign % StackProbeSize);
746 } else {
747 emitStackProbeInlineGenericBlock(MF, MBB, MBBI, DL, Offset,
748 Align: MaxAlign % StackProbeSize);
749 }
750}
751
752void X86FrameLowering::emitStackProbeInlineGenericBlock(
753 MachineFunction &MF, MachineBasicBlock &MBB,
754 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, uint64_t Offset,
755 uint64_t AlignOffset) const {
756
757 const bool NeedsDwarfCFI = needsDwarfCFI(MF);
758 const bool HasFP = hasFP(MF);
759 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
760 const X86TargetLowering &TLI = *STI.getTargetLowering();
761 const unsigned MovMIOpc = Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
762 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
763
764 uint64_t CurrentOffset = 0;
765
766 assert(AlignOffset < StackProbeSize);
767
768 // If the offset is so small it fits within a page, there's nothing to do.
769 if (StackProbeSize < Offset + AlignOffset) {
770
771 uint64_t StackAdjustment = StackProbeSize - AlignOffset;
772 BuildStackAdjustment(MBB, MBBI, DL, Offset: -StackAdjustment, /*InEpilogue=*/false)
773 .setMIFlag(MachineInstr::FrameSetup);
774 if (!HasFP && NeedsDwarfCFI) {
775 BuildCFI(
776 MBB, MBBI, DL,
777 CFIInst: MCCFIInstruction::createAdjustCfaOffset(L: nullptr, Adjustment: StackAdjustment));
778 }
779
780 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: MovMIOpc))
781 .setMIFlag(MachineInstr::FrameSetup),
782 Reg: StackPtr, isKill: false, Offset: 0)
783 .addImm(Val: 0)
784 .setMIFlag(MachineInstr::FrameSetup);
785 NumFrameExtraProbe++;
786 CurrentOffset = StackProbeSize - AlignOffset;
787 }
788
789 // For the next N - 1 pages, just probe. I tried to take advantage of
790 // natural probes but it implies much more logic and there was very few
791 // interesting natural probes to interleave.
792 while (CurrentOffset + StackProbeSize < Offset) {
793 BuildStackAdjustment(MBB, MBBI, DL, Offset: -StackProbeSize, /*InEpilogue=*/false)
794 .setMIFlag(MachineInstr::FrameSetup);
795
796 if (!HasFP && NeedsDwarfCFI) {
797 BuildCFI(
798 MBB, MBBI, DL,
799 CFIInst: MCCFIInstruction::createAdjustCfaOffset(L: nullptr, Adjustment: StackProbeSize));
800 }
801 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: MovMIOpc))
802 .setMIFlag(MachineInstr::FrameSetup),
803 Reg: StackPtr, isKill: false, Offset: 0)
804 .addImm(Val: 0)
805 .setMIFlag(MachineInstr::FrameSetup);
806 NumFrameExtraProbe++;
807 CurrentOffset += StackProbeSize;
808 }
809
810 // No need to probe the tail, it is smaller than a Page.
811 uint64_t ChunkSize = Offset - CurrentOffset;
812 if (ChunkSize == SlotSize) {
813 // Use push for slot sized adjustments as a size optimization,
814 // like emitSPUpdate does when not probing.
815 unsigned Reg = Is64Bit ? X86::RAX : X86::EAX;
816 unsigned Opc = Is64Bit ? X86::PUSH64r : X86::PUSH32r;
817 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opc))
818 .addReg(RegNo: Reg, Flags: RegState::Undef)
819 .setMIFlag(MachineInstr::FrameSetup);
820 } else {
821 BuildStackAdjustment(MBB, MBBI, DL, Offset: -ChunkSize, /*InEpilogue=*/false)
822 .setMIFlag(MachineInstr::FrameSetup);
823 }
824 // No need to adjust Dwarf CFA offset here, the last position of the stack has
825 // been defined
826}
827
828void X86FrameLowering::emitStackProbeInlineGenericLoop(
829 MachineFunction &MF, MachineBasicBlock &MBB,
830 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, uint64_t Offset,
831 uint64_t AlignOffset) const {
832 assert(Offset && "null offset");
833
834 assert(MBB.computeRegisterLiveness(TRI, X86::EFLAGS, MBBI) !=
835 MachineBasicBlock::LQR_Live &&
836 "Inline stack probe loop will clobber live EFLAGS.");
837
838 const bool NeedsDwarfCFI = needsDwarfCFI(MF);
839 const bool HasFP = hasFP(MF);
840 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
841 const X86TargetLowering &TLI = *STI.getTargetLowering();
842 const unsigned MovMIOpc = Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
843 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
844
845 if (AlignOffset) {
846 if (AlignOffset < StackProbeSize) {
847 // Perform a first smaller allocation followed by a probe.
848 BuildStackAdjustment(MBB, MBBI, DL, Offset: -AlignOffset, /*InEpilogue=*/false)
849 .setMIFlag(MachineInstr::FrameSetup);
850
851 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: MovMIOpc))
852 .setMIFlag(MachineInstr::FrameSetup),
853 Reg: StackPtr, isKill: false, Offset: 0)
854 .addImm(Val: 0)
855 .setMIFlag(MachineInstr::FrameSetup);
856 NumFrameExtraProbe++;
857 Offset -= AlignOffset;
858 }
859 }
860
861 // Synthesize a loop
862 NumFrameLoopProbe++;
863 const BasicBlock *LLVM_BB = MBB.getBasicBlock();
864
865 MachineBasicBlock *testMBB = MF.CreateMachineBasicBlock(BB: LLVM_BB);
866 MachineBasicBlock *tailMBB = MF.CreateMachineBasicBlock(BB: LLVM_BB);
867
868 MachineFunction::iterator MBBIter = ++MBB.getIterator();
869 MF.insert(MBBI: MBBIter, MBB: testMBB);
870 MF.insert(MBBI: MBBIter, MBB: tailMBB);
871
872 Register FinalStackProbed = Uses64BitFramePtr ? X86::R11
873 : Is64Bit ? X86::R11D
874 : X86::EAX;
875
876 // save loop bound
877 {
878 const uint64_t BoundOffset = alignDown(Value: Offset, Align: StackProbeSize);
879
880 // Can we calculate the loop bound using SUB with a 32-bit immediate?
881 // Note that the immediate gets sign-extended when used with a 64-bit
882 // register, so in that case we only have 31 bits to work with.
883 bool canUseSub =
884 Uses64BitFramePtr ? isUInt<31>(x: BoundOffset) : isUInt<32>(x: BoundOffset);
885
886 if (canUseSub) {
887 const unsigned SUBOpc = getSUBriOpcode(IsLP64: Uses64BitFramePtr);
888
889 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: FinalStackProbed)
890 .addReg(RegNo: StackPtr)
891 .setMIFlag(MachineInstr::FrameSetup);
892 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: SUBOpc), DestReg: FinalStackProbed)
893 .addReg(RegNo: FinalStackProbed)
894 .addImm(Val: BoundOffset)
895 .setMIFlag(MachineInstr::FrameSetup);
896 } else if (Uses64BitFramePtr) {
897 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64ri), DestReg: FinalStackProbed)
898 .addImm(Val: -BoundOffset)
899 .setMIFlag(MachineInstr::FrameSetup);
900 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::ADD64rr), DestReg: FinalStackProbed)
901 .addReg(RegNo: FinalStackProbed)
902 .addReg(RegNo: StackPtr)
903 .setMIFlag(MachineInstr::FrameSetup);
904 } else {
905 llvm_unreachable("Offset too large for 32-bit stack pointer");
906 }
907
908 // while in the loop, use loop-invariant reg for CFI,
909 // instead of the stack pointer, which changes during the loop
910 if (!HasFP && NeedsDwarfCFI) {
911 // x32 uses the same DWARF register numbers as x86-64,
912 // so there isn't a register number for r11d, we must use r11 instead
913 const Register DwarfFinalStackProbed =
914 STI.isTarget64BitILP32()
915 ? Register(getX86SubSuperRegister(Reg: FinalStackProbed, Size: 64))
916 : FinalStackProbed;
917
918 BuildCFI(MBB, MBBI, DL,
919 CFIInst: MCCFIInstruction::createDefCfaRegister(
920 L: nullptr, Register: TRI->getDwarfRegNum(Reg: DwarfFinalStackProbed, isEH: true)));
921 BuildCFI(MBB, MBBI, DL,
922 CFIInst: MCCFIInstruction::createAdjustCfaOffset(L: nullptr, Adjustment: BoundOffset));
923 }
924 }
925
926 // allocate a page
927 BuildStackAdjustment(MBB&: *testMBB, MBBI: testMBB->end(), DL, Offset: -StackProbeSize,
928 /*InEpilogue=*/false)
929 .setMIFlag(MachineInstr::FrameSetup);
930
931 // touch the page
932 addRegOffset(MIB: BuildMI(BB: testMBB, MIMD: DL, MCID: TII.get(Opcode: MovMIOpc))
933 .setMIFlag(MachineInstr::FrameSetup),
934 Reg: StackPtr, isKill: false, Offset: 0)
935 .addImm(Val: 0)
936 .setMIFlag(MachineInstr::FrameSetup);
937
938 // cmp with stack pointer bound
939 BuildMI(BB: testMBB, MIMD: DL, MCID: TII.get(Opcode: Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
940 .addReg(RegNo: StackPtr)
941 .addReg(RegNo: FinalStackProbed)
942 .setMIFlag(MachineInstr::FrameSetup);
943
944 // jump
945 BuildMI(BB: testMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
946 .addMBB(MBB: testMBB)
947 .addImm(Val: X86::COND_NE)
948 .setMIFlag(MachineInstr::FrameSetup);
949 testMBB->addSuccessor(Succ: testMBB);
950 testMBB->addSuccessor(Succ: tailMBB);
951
952 // BB management
953 tailMBB->splice(Where: tailMBB->end(), Other: &MBB, From: MBBI, To: MBB.end());
954 tailMBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
955 MBB.addSuccessor(Succ: testMBB);
956
957 // handle tail
958 const uint64_t TailOffset = Offset % StackProbeSize;
959 MachineBasicBlock::iterator TailMBBIter = tailMBB->begin();
960 if (TailOffset) {
961 BuildStackAdjustment(MBB&: *tailMBB, MBBI: TailMBBIter, DL, Offset: -TailOffset,
962 /*InEpilogue=*/false)
963 .setMIFlag(MachineInstr::FrameSetup);
964 }
965
966 // after the loop, switch back to stack pointer for CFI
967 if (!HasFP && NeedsDwarfCFI) {
968 // x32 uses the same DWARF register numbers as x86-64,
969 // so there isn't a register number for esp, we must use rsp instead
970 const Register DwarfStackPtr =
971 STI.isTarget64BitILP32()
972 ? Register(getX86SubSuperRegister(Reg: StackPtr, Size: 64))
973 : Register(StackPtr);
974
975 BuildCFI(MBB&: *tailMBB, MBBI: TailMBBIter, DL,
976 CFIInst: MCCFIInstruction::createDefCfaRegister(
977 L: nullptr, Register: TRI->getDwarfRegNum(Reg: DwarfStackPtr, isEH: true)));
978 }
979
980 // Update Live In information
981 fullyRecomputeLiveIns(MBBs: {tailMBB, testMBB});
982}
983
984void X86FrameLowering::emitStackProbeInlineWindowsCoreCLR64(
985 MachineFunction &MF, MachineBasicBlock &MBB,
986 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog) const {
987 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
988 assert(STI.is64Bit() && "different expansion needed for 32 bit");
989 assert(STI.isTargetWindowsCoreCLR() && "custom expansion expects CoreCLR");
990 const TargetInstrInfo &TII = *STI.getInstrInfo();
991 const BasicBlock *LLVM_BB = MBB.getBasicBlock();
992
993 assert(MBB.computeRegisterLiveness(TRI, X86::EFLAGS, MBBI) !=
994 MachineBasicBlock::LQR_Live &&
995 "Inline stack probe loop will clobber live EFLAGS.");
996
997 // RAX contains the number of bytes of desired stack adjustment.
998 // The handling here assumes this value has already been updated so as to
999 // maintain stack alignment.
1000 //
1001 // We need to exit with RSP modified by this amount and execute suitable
1002 // page touches to notify the OS that we're growing the stack responsibly.
1003 // All stack probing must be done without modifying RSP.
1004 //
1005 // MBB:
1006 // SizeReg = RAX;
1007 // ZeroReg = 0
1008 // CopyReg = RSP
1009 // Flags, TestReg = CopyReg - SizeReg
1010 // FinalReg = !Flags.Ovf ? TestReg : ZeroReg
1011 // LimitReg = gs magic thread env access
1012 // if FinalReg >= LimitReg goto ContinueMBB
1013 // RoundBB:
1014 // RoundReg = page address of FinalReg
1015 // LoopMBB:
1016 // LoopReg = PHI(LimitReg,ProbeReg)
1017 // ProbeReg = LoopReg - PageSize
1018 // [ProbeReg] = 0
1019 // if (ProbeReg > RoundReg) goto LoopMBB
1020 // ContinueMBB:
1021 // RSP = RSP - RAX
1022 // [rest of original MBB]
1023
1024 // Set up the new basic blocks
1025 MachineBasicBlock *RoundMBB = MF.CreateMachineBasicBlock(BB: LLVM_BB);
1026 MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(BB: LLVM_BB);
1027 MachineBasicBlock *ContinueMBB = MF.CreateMachineBasicBlock(BB: LLVM_BB);
1028
1029 MachineFunction::iterator MBBIter = std::next(x: MBB.getIterator());
1030 MF.insert(MBBI: MBBIter, MBB: RoundMBB);
1031 MF.insert(MBBI: MBBIter, MBB: LoopMBB);
1032 MF.insert(MBBI: MBBIter, MBB: ContinueMBB);
1033
1034 // Split MBB and move the tail portion down to ContinueMBB.
1035 MachineBasicBlock::iterator BeforeMBBI = std::prev(x: MBBI);
1036 ContinueMBB->splice(Where: ContinueMBB->begin(), Other: &MBB, From: MBBI, To: MBB.end());
1037 ContinueMBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
1038
1039 // Some useful constants
1040 const int64_t ThreadEnvironmentStackLimit = 0x10;
1041 const int64_t PageSize = 0x1000;
1042 const int64_t PageMask = ~(PageSize - 1);
1043
1044 // Registers we need. For the normal case we use virtual
1045 // registers. For the prolog expansion we use RAX, RCX and RDX.
1046 MachineRegisterInfo &MRI = MF.getRegInfo();
1047 const TargetRegisterClass *RegClass = &X86::GR64RegClass;
1048 const Register
1049 SizeReg = InProlog ? X86::RAX : MRI.createVirtualRegister(RegClass),
1050 ZeroReg = InProlog ? X86::RCX : MRI.createVirtualRegister(RegClass),
1051 CopyReg = InProlog ? X86::RDX : MRI.createVirtualRegister(RegClass),
1052 TestReg = InProlog ? X86::RDX : MRI.createVirtualRegister(RegClass),
1053 FinalReg = InProlog ? X86::RDX : MRI.createVirtualRegister(RegClass),
1054 RoundedReg = InProlog ? X86::RDX : MRI.createVirtualRegister(RegClass),
1055 LimitReg = InProlog ? X86::RCX : MRI.createVirtualRegister(RegClass),
1056 JoinReg = InProlog ? X86::RCX : MRI.createVirtualRegister(RegClass),
1057 ProbeReg = InProlog ? X86::RCX : MRI.createVirtualRegister(RegClass);
1058
1059 // SP-relative offsets where we can save RCX and RDX.
1060 int64_t RCXShadowSlot = 0;
1061 int64_t RDXShadowSlot = 0;
1062
1063 // If inlining in the prolog, save RCX and RDX.
1064 if (InProlog) {
1065 // Compute the offsets. We need to account for things already
1066 // pushed onto the stack at this point: return address, frame
1067 // pointer (if used), and callee saves.
1068 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
1069 const int64_t CalleeSaveSize = X86FI->getCalleeSavedFrameSize();
1070 const bool HasFP = hasFP(MF);
1071
1072 // Check if we need to spill RCX and/or RDX.
1073 // Here we assume that no earlier prologue instruction changes RCX and/or
1074 // RDX, so checking the block live-ins is enough.
1075 const bool IsRCXLiveIn = MBB.isLiveIn(Reg: X86::RCX);
1076 const bool IsRDXLiveIn = MBB.isLiveIn(Reg: X86::RDX);
1077 int64_t InitSlot = 8 + CalleeSaveSize + (HasFP ? 8 : 0);
1078 // Assign the initial slot to both registers, then change RDX's slot if both
1079 // need to be spilled.
1080 if (IsRCXLiveIn)
1081 RCXShadowSlot = InitSlot;
1082 if (IsRDXLiveIn)
1083 RDXShadowSlot = InitSlot;
1084 if (IsRDXLiveIn && IsRCXLiveIn)
1085 RDXShadowSlot += 8;
1086 // Emit the saves if needed.
1087 if (IsRCXLiveIn)
1088 addRegOffset(MIB: BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64mr)), Reg: X86::RSP, isKill: false,
1089 Offset: RCXShadowSlot)
1090 .addReg(RegNo: X86::RCX);
1091 if (IsRDXLiveIn)
1092 addRegOffset(MIB: BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64mr)), Reg: X86::RSP, isKill: false,
1093 Offset: RDXShadowSlot)
1094 .addReg(RegNo: X86::RDX);
1095 } else {
1096 // Not in the prolog. Copy RAX to a virtual reg.
1097 BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64rr), DestReg: SizeReg).addReg(RegNo: X86::RAX);
1098 }
1099
1100 // Add code to MBB to check for overflow and set the new target stack pointer
1101 // to zero if so.
1102 BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::XOR64rr), DestReg: ZeroReg)
1103 .addReg(RegNo: ZeroReg, Flags: RegState::Undef)
1104 .addReg(RegNo: ZeroReg, Flags: RegState::Undef);
1105 BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64rr), DestReg: CopyReg).addReg(RegNo: X86::RSP);
1106 BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::SUB64rr), DestReg: TestReg)
1107 .addReg(RegNo: CopyReg)
1108 .addReg(RegNo: SizeReg);
1109 BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::CMOV64rr), DestReg: FinalReg)
1110 .addReg(RegNo: TestReg)
1111 .addReg(RegNo: ZeroReg)
1112 .addImm(Val: X86::COND_B);
1113
1114 // FinalReg now holds final stack pointer value, or zero if
1115 // allocation would overflow. Compare against the current stack
1116 // limit from the thread environment block. Note this limit is the
1117 // lowest touched page on the stack, not the point at which the OS
1118 // will cause an overflow exception, so this is just an optimization
1119 // to avoid unnecessarily touching pages that are below the current
1120 // SP but already committed to the stack by the OS.
1121 BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64rm), DestReg: LimitReg)
1122 .addReg(RegNo: 0)
1123 .addImm(Val: 1)
1124 .addReg(RegNo: 0)
1125 .addImm(Val: ThreadEnvironmentStackLimit)
1126 .addReg(RegNo: X86::GS);
1127 BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::CMP64rr)).addReg(RegNo: FinalReg).addReg(RegNo: LimitReg);
1128 // Jump if the desired stack pointer is at or above the stack limit.
1129 BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
1130 .addMBB(MBB: ContinueMBB)
1131 .addImm(Val: X86::COND_AE);
1132
1133 // Add code to roundMBB to round the final stack pointer to a page boundary.
1134 if (InProlog)
1135 RoundMBB->addLiveIn(PhysReg: FinalReg);
1136 BuildMI(BB: RoundMBB, MIMD: DL, MCID: TII.get(Opcode: X86::AND64ri32), DestReg: RoundedReg)
1137 .addReg(RegNo: FinalReg)
1138 .addImm(Val: PageMask);
1139 BuildMI(BB: RoundMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JMP_1)).addMBB(MBB: LoopMBB);
1140
1141 // LimitReg now holds the current stack limit, RoundedReg page-rounded
1142 // final RSP value. Add code to loopMBB to decrement LimitReg page-by-page
1143 // and probe until we reach RoundedReg.
1144 if (!InProlog) {
1145 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII.get(Opcode: X86::PHI), DestReg: JoinReg)
1146 .addReg(RegNo: LimitReg)
1147 .addMBB(MBB: RoundMBB)
1148 .addReg(RegNo: ProbeReg)
1149 .addMBB(MBB: LoopMBB);
1150 }
1151
1152 if (InProlog)
1153 LoopMBB->addLiveIn(PhysReg: JoinReg);
1154 addRegOffset(MIB: BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII.get(Opcode: X86::LEA64r), DestReg: ProbeReg), Reg: JoinReg,
1155 isKill: false, Offset: -PageSize);
1156
1157 // Probe by storing a byte onto the stack.
1158 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII.get(Opcode: X86::MOV8mi))
1159 .addReg(RegNo: ProbeReg)
1160 .addImm(Val: 1)
1161 .addReg(RegNo: 0)
1162 .addImm(Val: 0)
1163 .addReg(RegNo: 0)
1164 .addImm(Val: 0);
1165
1166 if (InProlog)
1167 LoopMBB->addLiveIn(PhysReg: RoundedReg);
1168 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII.get(Opcode: X86::CMP64rr))
1169 .addReg(RegNo: RoundedReg)
1170 .addReg(RegNo: ProbeReg);
1171 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
1172 .addMBB(MBB: LoopMBB)
1173 .addImm(Val: X86::COND_NE);
1174
1175 MachineBasicBlock::iterator ContinueMBBI = ContinueMBB->getFirstNonPHI();
1176
1177 // If in prolog, restore RDX and RCX.
1178 if (InProlog) {
1179 if (RCXShadowSlot) // It means we spilled RCX in the prologue.
1180 addRegOffset(MIB: BuildMI(BB&: *ContinueMBB, I: ContinueMBBI, MIMD: DL,
1181 MCID: TII.get(Opcode: X86::MOV64rm), DestReg: X86::RCX),
1182 Reg: X86::RSP, isKill: false, Offset: RCXShadowSlot);
1183 if (RDXShadowSlot) // It means we spilled RDX in the prologue.
1184 addRegOffset(MIB: BuildMI(BB&: *ContinueMBB, I: ContinueMBBI, MIMD: DL,
1185 MCID: TII.get(Opcode: X86::MOV64rm), DestReg: X86::RDX),
1186 Reg: X86::RSP, isKill: false, Offset: RDXShadowSlot);
1187 }
1188
1189 // Now that the probing is done, add code to continueMBB to update
1190 // the stack pointer for real.
1191 BuildMI(BB&: *ContinueMBB, I: ContinueMBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SUB64rr), DestReg: X86::RSP)
1192 .addReg(RegNo: X86::RSP)
1193 .addReg(RegNo: SizeReg);
1194
1195 // Add the control flow edges we need.
1196 MBB.addSuccessor(Succ: ContinueMBB);
1197 MBB.addSuccessor(Succ: RoundMBB);
1198 RoundMBB->addSuccessor(Succ: LoopMBB);
1199 LoopMBB->addSuccessor(Succ: ContinueMBB);
1200 LoopMBB->addSuccessor(Succ: LoopMBB);
1201
1202 if (InProlog) {
1203 LivePhysRegs LiveRegs;
1204 computeAndAddLiveIns(LiveRegs, MBB&: *ContinueMBB);
1205 }
1206
1207 // Mark all the instructions added to the prolog as frame setup.
1208 if (InProlog) {
1209 for (++BeforeMBBI; BeforeMBBI != MBB.end(); ++BeforeMBBI) {
1210 BeforeMBBI->setFlag(MachineInstr::FrameSetup);
1211 }
1212 for (MachineInstr &MI : *RoundMBB) {
1213 MI.setFlag(MachineInstr::FrameSetup);
1214 }
1215 for (MachineInstr &MI : *LoopMBB) {
1216 MI.setFlag(MachineInstr::FrameSetup);
1217 }
1218 for (MachineInstr &MI :
1219 llvm::make_range(x: ContinueMBB->begin(), y: ContinueMBBI)) {
1220 MI.setFlag(MachineInstr::FrameSetup);
1221 }
1222 }
1223}
1224
1225void X86FrameLowering::emitStackProbeCall(
1226 MachineFunction &MF, MachineBasicBlock &MBB,
1227 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog,
1228 std::optional<MachineFunction::DebugInstrOperandPair> InstrNum) const {
1229 bool IsLargeCodeModel = MF.getTarget().getCodeModel() == CodeModel::Large;
1230
1231 // FIXME: Add indirect thunk support and remove this.
1232 if (Is64Bit && IsLargeCodeModel && STI.useIndirectThunkCalls())
1233 report_fatal_error(reason: "Emitting stack probe calls on 64-bit with the large "
1234 "code model and indirect thunks not yet implemented.");
1235
1236 assert(MBB.computeRegisterLiveness(TRI, X86::EFLAGS, MBBI) !=
1237 MachineBasicBlock::LQR_Live &&
1238 "Stack probe calls will clobber live EFLAGS.");
1239
1240 unsigned CallOp;
1241 if (Is64Bit)
1242 CallOp = IsLargeCodeModel ? X86::CALL64r : X86::CALL64pcrel32;
1243 else
1244 CallOp = X86::CALLpcrel32;
1245
1246 StringRef Symbol = STI.getTargetLowering()->getStackProbeSymbolName(MF);
1247
1248 MachineInstrBuilder CI;
1249 MachineBasicBlock::iterator ExpansionMBBI = std::prev(x: MBBI);
1250
1251 // All current stack probes take AX and SP as input, clobber flags, and
1252 // preserve all registers. x86_64 probes leave RSP unmodified.
1253 if (Is64Bit && MF.getTarget().getCodeModel() == CodeModel::Large) {
1254 // For the large code model, we have to call through a register. Use R11,
1255 // as it is scratch in all supported calling conventions.
1256 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64ri), DestReg: X86::R11)
1257 .addExternalSymbol(FnName: MF.createExternalSymbolName(Name: Symbol));
1258 CI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: CallOp)).addReg(RegNo: X86::R11);
1259 } else {
1260 CI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: CallOp))
1261 .addExternalSymbol(FnName: MF.createExternalSymbolName(Name: Symbol));
1262 }
1263
1264 unsigned AX = Uses64BitFramePtr ? X86::RAX : X86::EAX;
1265 unsigned SP = Uses64BitFramePtr ? X86::RSP : X86::ESP;
1266 CI.addReg(RegNo: AX, Flags: RegState::Implicit)
1267 .addReg(RegNo: SP, Flags: RegState::Implicit)
1268 .addReg(RegNo: AX, Flags: RegState::Define | RegState::Implicit)
1269 .addReg(RegNo: SP, Flags: RegState::Define | RegState::Implicit)
1270 .addReg(RegNo: X86::EFLAGS, Flags: RegState::Define | RegState::Implicit);
1271
1272 MachineInstr *ModInst = CI;
1273 if (STI.isTargetWin64() || !STI.isOSWindows()) {
1274 // MSVC x32's _chkstk and cygwin/mingw's _alloca adjust %esp themselves.
1275 // MSVC x64's __chkstk and cygwin/mingw's ___chkstk_ms do not adjust %rsp
1276 // themselves. They also does not clobber %rax so we can reuse it when
1277 // adjusting %rsp.
1278 // All other platforms do not specify a particular ABI for the stack probe
1279 // function, so we arbitrarily define it to not adjust %esp/%rsp itself.
1280 ModInst =
1281 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: getSUBrrOpcode(IsLP64: Uses64BitFramePtr)), DestReg: SP)
1282 .addReg(RegNo: SP)
1283 .addReg(RegNo: AX);
1284 }
1285
1286 // DebugInfo variable locations -- if there's an instruction number for the
1287 // allocation (i.e., DYN_ALLOC_*), substitute it for the instruction that
1288 // modifies SP.
1289 if (InstrNum) {
1290 if (STI.isTargetWin64() || !STI.isOSWindows()) {
1291 // Label destination operand of the subtract.
1292 MF.makeDebugValueSubstitution(*InstrNum,
1293 {ModInst->getDebugInstrNum(), 0});
1294 } else {
1295 // Label the call. The operand number is the penultimate operand, zero
1296 // based.
1297 unsigned SPDefOperand = ModInst->getNumOperands() - 2;
1298 MF.makeDebugValueSubstitution(
1299 *InstrNum, {ModInst->getDebugInstrNum(), SPDefOperand});
1300 }
1301 }
1302
1303 if (InProlog) {
1304 // Apply the frame setup flag to all inserted instrs.
1305 for (++ExpansionMBBI; ExpansionMBBI != MBBI; ++ExpansionMBBI)
1306 ExpansionMBBI->setFlag(MachineInstr::FrameSetup);
1307 }
1308}
1309
1310static unsigned calculateSetFPREG(uint64_t SPAdjust) {
1311 // Win64 ABI has a less restrictive limitation of 240; 128 works equally well
1312 // and might require smaller successive adjustments.
1313 const uint64_t Win64MaxSEHOffset = 128;
1314 uint64_t SEHFrameOffset = std::min(a: SPAdjust, b: Win64MaxSEHOffset);
1315 // Win64 ABI requires 16-byte alignment for the UWOP_SET_FPREG opcode.
1316 return SEHFrameOffset & -16;
1317}
1318
1319// If we're forcing a stack realignment we can't rely on just the frame
1320// info, we need to know the ABI stack alignment as well in case we
1321// have a call out. Otherwise just make sure we have some alignment - we'll
1322// go with the minimum SlotSize.
1323uint64_t
1324X86FrameLowering::calculateMaxStackAlign(const MachineFunction &MF) const {
1325 const MachineFrameInfo &MFI = MF.getFrameInfo();
1326 Align MaxAlign = MFI.getMaxAlign(); // Desired stack alignment.
1327 Align StackAlign = getStackAlign();
1328 bool HasRealign = MF.getFunction().hasFnAttribute(Kind: "stackrealign");
1329 if (HasRealign) {
1330 if (MFI.hasCalls())
1331 MaxAlign = (StackAlign > MaxAlign) ? StackAlign : MaxAlign;
1332 else if (MaxAlign < SlotSize)
1333 MaxAlign = Align(SlotSize);
1334 }
1335
1336 if (!Is64Bit && MF.getFunction().getCallingConv() == CallingConv::X86_INTR) {
1337 if (HasRealign)
1338 MaxAlign = (MaxAlign > 16) ? MaxAlign : Align(16);
1339 else
1340 MaxAlign = Align(16);
1341 }
1342 return MaxAlign.value();
1343}
1344
1345void X86FrameLowering::BuildStackAlignAND(MachineBasicBlock &MBB,
1346 MachineBasicBlock::iterator MBBI,
1347 const DebugLoc &DL, Register Reg,
1348 uint64_t MaxAlign) const {
1349 uint64_t Val = -MaxAlign;
1350 unsigned AndOp = getANDriOpcode(IsLP64: Uses64BitFramePtr, Imm: Val);
1351
1352 MachineFunction &MF = *MBB.getParent();
1353 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
1354 const X86TargetLowering &TLI = *STI.getTargetLowering();
1355 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
1356 const bool EmitInlineStackProbe = TLI.hasInlineStackProbe(MF);
1357
1358 // We want to make sure that (in worst case) less than StackProbeSize bytes
1359 // are not probed after the AND. This assumption is used in
1360 // emitStackProbeInlineGeneric.
1361 if (Reg == StackPtr && EmitInlineStackProbe && MaxAlign >= StackProbeSize) {
1362 {
1363 NumFrameLoopProbe++;
1364 MachineBasicBlock *entryMBB =
1365 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
1366 MachineBasicBlock *headMBB =
1367 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
1368 MachineBasicBlock *bodyMBB =
1369 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
1370 MachineBasicBlock *footMBB =
1371 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
1372
1373 MachineFunction::iterator MBBIter = MBB.getIterator();
1374 MF.insert(MBBI: MBBIter, MBB: entryMBB);
1375 MF.insert(MBBI: MBBIter, MBB: headMBB);
1376 MF.insert(MBBI: MBBIter, MBB: bodyMBB);
1377 MF.insert(MBBI: MBBIter, MBB: footMBB);
1378 const unsigned MovMIOpc = Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
1379 Register FinalStackProbed = Uses64BitFramePtr ? X86::R11
1380 : Is64Bit ? X86::R11D
1381 : X86::EAX;
1382
1383 // Setup entry block
1384 {
1385
1386 entryMBB->splice(Where: entryMBB->end(), Other: &MBB, From: MBB.begin(), To: MBBI);
1387 BuildMI(BB: entryMBB, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: FinalStackProbed)
1388 .addReg(RegNo: StackPtr)
1389 .setMIFlag(MachineInstr::FrameSetup);
1390 MachineInstr *MI =
1391 BuildMI(BB: entryMBB, MIMD: DL, MCID: TII.get(Opcode: AndOp), DestReg: FinalStackProbed)
1392 .addReg(RegNo: FinalStackProbed)
1393 .addImm(Val)
1394 .setMIFlag(MachineInstr::FrameSetup);
1395
1396 // The EFLAGS implicit def is dead.
1397 MI->getOperand(i: 3).setIsDead();
1398
1399 BuildMI(BB: entryMBB, MIMD: DL,
1400 MCID: TII.get(Opcode: Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
1401 .addReg(RegNo: FinalStackProbed)
1402 .addReg(RegNo: StackPtr)
1403 .setMIFlag(MachineInstr::FrameSetup);
1404 BuildMI(BB: entryMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
1405 .addMBB(MBB: &MBB)
1406 .addImm(Val: X86::COND_E)
1407 .setMIFlag(MachineInstr::FrameSetup);
1408 entryMBB->addSuccessor(Succ: headMBB);
1409 entryMBB->addSuccessor(Succ: &MBB);
1410 }
1411
1412 // Loop entry block
1413
1414 {
1415 const unsigned SUBOpc = getSUBriOpcode(IsLP64: Uses64BitFramePtr);
1416 BuildMI(BB: headMBB, MIMD: DL, MCID: TII.get(Opcode: SUBOpc), DestReg: StackPtr)
1417 .addReg(RegNo: StackPtr)
1418 .addImm(Val: StackProbeSize)
1419 .setMIFlag(MachineInstr::FrameSetup);
1420
1421 BuildMI(BB: headMBB, MIMD: DL,
1422 MCID: TII.get(Opcode: Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
1423 .addReg(RegNo: StackPtr)
1424 .addReg(RegNo: FinalStackProbed)
1425 .setMIFlag(MachineInstr::FrameSetup);
1426
1427 // jump to the footer if StackPtr < FinalStackProbed
1428 BuildMI(BB: headMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
1429 .addMBB(MBB: footMBB)
1430 .addImm(Val: X86::COND_B)
1431 .setMIFlag(MachineInstr::FrameSetup);
1432
1433 headMBB->addSuccessor(Succ: bodyMBB);
1434 headMBB->addSuccessor(Succ: footMBB);
1435 }
1436
1437 // setup loop body
1438 {
1439 addRegOffset(MIB: BuildMI(BB: bodyMBB, MIMD: DL, MCID: TII.get(Opcode: MovMIOpc))
1440 .setMIFlag(MachineInstr::FrameSetup),
1441 Reg: StackPtr, isKill: false, Offset: 0)
1442 .addImm(Val: 0)
1443 .setMIFlag(MachineInstr::FrameSetup);
1444
1445 const unsigned SUBOpc = getSUBriOpcode(IsLP64: Uses64BitFramePtr);
1446 BuildMI(BB: bodyMBB, MIMD: DL, MCID: TII.get(Opcode: SUBOpc), DestReg: StackPtr)
1447 .addReg(RegNo: StackPtr)
1448 .addImm(Val: StackProbeSize)
1449 .setMIFlag(MachineInstr::FrameSetup);
1450
1451 // cmp with stack pointer bound
1452 BuildMI(BB: bodyMBB, MIMD: DL,
1453 MCID: TII.get(Opcode: Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
1454 .addReg(RegNo: FinalStackProbed)
1455 .addReg(RegNo: StackPtr)
1456 .setMIFlag(MachineInstr::FrameSetup);
1457
1458 // jump back while FinalStackProbed < StackPtr
1459 BuildMI(BB: bodyMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
1460 .addMBB(MBB: bodyMBB)
1461 .addImm(Val: X86::COND_B)
1462 .setMIFlag(MachineInstr::FrameSetup);
1463 bodyMBB->addSuccessor(Succ: bodyMBB);
1464 bodyMBB->addSuccessor(Succ: footMBB);
1465 }
1466
1467 // setup loop footer
1468 {
1469 BuildMI(BB: footMBB, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: StackPtr)
1470 .addReg(RegNo: FinalStackProbed)
1471 .setMIFlag(MachineInstr::FrameSetup);
1472 addRegOffset(MIB: BuildMI(BB: footMBB, MIMD: DL, MCID: TII.get(Opcode: MovMIOpc))
1473 .setMIFlag(MachineInstr::FrameSetup),
1474 Reg: StackPtr, isKill: false, Offset: 0)
1475 .addImm(Val: 0)
1476 .setMIFlag(MachineInstr::FrameSetup);
1477 footMBB->addSuccessor(Succ: &MBB);
1478 }
1479
1480 fullyRecomputeLiveIns(MBBs: {footMBB, bodyMBB, headMBB, &MBB});
1481 }
1482 } else {
1483 MachineInstr *MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AndOp), DestReg: Reg)
1484 .addReg(RegNo: Reg)
1485 .addImm(Val)
1486 .setMIFlag(MachineInstr::FrameSetup);
1487
1488 // The EFLAGS implicit def is dead.
1489 MI->getOperand(i: 3).setIsDead();
1490 }
1491}
1492
1493bool X86FrameLowering::has128ByteRedZone(const MachineFunction &MF) const {
1494 // x86-64 (non Win64) has a 128 byte red zone which is guaranteed not to be
1495 // clobbered by any interrupt handler.
1496 assert(&STI == &MF.getSubtarget<X86Subtarget>() &&
1497 "MF used frame lowering for wrong subtarget");
1498 const Function &Fn = MF.getFunction();
1499 const bool IsWin64CC = STI.isCallingConvWin64(CC: Fn.getCallingConv());
1500 return Is64Bit && !IsWin64CC && !Fn.hasFnAttribute(Kind: Attribute::NoRedZone);
1501}
1502
1503/// Return true if we need to use the restricted Windows x64 prologue and
1504/// epilogue code patterns that can be described with WinCFI (.seh_*
1505/// directives).
1506bool X86FrameLowering::isWin64Prologue(const MachineFunction &MF) const {
1507 return MF.getTarget().getMCAsmInfo().usesWindowsCFI();
1508}
1509
1510bool X86FrameLowering::needsDwarfCFI(const MachineFunction &MF) const {
1511 return !isWin64Prologue(MF) && MF.needsFrameMoves();
1512}
1513
1514/// Return true if an opcode is part of the REP group of instructions
1515static bool isOpcodeRep(unsigned Opcode) {
1516 switch (Opcode) {
1517 case X86::REPNE_PREFIX:
1518 case X86::REP_MOVSB_32:
1519 case X86::REP_MOVSB_64:
1520 case X86::REP_MOVSD_32:
1521 case X86::REP_MOVSD_64:
1522 case X86::REP_MOVSQ_32:
1523 case X86::REP_MOVSQ_64:
1524 case X86::REP_MOVSW_32:
1525 case X86::REP_MOVSW_64:
1526 case X86::REP_PREFIX:
1527 case X86::REP_STOSB_32:
1528 case X86::REP_STOSB_64:
1529 case X86::REP_STOSD_32:
1530 case X86::REP_STOSD_64:
1531 case X86::REP_STOSQ_32:
1532 case X86::REP_STOSQ_64:
1533 case X86::REP_STOSW_32:
1534 case X86::REP_STOSW_64:
1535 return true;
1536 default:
1537 break;
1538 }
1539 return false;
1540}
1541
1542/// emitPrologue - Push callee-saved registers onto the stack, which
1543/// automatically adjust the stack pointer. Adjust the stack pointer to allocate
1544/// space for local variables. Also emit labels used by the exception handler to
1545/// generate the exception handling frames.
1546
1547/*
1548 Here's a gist of what gets emitted:
1549
1550 ; Establish frame pointer, if needed
1551 [if needs FP]
1552 push %rbp
1553 .cfi_def_cfa_offset 16
1554 .cfi_offset %rbp, -16
1555 .seh_pushreg %rpb
1556 mov %rsp, %rbp
1557 .cfi_def_cfa_register %rbp
1558
1559 ; Spill general-purpose registers
1560 [for all callee-saved GPRs]
1561 pushq %<reg>
1562 [if not needs FP]
1563 .cfi_def_cfa_offset (offset from RETADDR)
1564 .seh_pushreg %<reg>
1565
1566 ; If the required stack alignment > default stack alignment
1567 ; rsp needs to be re-aligned. This creates a "re-alignment gap"
1568 ; of unknown size in the stack frame.
1569 [if stack needs re-alignment]
1570 and $MASK, %rsp
1571
1572 ; Allocate space for locals
1573 [if target is Windows and allocated space > 4096 bytes]
1574 ; Windows needs special care for allocations larger
1575 ; than one page.
1576 mov $NNN, %rax
1577 call ___chkstk_ms/___chkstk
1578 sub %rax, %rsp
1579 [else]
1580 sub $NNN, %rsp
1581
1582 [if needs FP]
1583 .seh_stackalloc (size of XMM spill slots)
1584 .seh_setframe %rbp, SEHFrameOffset ; = size of all spill slots
1585 [else]
1586 .seh_stackalloc NNN
1587
1588 ; Spill XMMs
1589 ; Note, that while only Windows 64 ABI specifies XMMs as callee-preserved,
1590 ; they may get spilled on any platform, if the current function
1591 ; calls @llvm.eh.unwind.init
1592 [if needs FP]
1593 [for all callee-saved XMM registers]
1594 movaps %<xmm reg>, -MMM(%rbp)
1595 [for all callee-saved XMM registers]
1596 .seh_savexmm %<xmm reg>, (-MMM + SEHFrameOffset)
1597 ; i.e. the offset relative to (%rbp - SEHFrameOffset)
1598 [else]
1599 [for all callee-saved XMM registers]
1600 movaps %<xmm reg>, KKK(%rsp)
1601 [for all callee-saved XMM registers]
1602 .seh_savexmm %<xmm reg>, KKK
1603
1604 .seh_endprologue
1605
1606 [if needs base pointer]
1607 mov %rsp, %rbx
1608 [if needs to restore base pointer]
1609 mov %rsp, -MMM(%rbp)
1610
1611 ; Emit CFI info
1612 [if needs FP]
1613 [for all callee-saved registers]
1614 .cfi_offset %<reg>, (offset from %rbp)
1615 [else]
1616 .cfi_def_cfa_offset (offset from RETADDR)
1617 [for all callee-saved registers]
1618 .cfi_offset %<reg>, (offset from %rsp)
1619
1620 Notes:
1621 - .seh directives are emitted only for Windows 64 ABI
1622 - .cv_fpo directives are emitted on win32 when emitting CodeView
1623 - .cfi directives are emitted for all other ABIs
1624 - for 32-bit code, substitute %e?? registers for %r??
1625*/
1626
1627void X86FrameLowering::emitPrologue(MachineFunction &MF,
1628 MachineBasicBlock &MBB) const {
1629 assert(&STI == &MF.getSubtarget<X86Subtarget>() &&
1630 "MF used frame lowering for wrong subtarget");
1631 MachineBasicBlock::iterator MBBI = MBB.begin();
1632 MachineFrameInfo &MFI = MF.getFrameInfo();
1633 const Function &Fn = MF.getFunction();
1634 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
1635 uint64_t MaxAlign = calculateMaxStackAlign(MF); // Desired stack alignment.
1636 uint64_t StackSize = MFI.getStackSize(); // Number of bytes to allocate.
1637 bool IsFunclet = MBB.isEHFuncletEntry();
1638 EHPersonality Personality = EHPersonality::Unknown;
1639 if (Fn.hasPersonalityFn())
1640 Personality = classifyEHPersonality(Pers: Fn.getPersonalityFn());
1641 bool FnHasClrFunclet =
1642 MF.hasEHFunclets() && Personality == EHPersonality::CoreCLR;
1643 bool IsClrFunclet = IsFunclet && FnHasClrFunclet;
1644 bool HasFP = hasFP(MF);
1645 bool IsWin64Prologue = isWin64Prologue(MF);
1646 bool NeedsWin64CFI = IsWin64Prologue && Fn.needsUnwindTableEntry();
1647 // FIXME: Emit FPO data for EH funclets.
1648 bool NeedsWinFPO = !IsFunclet && STI.isTargetWin32() &&
1649 MF.getFunction().getParent()->getCodeViewFlag();
1650 bool NeedsWinCFI = NeedsWin64CFI || NeedsWinFPO;
1651 bool NeedsDwarfCFI = needsDwarfCFI(MF);
1652 bool IsWin64UnwindV3 = NeedsWin64CFI && requireWinX64UnwindV3(MF);
1653 Register FramePtr = TRI->getFrameRegister(MF);
1654 const Register MachineFramePtr =
1655 STI.isTarget64BitILP32() ? Register(getX86SubSuperRegister(Reg: FramePtr, Size: 64))
1656 : FramePtr;
1657 Register BasePtr = TRI->getBaseRegister();
1658 bool HasWinCFI = false;
1659
1660 // Helpers to emit Windows x64 unwind SEH pseudos with the correct placement.
1661 // V1/V2: pseudo goes after the real instruction.
1662 // V3: pseudo goes before the real instruction.
1663 // Usage:
1664 // EmitSEHBefore([&]{ BuildMI(...SEH_PushReg...); });
1665 // BuildMI(... real instruction ...);
1666 // EmitSEHAfter([&]{ BuildMI(...SEH_PushReg...); });
1667 auto EmitSEHBefore = [&](auto EmitFn) {
1668 if (NeedsWinCFI && IsWin64UnwindV3) {
1669 HasWinCFI = true;
1670 EmitFn();
1671 }
1672 };
1673 auto EmitSEHAfter = [&](auto EmitFn) {
1674 if (NeedsWinCFI && !IsWin64UnwindV3) {
1675 HasWinCFI = true;
1676 EmitFn();
1677 }
1678 };
1679
1680 // Debug location must be unknown since the first debug location is used
1681 // to determine the end of the prologue.
1682 DebugLoc DL;
1683 Register ArgBaseReg;
1684
1685 // Emit extra prolog for argument stack slot reference.
1686 if (auto *MI = X86FI->getStackPtrSaveMI()) {
1687 // MI is lea instruction that created in X86ArgumentStackSlotPass.
1688 // Creat extra prolog for stack realignment.
1689 ArgBaseReg = MI->getOperand(i: 0).getReg();
1690 // leal 4(%esp), %basereg
1691 // .cfi_def_cfa %basereg, 0
1692 // andl $-128, %esp
1693 // pushl -4(%basereg)
1694 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Is64Bit ? X86::LEA64r : X86::LEA32r),
1695 DestReg: ArgBaseReg)
1696 .addUse(RegNo: StackPtr)
1697 .addImm(Val: 1)
1698 .addUse(RegNo: X86::NoRegister)
1699 .addImm(Val: SlotSize)
1700 .addUse(RegNo: X86::NoRegister)
1701 .setMIFlag(MachineInstr::FrameSetup);
1702 if (NeedsDwarfCFI) {
1703 // .cfi_def_cfa %basereg, 0
1704 unsigned DwarfStackPtr = TRI->getDwarfRegNum(Reg: ArgBaseReg, isEH: true);
1705 BuildCFI(MBB, MBBI, DL,
1706 CFIInst: MCCFIInstruction::cfiDefCfa(L: nullptr, Register: DwarfStackPtr, Offset: 0),
1707 Flag: MachineInstr::FrameSetup);
1708 }
1709 BuildStackAlignAND(MBB, MBBI, DL, Reg: StackPtr, MaxAlign);
1710 int64_t Offset = -(int64_t)SlotSize;
1711 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Is64Bit ? X86::PUSH64rmm : X86::PUSH32rmm))
1712 .addReg(RegNo: ArgBaseReg)
1713 .addImm(Val: 1)
1714 .addReg(RegNo: X86::NoRegister)
1715 .addImm(Val: Offset)
1716 .addReg(RegNo: X86::NoRegister)
1717 .setMIFlag(MachineInstr::FrameSetup);
1718 }
1719
1720 // Space reserved for stack-based arguments when making a (ABI-guaranteed)
1721 // tail call.
1722 unsigned TailCallArgReserveSize = -X86FI->getTCReturnAddrDelta();
1723 if (TailCallArgReserveSize && IsWin64Prologue)
1724 report_fatal_error(reason: "Can't handle guaranteed tail call under win64 yet");
1725
1726 const bool EmitStackProbeCall =
1727 STI.getTargetLowering()->hasStackProbeSymbol(MF);
1728 unsigned StackProbeSize = STI.getTargetLowering()->getStackProbeSize(MF);
1729
1730 if (HasFP && X86FI->hasSwiftAsyncContext()) {
1731 switch (MF.getTarget().Options.SwiftAsyncFramePointer) {
1732 case SwiftAsyncFramePointerMode::DeploymentBased:
1733 if (STI.swiftAsyncContextIsDynamicallySet()) {
1734 // The special symbol below is absolute and has a *value* suitable to be
1735 // combined with the frame pointer directly.
1736 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::OR64rm), DestReg: MachineFramePtr)
1737 .addUse(RegNo: MachineFramePtr)
1738 .addUse(RegNo: X86::RIP)
1739 .addImm(Val: 1)
1740 .addUse(RegNo: X86::NoRegister)
1741 .addExternalSymbol(FnName: "swift_async_extendedFramePointerFlags",
1742 TargetFlags: X86II::MO_GOTPCREL)
1743 .addUse(RegNo: X86::NoRegister);
1744 break;
1745 }
1746 [[fallthrough]];
1747
1748 case SwiftAsyncFramePointerMode::Always:
1749 assert(
1750 !IsWin64Prologue &&
1751 "win64 prologue does not set the bit 60 in the saved frame pointer");
1752 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::BTS64ri8), DestReg: MachineFramePtr)
1753 .addUse(RegNo: MachineFramePtr)
1754 .addImm(Val: 60)
1755 .setMIFlag(MachineInstr::FrameSetup);
1756 break;
1757
1758 case SwiftAsyncFramePointerMode::Never:
1759 break;
1760 }
1761 }
1762
1763 // Re-align the stack on 64-bit if the x86-interrupt calling convention is
1764 // used and an error code was pushed, since the x86-64 ABI requires a 16-byte
1765 // stack alignment.
1766 if (Fn.getCallingConv() == CallingConv::X86_INTR && Is64Bit &&
1767 Fn.arg_size() == 2) {
1768 StackSize += 8;
1769 MFI.setStackSize(StackSize);
1770
1771 // Update the stack pointer by pushing a register. This is the instruction
1772 // emitted that would be end up being emitted by a call to `emitSPUpdate`.
1773 // Hard-coding the update to a push avoids emitting a second
1774 // `STACKALLOC_W_PROBING` instruction in the save block: We know that stack
1775 // probing isn't needed anyways for an 8-byte update.
1776 // Pushing a register leaves us in a similar situation to a regular
1777 // function call where we know that the address at (rsp-8) is writeable.
1778 // That way we avoid any off-by-ones with stack probing for additional
1779 // stack pointer updates later on.
1780 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH64r))
1781 .addReg(RegNo: X86::RAX, Flags: RegState::Undef)
1782 .setMIFlag(MachineInstr::FrameSetup);
1783 }
1784
1785 // If this is x86-64 and the Red Zone is not disabled, if we are a leaf
1786 // function, and use up to 128 bytes of stack space, don't have a frame
1787 // pointer, calls, or dynamic alloca then we do not need to adjust the
1788 // stack pointer (we fit in the Red Zone). We also check that we don't
1789 // push and pop from the stack.
1790 if (has128ByteRedZone(MF) && !TRI->hasStackRealignment(MF) &&
1791 !MFI.hasVarSizedObjects() && // No dynamic alloca.
1792 !MFI.adjustsStack() && // No calls.
1793 !EmitStackProbeCall && // No stack probes.
1794 !MFI.hasCopyImplyingStackAdjustment() && // Don't push and pop.
1795 !MF.shouldSplitStack()) { // Regular stack
1796 uint64_t MinSize =
1797 X86FI->getCalleeSavedFrameSize() - X86FI->getTCReturnAddrDelta();
1798 if (HasFP)
1799 MinSize += SlotSize;
1800 X86FI->setUsesRedZone(MinSize > 0 || StackSize > 0);
1801 StackSize = std::max(a: MinSize, b: StackSize > 128 ? StackSize - 128 : 0);
1802 MFI.setStackSize(StackSize);
1803 }
1804
1805 // Insert stack pointer adjustment for later moving of return addr. Only
1806 // applies to tail call optimized functions where the callee argument stack
1807 // size is bigger than the callers.
1808 if (TailCallArgReserveSize != 0) {
1809 BuildStackAdjustment(MBB, MBBI, DL, Offset: -(int)TailCallArgReserveSize,
1810 /*InEpilogue=*/false)
1811 .setMIFlag(MachineInstr::FrameSetup);
1812 }
1813
1814 // Mapping for machine moves:
1815 //
1816 // DST: VirtualFP AND
1817 // SRC: VirtualFP => DW_CFA_def_cfa_offset
1818 // ELSE => DW_CFA_def_cfa
1819 //
1820 // SRC: VirtualFP AND
1821 // DST: Register => DW_CFA_def_cfa_register
1822 //
1823 // ELSE
1824 // OFFSET < 0 => DW_CFA_offset_extended_sf
1825 // REG < 64 => DW_CFA_offset + Reg
1826 // ELSE => DW_CFA_offset_extended
1827
1828 uint64_t NumBytes = 0;
1829 int stackGrowth = -SlotSize;
1830
1831 // Find the funclet establisher parameter
1832 MCRegister Establisher;
1833 if (IsClrFunclet)
1834 Establisher = Uses64BitFramePtr ? X86::RCX : X86::ECX;
1835 else if (IsFunclet)
1836 Establisher = Uses64BitFramePtr ? X86::RDX : X86::EDX;
1837
1838 if (IsWin64Prologue && IsFunclet && !IsClrFunclet) {
1839 // Immediately spill establisher into the home slot.
1840 // The runtime cares about this.
1841 // MOV64mr %rdx, 16(%rsp)
1842 unsigned MOVmr = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr;
1843 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: MOVmr)), Reg: StackPtr, isKill: true, Offset: 16)
1844 .addReg(RegNo: Establisher)
1845 .setMIFlag(MachineInstr::FrameSetup);
1846 MBB.addLiveIn(PhysReg: Establisher);
1847 }
1848
1849 if (HasFP) {
1850 assert(MF.getRegInfo().isReserved(MachineFramePtr) && "FP reserved");
1851
1852 // Calculate required stack adjustment.
1853 uint64_t FrameSize = StackSize - SlotSize;
1854 NumBytes =
1855 FrameSize - (X86FI->getCalleeSavedFrameSize() + TailCallArgReserveSize);
1856
1857 // Callee-saved registers are pushed on stack before the stack is realigned.
1858 if (TRI->hasStackRealignment(MF) && !IsWin64Prologue)
1859 NumBytes = alignTo(Value: NumBytes, Align: MaxAlign);
1860
1861 // Save EBP/RBP into the appropriate stack slot.
1862 auto EmitSEHPushFramePtr = [&]() {
1863 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_PushReg))
1864 .addImm(Val: FramePtr)
1865 .setMIFlag(MachineInstr::FrameSetup);
1866 };
1867 EmitSEHBefore(EmitSEHPushFramePtr);
1868 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
1869 MCID: TII.get(Opcode: getPUSHOpcode(ST: MF.getSubtarget<X86Subtarget>())))
1870 .addReg(RegNo: MachineFramePtr, Flags: RegState::Kill)
1871 .setMIFlag(MachineInstr::FrameSetup);
1872 EmitSEHAfter(EmitSEHPushFramePtr);
1873
1874 if (NeedsDwarfCFI && !ArgBaseReg.isValid()) {
1875 // Mark the place where EBP/RBP was saved.
1876 // Define the current CFA rule to use the provided offset.
1877 assert(StackSize);
1878 BuildCFI(MBB, MBBI, DL,
1879 CFIInst: MCCFIInstruction::cfiDefCfaOffset(
1880 L: nullptr, Offset: -2 * stackGrowth + (int)TailCallArgReserveSize),
1881 Flag: MachineInstr::FrameSetup);
1882
1883 // Change the rule for the FramePtr to be an "offset" rule.
1884 unsigned DwarfFramePtr = TRI->getDwarfRegNum(Reg: MachineFramePtr, isEH: true);
1885 BuildCFI(MBB, MBBI, DL,
1886 CFIInst: MCCFIInstruction::createOffset(L: nullptr, Register: DwarfFramePtr,
1887 Offset: 2 * stackGrowth -
1888 (int)TailCallArgReserveSize),
1889 Flag: MachineInstr::FrameSetup);
1890 }
1891
1892 if (!IsFunclet) {
1893 if (X86FI->hasSwiftAsyncContext()) {
1894 assert(!IsWin64Prologue &&
1895 "win64 prologue does not store async context right below rbp");
1896 const auto &Attrs = MF.getFunction().getAttributes();
1897
1898 // Before we update the live frame pointer we have to ensure there's a
1899 // valid (or null) asynchronous context in its slot just before FP in
1900 // the frame record, so store it now.
1901 auto EmitSEHPushR14 = [&]() {
1902 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_PushReg))
1903 .addImm(Val: X86::R14)
1904 .setMIFlag(MachineInstr::FrameSetup);
1905 };
1906 EmitSEHBefore(EmitSEHPushR14);
1907 if (Attrs.hasAttrSomewhere(Kind: Attribute::SwiftAsync)) {
1908 // We have an initial context in r14, store it just before the frame
1909 // pointer.
1910 MBB.addLiveIn(PhysReg: X86::R14);
1911 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH64r))
1912 .addReg(RegNo: X86::R14)
1913 .setMIFlag(MachineInstr::FrameSetup);
1914 } else {
1915 // No initial context, store null so that there's no pointer that
1916 // could be misused.
1917 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH64i32))
1918 .addImm(Val: 0)
1919 .setMIFlag(MachineInstr::FrameSetup);
1920 }
1921
1922 // Update CFA offset for the async-context push.
1923 if (NeedsDwarfCFI && !ArgBaseReg.isValid()) {
1924 BuildCFI(
1925 MBB, MBBI, DL,
1926 CFIInst: MCCFIInstruction::createAdjustCfaOffset(L: nullptr, Adjustment: -stackGrowth),
1927 Flag: MachineInstr::FrameSetup);
1928 }
1929
1930 EmitSEHAfter(EmitSEHPushR14);
1931
1932 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::LEA64r), DestReg: FramePtr)
1933 .addUse(RegNo: X86::RSP)
1934 .addImm(Val: 1)
1935 .addUse(RegNo: X86::NoRegister)
1936 .addImm(Val: 8)
1937 .addUse(RegNo: X86::NoRegister)
1938 .setMIFlag(MachineInstr::FrameSetup);
1939
1940 // Switch to an FP-relative CFA before adjusting RSP below.
1941 if (NeedsDwarfCFI && !ArgBaseReg.isValid()) {
1942 unsigned DwarfFramePtr = TRI->getDwarfRegNum(Reg: MachineFramePtr, isEH: true);
1943 BuildCFI(MBB, MBBI, DL,
1944 CFIInst: MCCFIInstruction::cfiDefCfa(L: nullptr, Register: DwarfFramePtr,
1945 Offset: -2 * stackGrowth +
1946 (int)TailCallArgReserveSize),
1947 Flag: MachineInstr::FrameSetup);
1948 }
1949
1950 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SUB64ri32), DestReg: X86::RSP)
1951 .addUse(RegNo: X86::RSP)
1952 .addImm(Val: 8)
1953 .setMIFlag(MachineInstr::FrameSetup);
1954 }
1955
1956 if (!IsWin64Prologue && !IsFunclet) {
1957 // Update EBP with the new base value.
1958 if (!X86FI->hasSwiftAsyncContext())
1959 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
1960 MCID: TII.get(Opcode: Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr),
1961 DestReg: FramePtr)
1962 .addReg(RegNo: StackPtr)
1963 .setMIFlag(MachineInstr::FrameSetup);
1964
1965 if (NeedsDwarfCFI) {
1966 if (ArgBaseReg.isValid()) {
1967 SmallString<64> CfaExpr;
1968 CfaExpr.push_back(Elt: dwarf::DW_CFA_expression);
1969 uint8_t buffer[16];
1970 unsigned DwarfReg = TRI->getDwarfRegNum(Reg: MachineFramePtr, isEH: true);
1971 CfaExpr.append(in_start: buffer, in_end: buffer + encodeULEB128(Value: DwarfReg, p: buffer));
1972 CfaExpr.push_back(Elt: 2);
1973 CfaExpr.push_back(Elt: (uint8_t)(dwarf::DW_OP_breg0 + DwarfReg));
1974 CfaExpr.push_back(Elt: 0);
1975 // DW_CFA_expression: reg5 DW_OP_breg5 +0
1976 BuildCFI(MBB, MBBI, DL,
1977 CFIInst: MCCFIInstruction::createEscape(L: nullptr, Vals: CfaExpr.str()),
1978 Flag: MachineInstr::FrameSetup);
1979 } else if (!X86FI->hasSwiftAsyncContext()) {
1980 // Mark effective beginning of when frame pointer becomes valid.
1981 // Define the current CFA to use the EBP/RBP register.
1982 unsigned DwarfFramePtr = TRI->getDwarfRegNum(Reg: MachineFramePtr, isEH: true);
1983 BuildCFI(
1984 MBB, MBBI, DL,
1985 CFIInst: MCCFIInstruction::createDefCfaRegister(L: nullptr, Register: DwarfFramePtr),
1986 Flag: MachineInstr::FrameSetup);
1987 }
1988 }
1989
1990 if (NeedsWinFPO) {
1991 // .cv_fpo_setframe $FramePtr
1992 // NeedsWinFPO is Win32 only, so we're never using Unwind v3, hence it
1993 // is always inserted afterwards.
1994 assert(!IsWin64UnwindV3);
1995 HasWinCFI = true;
1996 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_SetFrame))
1997 .addImm(Val: FramePtr)
1998 .addImm(Val: 0)
1999 .setMIFlag(MachineInstr::FrameSetup);
2000 }
2001 }
2002 }
2003 } else {
2004 assert(!IsFunclet && "funclets without FPs not yet implemented");
2005 NumBytes =
2006 StackSize - (X86FI->getCalleeSavedFrameSize() + TailCallArgReserveSize);
2007 }
2008
2009 // Update the offset adjustment, which is mainly used by codeview to translate
2010 // from ESP to VFRAME relative local variable offsets.
2011 if (!IsFunclet) {
2012 if (HasFP && TRI->hasStackRealignment(MF))
2013 MFI.setOffsetAdjustment(-NumBytes);
2014 else
2015 MFI.setOffsetAdjustment(-StackSize);
2016 }
2017
2018 // For EH funclets, only allocate enough space for outgoing calls. Save the
2019 // NumBytes value that we would've used for the parent frame.
2020 unsigned ParentFrameNumBytes = NumBytes;
2021 if (IsFunclet)
2022 NumBytes = getWinEHFuncletFrameSize(MF);
2023
2024 // Skip the callee-saved push instructions.
2025 bool PushedRegs = false;
2026 int StackOffset = 2 * stackGrowth;
2027 MachineBasicBlock::const_iterator LastCSPush = MBBI;
2028 auto IsCSPush = [&](const MachineBasicBlock::iterator &MBBI) {
2029 if (MBBI == MBB.end() || !MBBI->getFlag(Flag: MachineInstr::FrameSetup))
2030 return false;
2031 unsigned Opc = MBBI->getOpcode();
2032 return Opc == X86::PUSH32r || Opc == X86::PUSH64r || Opc == X86::PUSHP64r ||
2033 Opc == X86::PUSH2 || Opc == X86::PUSH2P;
2034 };
2035
2036 while (IsCSPush(MBBI)) {
2037 PushedRegs = true;
2038 Register Reg = MBBI->getOperand(i: 0).getReg();
2039 LastCSPush = MBBI;
2040 unsigned Opc = LastCSPush->getOpcode();
2041 bool IsPush2 = Opc == X86::PUSH2 || Opc == X86::PUSH2P;
2042
2043 // V3: emit SEH pseudo before the real instruction.
2044 EmitSEHBefore([&]() {
2045 if (IsPush2) {
2046 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_Push2Regs))
2047 .addImm(Val: Reg)
2048 .addImm(Val: LastCSPush->getOperand(i: 1).getReg())
2049 .setMIFlag(MachineInstr::FrameSetup);
2050 } else {
2051 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_PushReg))
2052 .addImm(Val: Reg)
2053 .setMIFlag(MachineInstr::FrameSetup);
2054 }
2055 });
2056 ++MBBI;
2057
2058 if (!HasFP && NeedsDwarfCFI) {
2059 // Mark callee-saved push instruction.
2060 // Define the current CFA rule to use the provided offset.
2061 assert(StackSize);
2062 // Compared to push, push2 introduces more stack offset (one more
2063 // register).
2064 if (IsPush2)
2065 StackOffset += stackGrowth;
2066 BuildCFI(MBB, MBBI, DL,
2067 CFIInst: MCCFIInstruction::cfiDefCfaOffset(L: nullptr, Offset: -StackOffset),
2068 Flag: MachineInstr::FrameSetup);
2069 StackOffset += stackGrowth;
2070 }
2071
2072 // V1/V2: emit SEH pseudo after the real instruction.
2073 EmitSEHAfter([&]() {
2074 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_PushReg))
2075 .addImm(Val: Reg)
2076 .setMIFlag(MachineInstr::FrameSetup);
2077 if (IsPush2)
2078 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_PushReg))
2079 .addImm(Val: LastCSPush->getOperand(i: 1).getReg())
2080 .setMIFlag(MachineInstr::FrameSetup);
2081 });
2082 }
2083
2084 // Realign stack after we pushed callee-saved registers (so that we'll be
2085 // able to calculate their offsets from the frame pointer).
2086 // Don't do this for Win64, it needs to realign the stack after the prologue.
2087 if (!IsWin64Prologue && !IsFunclet && TRI->hasStackRealignment(MF) &&
2088 !ArgBaseReg.isValid()) {
2089 assert(HasFP && "There should be a frame pointer if stack is realigned.");
2090 auto EmitSEHStackAlign = [&]() {
2091 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_StackAlign))
2092 .addImm(Val: MaxAlign)
2093 .setMIFlag(MachineInstr::FrameSetup);
2094 };
2095 EmitSEHBefore(EmitSEHStackAlign);
2096 BuildStackAlignAND(MBB, MBBI, DL, Reg: StackPtr, MaxAlign);
2097 EmitSEHAfter(EmitSEHStackAlign);
2098 }
2099
2100 // If there is an SUB32ri of ESP immediately before this instruction, merge
2101 // the two. This can be the case when tail call elimination is enabled and
2102 // the callee has more arguments than the caller.
2103 NumBytes = mergeSPUpdates(
2104 MBB, MBBI, CalcNewOffset: [NumBytes](int64_t Offset) { return NumBytes - Offset; },
2105 doMergeWithPrevious: true);
2106
2107 // Adjust stack pointer: ESP -= numbytes.
2108
2109 // Windows and cygwin/mingw require a prologue helper routine when allocating
2110 // more than 4K bytes on the stack. Windows uses __chkstk and cygwin/mingw
2111 // uses __alloca. __alloca and the 32-bit version of __chkstk will probe the
2112 // stack and adjust the stack pointer in one go. The 64-bit version of
2113 // __chkstk is only responsible for probing the stack. The 64-bit prologue is
2114 // responsible for adjusting the stack pointer. Touching the stack at 4K
2115 // increments is necessary to ensure that the guard pages used by the OS
2116 // virtual memory manager are allocated in correct sequence.
2117 uint64_t AlignedNumBytes = NumBytes;
2118 if (IsWin64Prologue && !IsFunclet && TRI->hasStackRealignment(MF))
2119 AlignedNumBytes = alignTo(Value: AlignedNumBytes, Align: MaxAlign);
2120
2121 auto EmitSEHStackAlloc = [&]() {
2122 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_StackAlloc))
2123 .addImm(Val: NumBytes)
2124 .setMIFlag(MachineInstr::FrameSetup);
2125 };
2126 if (NumBytes)
2127 EmitSEHBefore(EmitSEHStackAlloc);
2128
2129 if (AlignedNumBytes >= StackProbeSize && EmitStackProbeCall) {
2130 assert(!X86FI->getUsesRedZone() &&
2131 "The Red Zone is not accounted for in stack probes");
2132
2133 // Check whether EAX is livein for this block.
2134 bool isEAXAlive = isEAXLiveIn(MBB);
2135
2136 if (isEAXAlive) {
2137 if (Is64Bit) {
2138 // Save RAX
2139 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH64r))
2140 .addReg(RegNo: X86::RAX, Flags: RegState::Kill)
2141 .setMIFlag(MachineInstr::FrameSetup);
2142 } else {
2143 // Save EAX
2144 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH32r))
2145 .addReg(RegNo: X86::EAX, Flags: RegState::Kill)
2146 .setMIFlag(MachineInstr::FrameSetup);
2147 }
2148 }
2149
2150 if (Is64Bit) {
2151 // Handle the 64-bit Windows ABI case where we need to call __chkstk.
2152 // Function prologue is responsible for adjusting the stack pointer.
2153 int64_t Alloc = isEAXAlive ? NumBytes - 8 : NumBytes;
2154 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::getMOVriOpcode(Use64BitReg: Is64Bit, Imm: Alloc)),
2155 DestReg: X86::RAX)
2156 .addImm(Val: Alloc)
2157 .setMIFlag(MachineInstr::FrameSetup);
2158 } else {
2159 // Allocate NumBytes-4 bytes on stack in case of isEAXAlive.
2160 // We'll also use 4 already allocated bytes for EAX.
2161 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV32ri), DestReg: X86::EAX)
2162 .addImm(Val: isEAXAlive ? NumBytes - 4 : NumBytes)
2163 .setMIFlag(MachineInstr::FrameSetup);
2164 }
2165
2166 // Call __chkstk, __chkstk_ms, or __alloca.
2167 emitStackProbe(MF, MBB, MBBI, DL, InProlog: true);
2168
2169 if (isEAXAlive) {
2170 // Restore RAX/EAX
2171 MachineInstr *MI;
2172 if (Is64Bit)
2173 MI = addRegOffset(MIB: BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64rm), DestReg: X86::RAX),
2174 Reg: StackPtr, isKill: false, Offset: NumBytes - 8);
2175 else
2176 MI = addRegOffset(MIB: BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: X86::MOV32rm), DestReg: X86::EAX),
2177 Reg: StackPtr, isKill: false, Offset: NumBytes - 4);
2178 MI->setFlag(MachineInstr::FrameSetup);
2179 MBB.insert(I: MBBI, MI);
2180 }
2181 } else if (NumBytes) {
2182 emitSPUpdate(MBB, MBBI, DL, NumBytes: -(int64_t)NumBytes, /*InEpilogue=*/false);
2183 }
2184
2185 if (NumBytes)
2186 EmitSEHAfter(EmitSEHStackAlloc);
2187
2188 int SEHFrameOffset = 0;
2189 Register SPOrEstablisher;
2190 if (IsFunclet) {
2191 if (IsClrFunclet) {
2192 // The establisher parameter passed to a CLR funclet is actually a pointer
2193 // to the (mostly empty) frame of its nearest enclosing funclet; we have
2194 // to find the root function establisher frame by loading the PSPSym from
2195 // the intermediate frame.
2196 unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF);
2197 MachinePointerInfo NoInfo;
2198 MBB.addLiveIn(PhysReg: Establisher);
2199 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64rm), DestReg: Establisher),
2200 Reg: Establisher, isKill: false, Offset: PSPSlotOffset)
2201 .addMemOperand(MMO: MF.getMachineMemOperand(
2202 PtrInfo: NoInfo, F: MachineMemOperand::MOLoad, Size: SlotSize, BaseAlignment: Align(SlotSize)));
2203 ;
2204 // Save the root establisher back into the current funclet's (mostly
2205 // empty) frame, in case a sub-funclet or the GC needs it.
2206 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64mr)), Reg: StackPtr,
2207 isKill: false, Offset: PSPSlotOffset)
2208 .addReg(RegNo: Establisher)
2209 .addMemOperand(MMO: MF.getMachineMemOperand(
2210 PtrInfo: NoInfo,
2211 F: MachineMemOperand::MOStore | MachineMemOperand::MOVolatile,
2212 Size: SlotSize, BaseAlignment: Align(SlotSize)));
2213 }
2214 SPOrEstablisher = Establisher;
2215 } else {
2216 SPOrEstablisher = StackPtr;
2217 }
2218
2219 if (IsWin64Prologue && HasFP) {
2220 // Set RBP to a small fixed offset from RSP. In the funclet case, we base
2221 // this calculation on the incoming establisher, which holds the value of
2222 // RSP from the parent frame at the end of the prologue.
2223 SEHFrameOffset = calculateSetFPREG(SPAdjust: ParentFrameNumBytes);
2224
2225 // If this is not a funclet, emit the CFI describing our frame pointer.
2226 if (NeedsWinCFI && !IsFunclet) {
2227 assert(!NeedsWinFPO && "this setframe incompatible with FPO data");
2228 HasWinCFI = true;
2229 if (isAsynchronousEHPersonality(Pers: Personality) || MF.hasEHFunclets()) {
2230 if (TRI->hasBasePointer(MF))
2231 MF.getWinEHFuncInfo()->SEHSetFrameOffset =
2232 getWinEHParentFrameOffset(MF);
2233 else
2234 MF.getWinEHFuncInfo()->SEHSetFrameOffset = SEHFrameOffset;
2235 }
2236 }
2237
2238 auto EmitSEHSetFrame = [&]() {
2239 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_SetFrame))
2240 .addImm(Val: FramePtr)
2241 .addImm(Val: SEHFrameOffset)
2242 .setMIFlag(MachineInstr::FrameSetup);
2243 };
2244
2245 if (!IsFunclet)
2246 EmitSEHBefore(EmitSEHSetFrame);
2247
2248 if (SEHFrameOffset)
2249 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::LEA64r), DestReg: FramePtr),
2250 Reg: SPOrEstablisher, isKill: false, Offset: SEHFrameOffset);
2251 else
2252 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64rr), DestReg: FramePtr)
2253 .addReg(RegNo: SPOrEstablisher);
2254
2255 if (!IsFunclet)
2256 EmitSEHAfter(EmitSEHSetFrame);
2257 } else if (IsFunclet && STI.is32Bit()) {
2258 // Reset EBP / ESI to something good for funclets.
2259 MBBI = restoreWin32EHStackPointers(MBB, MBBI, DL);
2260 // If we're a catch funclet, we can be returned to via catchret. Save ESP
2261 // into the registration node so that the runtime will restore it for us.
2262 if (!MBB.isCleanupFuncletEntry()) {
2263 assert(Personality == EHPersonality::MSVC_CXX);
2264 Register FrameReg;
2265 int FI = MF.getWinEHFuncInfo()->EHRegNodeFrameIndex;
2266 int64_t EHRegOffset = getFrameIndexReference(MF, FI, FrameReg).getFixed();
2267 // ESP is the first field, so no extra displacement is needed.
2268 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV32mr)), Reg: FrameReg,
2269 isKill: false, Offset: EHRegOffset)
2270 .addReg(RegNo: X86::ESP);
2271 }
2272 }
2273
2274 while (MBBI != MBB.end() && MBBI->getFlag(Flag: MachineInstr::FrameSetup)) {
2275 const MachineInstr &FrameInstr = *MBBI;
2276
2277 if (NeedsWinCFI) {
2278 int FI;
2279 if (Register Reg = TII.isStoreToStackSlot(MI: FrameInstr, FrameIndex&: FI)) {
2280 if (X86::FR64RegClass.contains(Reg)) {
2281 int Offset;
2282 Register IgnoredFrameReg;
2283 if (IsWin64Prologue && IsFunclet)
2284 Offset = getWin64EHFrameIndexRef(MF, FI, SPReg&: IgnoredFrameReg);
2285 else
2286 Offset =
2287 getFrameIndexReference(MF, FI, FrameReg&: IgnoredFrameReg).getFixed() +
2288 SEHFrameOffset;
2289
2290 assert(!NeedsWinFPO && "SEH_SaveXMM incompatible with FPO data");
2291 auto EmitSEHSaveXMM = [&]() {
2292 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_SaveXMM))
2293 .addImm(Val: Reg)
2294 .addImm(Val: Offset)
2295 .setMIFlag(MachineInstr::FrameSetup);
2296 };
2297 EmitSEHBefore(EmitSEHSaveXMM);
2298 ++MBBI;
2299 EmitSEHAfter(EmitSEHSaveXMM);
2300 continue;
2301 }
2302 }
2303 }
2304 ++MBBI;
2305 }
2306
2307 if (NeedsWinCFI && HasWinCFI) {
2308 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_EndPrologue))
2309 .setMIFlag(MachineInstr::FrameSetup);
2310 }
2311
2312 if (FnHasClrFunclet && !IsFunclet) {
2313 // Save the so-called Initial-SP (i.e. the value of the stack pointer
2314 // immediately after the prolog) into the PSPSlot so that funclets
2315 // and the GC can recover it.
2316 unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF);
2317 auto PSPInfo = MachinePointerInfo::getFixedStack(
2318 MF, FI: MF.getWinEHFuncInfo()->PSPSymFrameIdx);
2319 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64mr)), Reg: StackPtr, isKill: false,
2320 Offset: PSPSlotOffset)
2321 .addReg(RegNo: StackPtr)
2322 .addMemOperand(MMO: MF.getMachineMemOperand(
2323 PtrInfo: PSPInfo, F: MachineMemOperand::MOStore | MachineMemOperand::MOVolatile,
2324 Size: SlotSize, BaseAlignment: Align(SlotSize)));
2325 }
2326
2327 // Realign stack after we spilled callee-saved registers (so that we'll be
2328 // able to calculate their offsets from the frame pointer).
2329 // Win64 requires aligning the stack after the prologue.
2330 if (IsWin64Prologue && TRI->hasStackRealignment(MF)) {
2331 assert(HasFP && "There should be a frame pointer if stack is realigned.");
2332 BuildStackAlignAND(MBB, MBBI, DL, Reg: SPOrEstablisher, MaxAlign);
2333 }
2334
2335 // We already dealt with stack realignment and funclets above.
2336 if (IsFunclet && STI.is32Bit())
2337 return;
2338
2339 // If we need a base pointer, set it up here. It's whatever the value
2340 // of the stack pointer is at this point. Any variable size objects
2341 // will be allocated after this, so we can still use the base pointer
2342 // to reference locals.
2343 if (TRI->hasBasePointer(MF)) {
2344 // Update the base pointer with the current stack pointer.
2345 unsigned Opc = Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr;
2346 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: BasePtr)
2347 .addReg(RegNo: SPOrEstablisher)
2348 .setMIFlag(MachineInstr::FrameSetup);
2349 if (X86FI->getRestoreBasePointer()) {
2350 // Stash value of base pointer. Saving RSP instead of EBP shortens
2351 // dependence chain. Used by SjLj EH.
2352 unsigned Opm = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr;
2353 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opm)), Reg: FramePtr, isKill: true,
2354 Offset: X86FI->getRestoreBasePointerOffset())
2355 .addReg(RegNo: SPOrEstablisher)
2356 .setMIFlag(MachineInstr::FrameSetup);
2357 }
2358
2359 if (X86FI->getHasSEHFramePtrSave() && !IsFunclet) {
2360 // Stash the value of the frame pointer relative to the base pointer for
2361 // Win32 EH. This supports Win32 EH, which does the inverse of the above:
2362 // it recovers the frame pointer from the base pointer rather than the
2363 // other way around.
2364 unsigned Opm = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr;
2365 Register UsedReg;
2366 int Offset =
2367 getFrameIndexReference(MF, FI: X86FI->getSEHFramePtrSaveIndex(), FrameReg&: UsedReg)
2368 .getFixed();
2369 assert(UsedReg == BasePtr);
2370 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opm)), Reg: UsedReg, isKill: true, Offset)
2371 .addReg(RegNo: FramePtr)
2372 .setMIFlag(MachineInstr::FrameSetup);
2373 }
2374 }
2375 if (ArgBaseReg.isValid()) {
2376 // Save argument base pointer.
2377 auto *MI = X86FI->getStackPtrSaveMI();
2378 int FI = MI->getOperand(i: 1).getIndex();
2379 unsigned MOVmr = Is64Bit ? X86::MOV64mr : X86::MOV32mr;
2380 // movl %basereg, offset(%ebp)
2381 addFrameReference(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: MOVmr)), FI)
2382 .addReg(RegNo: ArgBaseReg)
2383 .setMIFlag(MachineInstr::FrameSetup);
2384 }
2385
2386 if (((!HasFP && NumBytes) || PushedRegs) && NeedsDwarfCFI) {
2387 // Mark end of stack pointer adjustment.
2388 if (!HasFP && NumBytes) {
2389 // Define the current CFA rule to use the provided offset.
2390 assert(StackSize);
2391 BuildCFI(
2392 MBB, MBBI, DL,
2393 CFIInst: MCCFIInstruction::cfiDefCfaOffset(L: nullptr, Offset: StackSize - stackGrowth),
2394 Flag: MachineInstr::FrameSetup);
2395 }
2396
2397 // Emit DWARF info specifying the offsets of the callee-saved registers.
2398 emitCalleeSavedFrameMoves(MBB, MBBI, DL, IsPrologue: true);
2399 }
2400
2401 // X86 Interrupt handling function cannot assume anything about the direction
2402 // flag (DF in EFLAGS register). Clear this flag by creating "cld" instruction
2403 // in each prologue of interrupt handler function.
2404 //
2405 // Create "cld" instruction only in these cases:
2406 // 1. The interrupt handling function uses any of the "rep" instructions.
2407 // 2. Interrupt handling function calls another function.
2408 // 3. If there are any inline asm blocks, as we do not know what they do
2409 //
2410 // TODO: We should also emit cld if we detect the use of std, but as of now,
2411 // the compiler does not even emit that instruction or even define it, so in
2412 // practice, this would only happen with inline asm, which we cover anyway.
2413 if (Fn.getCallingConv() == CallingConv::X86_INTR) {
2414 bool NeedsCLD = false;
2415
2416 for (const MachineBasicBlock &B : MF) {
2417 for (const MachineInstr &MI : B) {
2418 if (MI.isCall()) {
2419 NeedsCLD = true;
2420 break;
2421 }
2422
2423 if (isOpcodeRep(Opcode: MI.getOpcode())) {
2424 NeedsCLD = true;
2425 break;
2426 }
2427
2428 if (MI.isInlineAsm()) {
2429 // TODO: Parse asm for rep instructions or call sites?
2430 // For now, let's play it safe and emit a cld instruction
2431 // just in case.
2432 NeedsCLD = true;
2433 break;
2434 }
2435 }
2436 }
2437
2438 if (NeedsCLD) {
2439 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::CLD))
2440 .setMIFlag(MachineInstr::FrameSetup);
2441 }
2442 }
2443
2444 // At this point we know if the function has WinCFI or not.
2445 MF.setHasWinCFI(HasWinCFI);
2446}
2447
2448bool X86FrameLowering::canUseLEAForSPInEpilogue(
2449 const MachineFunction &MF) const {
2450 // We can't use LEA instructions for adjusting the stack pointer if we don't
2451 // have a frame pointer in the Win64 ABI. Only ADD instructions may be used
2452 // to deallocate the stack.
2453 // This means that we can use LEA for SP in two situations:
2454 // 1. We *aren't* using the Win64 ABI which means we are free to use LEA.
2455 // 2. We *have* a frame pointer which means we are permitted to use LEA.
2456 return !MF.getTarget().getMCAsmInfo().usesWindowsCFI() || hasFP(MF);
2457}
2458
2459static bool isFuncletReturnInstr(MachineInstr &MI) {
2460 switch (MI.getOpcode()) {
2461 case X86::CATCHRET:
2462 case X86::CLEANUPRET:
2463 return true;
2464 default:
2465 return false;
2466 }
2467 llvm_unreachable("impossible");
2468}
2469
2470// CLR funclets use a special "Previous Stack Pointer Symbol" slot on the
2471// stack. It holds a pointer to the bottom of the root function frame. The
2472// establisher frame pointer passed to a nested funclet may point to the
2473// (mostly empty) frame of its parent funclet, but it will need to find
2474// the frame of the root function to access locals. To facilitate this,
2475// every funclet copies the pointer to the bottom of the root function
2476// frame into a PSPSym slot in its own (mostly empty) stack frame. Using the
2477// same offset for the PSPSym in the root function frame that's used in the
2478// funclets' frames allows each funclet to dynamically accept any ancestor
2479// frame as its establisher argument (the runtime doesn't guarantee the
2480// immediate parent for some reason lost to history), and also allows the GC,
2481// which uses the PSPSym for some bookkeeping, to find it in any funclet's
2482// frame with only a single offset reported for the entire method.
2483unsigned
2484X86FrameLowering::getPSPSlotOffsetFromSP(const MachineFunction &MF) const {
2485 const WinEHFuncInfo &Info = *MF.getWinEHFuncInfo();
2486 Register SPReg;
2487 int Offset = getFrameIndexReferencePreferSP(MF, FI: Info.PSPSymFrameIdx, FrameReg&: SPReg,
2488 /*IgnoreSPUpdates*/ true)
2489 .getFixed();
2490 assert(Offset >= 0 && SPReg == TRI->getStackRegister());
2491 return static_cast<unsigned>(Offset);
2492}
2493
2494unsigned
2495X86FrameLowering::getWinEHFuncletFrameSize(const MachineFunction &MF) const {
2496 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
2497 // This is the size of the pushed CSRs.
2498 unsigned CSSize = X86FI->getCalleeSavedFrameSize();
2499 // This is the size of callee saved XMMs.
2500 const auto &WinEHXMMSlotInfo = X86FI->getWinEHXMMSlotInfo();
2501 unsigned XMMSize =
2502 WinEHXMMSlotInfo.size() * TRI->getSpillSize(RC: X86::VR128RegClass);
2503 // This is the amount of stack a funclet needs to allocate.
2504 unsigned UsedSize;
2505 EHPersonality Personality =
2506 classifyEHPersonality(Pers: MF.getFunction().getPersonalityFn());
2507 if (Personality == EHPersonality::CoreCLR) {
2508 // CLR funclets need to hold enough space to include the PSPSym, at the
2509 // same offset from the stack pointer (immediately after the prolog) as it
2510 // resides at in the main function.
2511 UsedSize = getPSPSlotOffsetFromSP(MF) + SlotSize;
2512 } else {
2513 // Other funclets just need enough stack for outgoing call arguments.
2514 UsedSize = MF.getFrameInfo().getMaxCallFrameSize();
2515 }
2516 // RBP is not included in the callee saved register block. After pushing RBP,
2517 // everything is 16 byte aligned. Everything we allocate before an outgoing
2518 // call must also be 16 byte aligned.
2519 unsigned FrameSizeMinusRBP = alignTo(Size: CSSize + UsedSize, A: getStackAlign());
2520 // Subtract out the size of the callee saved registers. This is how much stack
2521 // each funclet will allocate.
2522 return FrameSizeMinusRBP + XMMSize - CSSize;
2523}
2524
2525static bool isTailCallOpcode(unsigned Opc) {
2526 return Opc == X86::TCRETURNri || Opc == X86::TCRETURN_WIN64ri ||
2527 Opc == X86::TCRETURN_HIPE32ri || Opc == X86::TCRETURNdi ||
2528 Opc == X86::TCRETURNmi || Opc == X86::TCRETURNri64 ||
2529 Opc == X86::TCRETURNri64_ImpCall || Opc == X86::TCRETURNdi64 ||
2530 Opc == X86::TCRETURNmi64 || Opc == X86::TCRETURN_WINmi64;
2531}
2532
2533void X86FrameLowering::emitEpilogue(MachineFunction &MF,
2534 MachineBasicBlock &MBB) const {
2535 const MachineFrameInfo &MFI = MF.getFrameInfo();
2536 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
2537 MachineBasicBlock::iterator Terminator = MBB.getFirstTerminator();
2538 MachineBasicBlock::iterator MBBI = Terminator;
2539 DebugLoc DL;
2540 if (MBBI != MBB.end())
2541 DL = MBBI->getDebugLoc();
2542 // standard x86_64 uses 64-bit frame/stack pointers, x32 - 32-bit.
2543 const bool Is64BitILP32 = STI.isTarget64BitILP32();
2544 Register FramePtr = TRI->getFrameRegister(MF);
2545 Register MachineFramePtr =
2546 Is64BitILP32 ? Register(getX86SubSuperRegister(Reg: FramePtr, Size: 64)) : FramePtr;
2547
2548 bool IsWin64Prologue = MF.getTarget().getMCAsmInfo().usesWindowsCFI();
2549 bool NeedsWin64CFI =
2550 IsWin64Prologue && MF.getFunction().needsUnwindTableEntry();
2551 // For V3 unwind, epilog SEH pseudos are emitted inline before each
2552 // unwind-effecting instruction.
2553 bool IsWin64UnwindV3 =
2554 NeedsWin64CFI && MF.hasWinCFI() && requireWinX64UnwindV3(MF);
2555 bool IsFunclet = MBBI == MBB.end() ? false : isFuncletReturnInstr(MI&: *MBBI);
2556
2557 // Get the number of bytes to allocate from the FrameInfo.
2558 uint64_t StackSize = MFI.getStackSize();
2559 uint64_t MaxAlign = calculateMaxStackAlign(MF);
2560 unsigned CSSize = X86FI->getCalleeSavedFrameSize();
2561 unsigned TailCallArgReserveSize = -X86FI->getTCReturnAddrDelta();
2562 bool HasFP = hasFP(MF);
2563 uint64_t NumBytes = 0;
2564
2565 bool NeedsDwarfCFI = (!MF.getTarget().getTargetTriple().isOSDarwin() &&
2566 !MF.getTarget().getTargetTriple().isOSWindows() &&
2567 !MF.getTarget().getTargetTriple().isUEFI()) &&
2568 MF.needsFrameMoves();
2569
2570 Register ArgBaseReg;
2571 if (auto *MI = X86FI->getStackPtrSaveMI()) {
2572 unsigned Opc = X86::LEA32r;
2573 Register StackReg = X86::ESP;
2574 ArgBaseReg = MI->getOperand(i: 0).getReg();
2575 if (STI.is64Bit()) {
2576 Opc = X86::LEA64r;
2577 StackReg = X86::RSP;
2578 }
2579 // leal -4(%basereg), %esp
2580 // .cfi_def_cfa %esp, 4
2581 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: StackReg)
2582 .addUse(RegNo: ArgBaseReg)
2583 .addImm(Val: 1)
2584 .addUse(RegNo: X86::NoRegister)
2585 .addImm(Val: -(int64_t)SlotSize)
2586 .addUse(RegNo: X86::NoRegister)
2587 .setMIFlag(MachineInstr::FrameDestroy);
2588 if (NeedsDwarfCFI) {
2589 unsigned DwarfStackPtr = TRI->getDwarfRegNum(Reg: StackReg, isEH: true);
2590 BuildCFI(MBB, MBBI, DL,
2591 CFIInst: MCCFIInstruction::cfiDefCfa(L: nullptr, Register: DwarfStackPtr, Offset: SlotSize),
2592 Flag: MachineInstr::FrameDestroy);
2593 --MBBI;
2594 }
2595 --MBBI;
2596 }
2597
2598 if (IsFunclet) {
2599 assert(HasFP && "EH funclets without FP not yet implemented");
2600 NumBytes = getWinEHFuncletFrameSize(MF);
2601 } else if (HasFP) {
2602 // Calculate required stack adjustment.
2603 uint64_t FrameSize = StackSize - SlotSize;
2604 NumBytes = FrameSize - CSSize - TailCallArgReserveSize;
2605
2606 // Callee-saved registers were pushed on stack before the stack was
2607 // realigned.
2608 if (TRI->hasStackRealignment(MF) && !IsWin64Prologue)
2609 NumBytes = alignTo(Value: FrameSize, Align: MaxAlign);
2610 } else {
2611 NumBytes = StackSize - CSSize - TailCallArgReserveSize;
2612 }
2613 uint64_t SEHStackAllocAmt = NumBytes;
2614
2615 unsigned SEHFrameOffset = 0;
2616 if (IsWin64Prologue && HasFP)
2617 SEHFrameOffset = calculateSetFPREG(SPAdjust: SEHStackAllocAmt);
2618
2619 // AfterPop is the position to insert .cfi_restore.
2620 MachineBasicBlock::iterator AfterPop = MBBI;
2621 if (HasFP) {
2622 if (X86FI->hasSwiftAsyncContext()) {
2623 // Discard the context.
2624 int64_t Offset = mergeSPAdd(MBB, MBBI, AddOffset: 16, doMergeWithPrevious: true);
2625 emitSPUpdate(MBB, MBBI, DL, NumBytes: Offset, /*InEpilogue*/ true);
2626 }
2627 // Pop EBP.
2628 if (IsWin64UnwindV3)
2629 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_PushReg))
2630 .addImm(Val: FramePtr)
2631 .setMIFlag(MachineInstr::FrameDestroy);
2632 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
2633 MCID: TII.get(Opcode: getPOPOpcode(ST: MF.getSubtarget<X86Subtarget>())),
2634 DestReg: MachineFramePtr)
2635 .setMIFlag(MachineInstr::FrameDestroy);
2636
2637 // We need to reset FP to its untagged state on return. Bit 60 is currently
2638 // used to show the presence of an extended frame.
2639 if (X86FI->hasSwiftAsyncContext()) {
2640 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::BTR64ri8), DestReg: MachineFramePtr)
2641 .addUse(RegNo: MachineFramePtr)
2642 .addImm(Val: 60)
2643 .setMIFlag(MachineInstr::FrameDestroy);
2644 }
2645
2646 if (NeedsDwarfCFI) {
2647 if (!ArgBaseReg.isValid()) {
2648 unsigned DwarfStackPtr =
2649 TRI->getDwarfRegNum(Reg: Is64Bit ? X86::RSP : X86::ESP, isEH: true);
2650 BuildCFI(MBB, MBBI, DL,
2651 CFIInst: MCCFIInstruction::cfiDefCfa(L: nullptr, Register: DwarfStackPtr, Offset: SlotSize),
2652 Flag: MachineInstr::FrameDestroy);
2653 }
2654 if (!MBB.succ_empty() && !MBB.isReturnBlock()) {
2655 unsigned DwarfFramePtr = TRI->getDwarfRegNum(Reg: MachineFramePtr, isEH: true);
2656 BuildCFI(MBB, MBBI: AfterPop, DL,
2657 CFIInst: MCCFIInstruction::createRestore(L: nullptr, Register: DwarfFramePtr),
2658 Flag: MachineInstr::FrameDestroy);
2659 --MBBI;
2660 --AfterPop;
2661 }
2662 --MBBI;
2663 }
2664 }
2665
2666 MachineBasicBlock::iterator FirstCSPop = MBBI;
2667 // Skip the callee-saved pop instructions.
2668 while (MBBI != MBB.begin()) {
2669 MachineBasicBlock::iterator PI = std::prev(x: MBBI);
2670 unsigned Opc = PI->getOpcode();
2671
2672 if (Opc != X86::DBG_VALUE && !PI->isTerminator()) {
2673 if (!PI->getFlag(Flag: MachineInstr::FrameDestroy) ||
2674 (Opc != X86::POP32r && Opc != X86::POP64r && Opc != X86::BTR64ri8 &&
2675 Opc != X86::ADD64ri32 && Opc != X86::POPP64r && Opc != X86::POP2 &&
2676 Opc != X86::POP2P && Opc != X86::LEA64r && Opc != X86::SEH_PushReg &&
2677 Opc != X86::SEH_Push2Regs && Opc != X86::SEH_StackAlloc &&
2678 Opc != X86::ADD64ri32_NF))
2679 break;
2680 FirstCSPop = PI;
2681 }
2682
2683 --MBBI;
2684 }
2685 if (ArgBaseReg.isValid()) {
2686 // Restore argument base pointer.
2687 auto *MI = X86FI->getStackPtrSaveMI();
2688 int FI = MI->getOperand(i: 1).getIndex();
2689 unsigned MOVrm = Is64Bit ? X86::MOV64rm : X86::MOV32rm;
2690 // movl offset(%ebp), %basereg
2691 addFrameReference(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: MOVrm), DestReg: ArgBaseReg), FI)
2692 .setMIFlag(MachineInstr::FrameDestroy);
2693 }
2694 MBBI = FirstCSPop;
2695
2696 if (IsFunclet && Terminator->getOpcode() == X86::CATCHRET)
2697 emitCatchRetReturnValue(MBB, MBBI: FirstCSPop, CatchRet: &*Terminator);
2698
2699 if (MBBI != MBB.end())
2700 DL = MBBI->getDebugLoc();
2701 // If there is an ADD32ri or SUB32ri of ESP immediately before this
2702 // instruction, merge the two instructions.
2703 if (NumBytes || MFI.hasVarSizedObjects())
2704 NumBytes = mergeSPAdd(MBB, MBBI, AddOffset: NumBytes, doMergeWithPrevious: true);
2705
2706 if (IsWin64UnwindV3 && NeedsWin64CFI && MF.hasWinCFI()) {
2707 // Find the XMM restores that were tagged with FrameDestroy, now that we
2708 // know the offset we can emit the SEH pseudos for them.
2709 auto EpilogStart = MBBI;
2710 {
2711 auto ScanIt = MBBI;
2712 while (ScanIt != MBB.begin()) {
2713 auto PI = std::prev(x: ScanIt);
2714 int FI;
2715 if (PI->getFlag(Flag: MachineInstr::FrameDestroy) &&
2716 TII.isLoadFromStackSlot(MI: *PI, FrameIndex&: FI)) {
2717 Register Reg = PI->getOperand(i: 0).getReg();
2718 if (X86::FR64RegClass.contains(Reg)) {
2719 Register IgnoredFrameReg;
2720 int Offset =
2721 getFrameIndexReference(MF, FI, FrameReg&: IgnoredFrameReg).getFixed() +
2722 SEHFrameOffset;
2723 BuildMI(BB&: MBB, I: PI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_SaveXMM))
2724 .addImm(Val: Reg)
2725 .addImm(Val: Offset)
2726 .setMIFlag(MachineInstr::FrameDestroy);
2727 // std::prev(PI) is the SEH_SaveXMM we just inserted (before PI).
2728 // We start ScanIt from that point so that the next
2729 // std::prev(ScanIt) will examine the instruction before the pseudo,
2730 // i.e. the next potential XMM restore further up the block.
2731 EpilogStart = std::prev(x: PI);
2732 ScanIt = EpilogStart;
2733 continue;
2734 }
2735 }
2736 break;
2737 }
2738 }
2739
2740 // For V3, SEH_BeginEpilogue must be emitted before any epilog SEH pseudos.
2741 BuildMI(BB&: MBB, I: EpilogStart, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_BeginEpilogue));
2742 }
2743
2744 // If dynamic alloca is used, then reset esp to point to the last callee-saved
2745 // slot before popping them off! Same applies for the case, when stack was
2746 // realigned. Don't do this if this was a funclet epilogue, since the funclets
2747 // will not do realignment or dynamic stack allocation.
2748 if (((TRI->hasStackRealignment(MF)) || MFI.hasVarSizedObjects()) &&
2749 !IsFunclet) {
2750 if (TRI->hasStackRealignment(MF))
2751 MBBI = FirstCSPop;
2752 uint64_t LEAAmount =
2753 IsWin64Prologue ? SEHStackAllocAmt - SEHFrameOffset : -CSSize;
2754
2755 if (X86FI->hasSwiftAsyncContext())
2756 LEAAmount -= 16;
2757
2758 // There are only two legal forms of epilogue:
2759 // - add SEHAllocationSize, %rsp
2760 // - lea SEHAllocationSize(%FramePtr), %rsp
2761 //
2762 // 'mov %FramePtr, %rsp' will not be recognized as an epilogue sequence.
2763 // However, we may use this sequence if we have a frame pointer because the
2764 // effects of the prologue can safely be undone.
2765 if (IsWin64UnwindV3) {
2766 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_SetFrame))
2767 .addImm(Val: FramePtr)
2768 .addImm(Val: SEHFrameOffset)
2769 .setMIFlag(MachineInstr::FrameDestroy);
2770 if (SEHStackAllocAmt)
2771 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_StackAlloc))
2772 .addImm(Val: SEHStackAllocAmt)
2773 .setMIFlag(MachineInstr::FrameDestroy);
2774 }
2775 if (LEAAmount != 0) {
2776 unsigned Opc = getLEArOpcode(IsLP64: Uses64BitFramePtr);
2777 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: StackPtr), Reg: FramePtr,
2778 isKill: false, Offset: LEAAmount);
2779 --MBBI;
2780 } else {
2781 unsigned Opc = (Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr);
2782 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: StackPtr).addReg(RegNo: FramePtr);
2783 --MBBI;
2784 }
2785 } else if (NumBytes) {
2786 // Adjust stack pointer back: ESP += numbytes.
2787 if (IsWin64UnwindV3)
2788 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_StackAlloc))
2789 .addImm(Val: NumBytes)
2790 .setMIFlag(MachineInstr::FrameDestroy);
2791 emitSPUpdate(MBB, MBBI, DL, NumBytes, /*InEpilogue=*/true);
2792 if (!HasFP && NeedsDwarfCFI) {
2793 // Define the current CFA rule to use the provided offset.
2794 BuildCFI(MBB, MBBI, DL,
2795 CFIInst: MCCFIInstruction::cfiDefCfaOffset(
2796 L: nullptr, Offset: CSSize + TailCallArgReserveSize + SlotSize),
2797 Flag: MachineInstr::FrameDestroy);
2798 }
2799 --MBBI;
2800 }
2801
2802 // For V1/V2, emit SEH_BeginEpilogue after stack restore code.
2803 if (!IsWin64UnwindV3 && NeedsWin64CFI && MF.hasWinCFI())
2804 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_BeginEpilogue));
2805
2806 if (!HasFP && NeedsDwarfCFI) {
2807 MBBI = FirstCSPop;
2808 int64_t Offset = -(int64_t)CSSize - SlotSize;
2809 // Mark callee-saved pop instruction.
2810 // Define the current CFA rule to use the provided offset.
2811 while (MBBI != MBB.end()) {
2812 MachineBasicBlock::iterator PI = MBBI;
2813 unsigned Opc = PI->getOpcode();
2814 ++MBBI;
2815 if (Opc == X86::POP32r || Opc == X86::POP64r || Opc == X86::POPP64r ||
2816 Opc == X86::POP2 || Opc == X86::POP2P) {
2817 Offset += SlotSize;
2818 // Compared to pop, pop2 introduces more stack offset (one more
2819 // register).
2820 if (Opc == X86::POP2 || Opc == X86::POP2P)
2821 Offset += SlotSize;
2822 BuildCFI(MBB, MBBI, DL,
2823 CFIInst: MCCFIInstruction::cfiDefCfaOffset(L: nullptr, Offset: -Offset),
2824 Flag: MachineInstr::FrameDestroy);
2825 }
2826 }
2827 }
2828
2829 // Emit DWARF info specifying the restores of the callee-saved registers.
2830 // For epilogue with return inside or being other block without successor,
2831 // no need to generate .cfi_restore for callee-saved registers.
2832 if (NeedsDwarfCFI && !MBB.succ_empty())
2833 emitCalleeSavedFrameMoves(MBB, MBBI: AfterPop, DL, IsPrologue: false);
2834
2835 if (Terminator == MBB.end() || !isTailCallOpcode(Opc: Terminator->getOpcode())) {
2836 // Add the return addr area delta back since we are not tail calling.
2837 int64_t Delta = X86FI->getTCReturnAddrDelta();
2838 assert(Delta <= 0 && "TCDelta should never be positive");
2839 if (Delta) {
2840 // Check for possible merge with preceding ADD instruction.
2841 int64_t Offset = mergeSPAdd(MBB, MBBI&: Terminator, AddOffset: -Delta, doMergeWithPrevious: true);
2842 emitSPUpdate(MBB, MBBI&: Terminator, DL, NumBytes: Offset, /*InEpilogue=*/true);
2843 }
2844 }
2845
2846 // Emit tilerelease for AMX kernel.
2847 if (X86FI->getAMXProgModel() == AMXProgModelEnum::ManagedRA)
2848 BuildMI(BB&: MBB, I: Terminator, MIMD: DL, MCID: TII.get(Opcode: X86::TILERELEASE));
2849
2850 if (NeedsWin64CFI && MF.hasWinCFI())
2851 BuildMI(BB&: MBB, I: Terminator, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_EndEpilogue));
2852}
2853
2854StackOffset X86FrameLowering::getFrameIndexReference(const MachineFunction &MF,
2855 int FI,
2856 Register &FrameReg) const {
2857 const MachineFrameInfo &MFI = MF.getFrameInfo();
2858
2859 bool IsFixed = MFI.isFixedObjectIndex(ObjectIdx: FI);
2860 // We can't calculate offset from frame pointer if the stack is realigned,
2861 // so enforce usage of stack/base pointer. The base pointer is used when we
2862 // have dynamic allocas in addition to dynamic realignment.
2863 if (TRI->hasBasePointer(MF))
2864 FrameReg = IsFixed ? TRI->getFramePtr() : TRI->getBaseRegister();
2865 else if (TRI->hasStackRealignment(MF))
2866 FrameReg = IsFixed ? TRI->getFramePtr() : TRI->getStackRegister();
2867 else
2868 FrameReg = TRI->getFrameRegister(MF);
2869
2870 // Offset will hold the offset from the stack pointer at function entry to the
2871 // object.
2872 // We need to factor in additional offsets applied during the prologue to the
2873 // frame, base, and stack pointer depending on which is used.
2874 int64_t Offset = MFI.getObjectOffset(ObjectIdx: FI) - getOffsetOfLocalArea();
2875 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
2876 unsigned CSSize = X86FI->getCalleeSavedFrameSize();
2877 uint64_t StackSize = MFI.getStackSize();
2878 bool IsWin64Prologue = MF.getTarget().getMCAsmInfo().usesWindowsCFI();
2879 int64_t FPDelta = 0;
2880
2881 // In an x86 interrupt, remove the offset we added to account for the return
2882 // address from any stack object allocated in the caller's frame. Interrupts
2883 // do not have a standard return address. Fixed objects in the current frame,
2884 // such as SSE register spills, should not get this treatment.
2885 if (MF.getFunction().getCallingConv() == CallingConv::X86_INTR &&
2886 Offset >= 0) {
2887 Offset += getOffsetOfLocalArea();
2888 }
2889
2890 if (IsWin64Prologue) {
2891 assert(!MFI.hasCalls() || (StackSize % 16) == 8);
2892
2893 // Calculate required stack adjustment.
2894 uint64_t FrameSize = StackSize - SlotSize;
2895 // If required, include space for extra hidden slot for stashing base
2896 // pointer.
2897 if (X86FI->getRestoreBasePointer())
2898 FrameSize += SlotSize;
2899 uint64_t NumBytes = FrameSize - CSSize;
2900
2901 uint64_t SEHFrameOffset = calculateSetFPREG(SPAdjust: NumBytes);
2902 if (FI && FI == X86FI->getFAIndex())
2903 return StackOffset::getFixed(Fixed: -SEHFrameOffset);
2904
2905 // FPDelta is the offset from the "traditional" FP location of the old base
2906 // pointer followed by return address and the location required by the
2907 // restricted Win64 prologue.
2908 // Add FPDelta to all offsets below that go through the frame pointer.
2909 FPDelta = FrameSize - SEHFrameOffset;
2910 assert((!MFI.hasCalls() || (FPDelta % 16) == 0) &&
2911 "FPDelta isn't aligned per the Win64 ABI!");
2912 }
2913
2914 if (FrameReg == TRI->getFramePtr()) {
2915 // Skip saved EBP/RBP
2916 Offset += SlotSize;
2917
2918 // Account for restricted Windows prologue.
2919 Offset += FPDelta;
2920
2921 // Skip the RETADDR move area
2922 int TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta();
2923 if (TailCallReturnAddrDelta < 0)
2924 Offset -= TailCallReturnAddrDelta;
2925
2926 return StackOffset::getFixed(Fixed: Offset);
2927 }
2928
2929 // FrameReg is either the stack pointer or a base pointer. But the base is
2930 // located at the end of the statically known StackSize so the distinction
2931 // doesn't really matter.
2932 if (TRI->hasStackRealignment(MF) || TRI->hasBasePointer(MF))
2933 assert(isAligned(MFI.getObjectAlign(FI), -(Offset + StackSize)));
2934 return StackOffset::getFixed(Fixed: Offset + StackSize);
2935}
2936
2937int X86FrameLowering::getWin64EHFrameIndexRef(const MachineFunction &MF, int FI,
2938 Register &FrameReg) const {
2939 const MachineFrameInfo &MFI = MF.getFrameInfo();
2940 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
2941 const auto &WinEHXMMSlotInfo = X86FI->getWinEHXMMSlotInfo();
2942 const auto it = WinEHXMMSlotInfo.find(Val: FI);
2943
2944 if (it == WinEHXMMSlotInfo.end())
2945 return getFrameIndexReference(MF, FI, FrameReg).getFixed();
2946
2947 FrameReg = TRI->getStackRegister();
2948 return alignDown(Value: MFI.getMaxCallFrameSize(), Align: getStackAlign().value()) +
2949 it->second;
2950}
2951
2952StackOffset
2953X86FrameLowering::getFrameIndexReferenceSP(const MachineFunction &MF, int FI,
2954 Register &FrameReg,
2955 int Adjustment) const {
2956 const MachineFrameInfo &MFI = MF.getFrameInfo();
2957 FrameReg = TRI->getStackRegister();
2958 return StackOffset::getFixed(Fixed: MFI.getObjectOffset(ObjectIdx: FI) -
2959 getOffsetOfLocalArea() + Adjustment);
2960}
2961
2962StackOffset
2963X86FrameLowering::getFrameIndexReferencePreferSP(const MachineFunction &MF,
2964 int FI, Register &FrameReg,
2965 bool IgnoreSPUpdates) const {
2966
2967 const MachineFrameInfo &MFI = MF.getFrameInfo();
2968 // Does not include any dynamic realign.
2969 const uint64_t StackSize = MFI.getStackSize();
2970 // LLVM arranges the stack as follows:
2971 // ...
2972 // ARG2
2973 // ARG1
2974 // RETADDR
2975 // PUSH RBP <-- RBP points here
2976 // PUSH CSRs
2977 // ~~~~~~~ <-- possible stack realignment (non-win64)
2978 // ...
2979 // STACK OBJECTS
2980 // ... <-- RSP after prologue points here
2981 // ~~~~~~~ <-- possible stack realignment (win64)
2982 //
2983 // if (hasVarSizedObjects()):
2984 // ... <-- "base pointer" (ESI/RBX) points here
2985 // DYNAMIC ALLOCAS
2986 // ... <-- RSP points here
2987 //
2988 // Case 1: In the simple case of no stack realignment and no dynamic
2989 // allocas, both "fixed" stack objects (arguments and CSRs) are addressable
2990 // with fixed offsets from RSP.
2991 //
2992 // Case 2: In the case of stack realignment with no dynamic allocas, fixed
2993 // stack objects are addressed with RBP and regular stack objects with RSP.
2994 //
2995 // Case 3: In the case of dynamic allocas and stack realignment, RSP is used
2996 // to address stack arguments for outgoing calls and nothing else. The "base
2997 // pointer" points to local variables, and RBP points to fixed objects.
2998 //
2999 // In cases 2 and 3, we can only answer for non-fixed stack objects, and the
3000 // answer we give is relative to the SP after the prologue, and not the
3001 // SP in the middle of the function.
3002
3003 if (MFI.isFixedObjectIndex(ObjectIdx: FI) && TRI->hasStackRealignment(MF) &&
3004 !STI.isTargetWin64())
3005 return getFrameIndexReference(MF, FI, FrameReg);
3006
3007 // If !hasReservedCallFrame the function might have SP adjustement in the
3008 // body. So, even though the offset is statically known, it depends on where
3009 // we are in the function.
3010 if (!IgnoreSPUpdates && !hasReservedCallFrame(MF))
3011 return getFrameIndexReference(MF, FI, FrameReg);
3012
3013 // We don't handle tail calls, and shouldn't be seeing them either.
3014 assert(MF.getInfo<X86MachineFunctionInfo>()->getTCReturnAddrDelta() >= 0 &&
3015 "we don't handle this case!");
3016
3017 // This is how the math works out:
3018 //
3019 // %rsp grows (i.e. gets lower) left to right. Each box below is
3020 // one word (eight bytes). Obj0 is the stack slot we're trying to
3021 // get to.
3022 //
3023 // ----------------------------------
3024 // | BP | Obj0 | Obj1 | ... | ObjN |
3025 // ----------------------------------
3026 // ^ ^ ^ ^
3027 // A B C E
3028 //
3029 // A is the incoming stack pointer.
3030 // (B - A) is the local area offset (-8 for x86-64) [1]
3031 // (C - A) is the Offset returned by MFI.getObjectOffset for Obj0 [2]
3032 //
3033 // |(E - B)| is the StackSize (absolute value, positive). For a
3034 // stack that grown down, this works out to be (B - E). [3]
3035 //
3036 // E is also the value of %rsp after stack has been set up, and we
3037 // want (C - E) -- the value we can add to %rsp to get to Obj0. Now
3038 // (C - E) == (C - A) - (B - A) + (B - E)
3039 // { Using [1], [2] and [3] above }
3040 // == getObjectOffset - LocalAreaOffset + StackSize
3041
3042 return getFrameIndexReferenceSP(MF, FI, FrameReg, Adjustment: StackSize);
3043}
3044
3045bool X86FrameLowering::assignCalleeSavedSpillSlots(
3046 MachineFunction &MF, const TargetRegisterInfo *TRI,
3047 std::vector<CalleeSavedInfo> &CSI) const {
3048 MachineFrameInfo &MFI = MF.getFrameInfo();
3049 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
3050
3051 unsigned CalleeSavedFrameSize = 0;
3052 unsigned XMMCalleeSavedFrameSize = 0;
3053 auto &WinEHXMMSlotInfo = X86FI->getWinEHXMMSlotInfo();
3054 int SpillSlotOffset = getOffsetOfLocalArea() + X86FI->getTCReturnAddrDelta();
3055
3056 int64_t TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta();
3057
3058 if (TailCallReturnAddrDelta < 0) {
3059 // create RETURNADDR area
3060 // arg
3061 // arg
3062 // RETADDR
3063 // { ...
3064 // RETADDR area
3065 // ...
3066 // }
3067 // [EBP]
3068 MFI.CreateFixedObject(Size: -TailCallReturnAddrDelta,
3069 SPOffset: TailCallReturnAddrDelta - SlotSize, IsImmutable: true);
3070 }
3071
3072 // Spill the BasePtr if it's used.
3073 if (this->TRI->hasBasePointer(MF)) {
3074 // Allocate a spill slot for EBP if we have a base pointer and EH funclets.
3075 if (MF.hasEHFunclets()) {
3076 int FI = MFI.CreateSpillStackObject(Size: SlotSize, Alignment: Align(SlotSize));
3077 X86FI->setHasSEHFramePtrSave(true);
3078 X86FI->setSEHFramePtrSaveIndex(FI);
3079 }
3080 }
3081
3082 bool IsFPRemovedFromCSI = false;
3083 if (hasFP(MF)) {
3084 // emitPrologue always spills frame register the first thing.
3085 SpillSlotOffset -= SlotSize;
3086 MFI.CreateFixedSpillStackObject(Size: SlotSize, SPOffset: SpillSlotOffset);
3087
3088 // The async context lives directly before the frame pointer, and we
3089 // allocate a second slot to preserve stack alignment.
3090 if (X86FI->hasSwiftAsyncContext()) {
3091 SpillSlotOffset -= SlotSize;
3092 MFI.CreateFixedSpillStackObject(Size: SlotSize, SPOffset: SpillSlotOffset);
3093 SpillSlotOffset -= SlotSize;
3094 }
3095
3096 // Since emitPrologue and emitEpilogue will handle spilling and restoring of
3097 // the frame register, we can delete it from CSI list and not have to worry
3098 // about avoiding it later.
3099 Register FPReg = TRI->getFrameRegister(MF);
3100 for (unsigned i = 0; i < CSI.size(); ++i) {
3101 if (TRI->regsOverlap(RegA: CSI[i].getReg(), RegB: FPReg)) {
3102 CSI.erase(position: CSI.begin() + i);
3103 IsFPRemovedFromCSI = true;
3104 break;
3105 }
3106 }
3107 }
3108
3109 // Strategy:
3110 // 1. Use push2 when
3111 // a) number of CSR > 1 if no need padding
3112 // b) number of CSR > 2 if need padding
3113 // c) stack alignment >= 16 bytes
3114 // 2. When the number of CSR push is odd
3115 // a. Start to use push2 from the 1st push if stack is 16B aligned.
3116 // b. Start to use push2 from the 2nd push if stack is not 16B aligned.
3117 // 3. When the number of CSR push is even, start to use push2 from the 1st
3118 // push and make the stack 16B aligned before the push
3119 unsigned NumRegsForPush2 = 0;
3120 if (STI.hasPush2Pop2() && getStackAlignment() >= 16) {
3121 unsigned NumCSGPR = llvm::count_if(Range&: CSI, P: [](const CalleeSavedInfo &I) {
3122 return X86::GR64RegClass.contains(Reg: I.getReg());
3123 });
3124 bool UsePush2Pop2 = !IsFPRemovedFromCSI ? NumCSGPR > 2 : NumCSGPR > 1;
3125 NumRegsForPush2 =
3126 UsePush2Pop2
3127 ? alignDown(Value: IsFPRemovedFromCSI ? NumCSGPR : NumCSGPR - 1, Align: 2)
3128 : 0;
3129 }
3130
3131 // Assign slots for GPRs. It increases frame size.
3132 for (CalleeSavedInfo &I : llvm::reverse(C&: CSI)) {
3133 MCRegister Reg = I.getReg();
3134
3135 if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg))
3136 continue;
3137
3138 // A CSR is a candidate for push2/pop2 when it's slot offset is 16B aligned
3139 // or only an odd number of registers in the candidates.
3140 if (X86FI->getNumCandidatesForPush2Pop2() < NumRegsForPush2 &&
3141 (SpillSlotOffset % 16 == 0 ||
3142 X86FI->getNumCandidatesForPush2Pop2() % 2))
3143 X86FI->addCandidateForPush2Pop2(Reg);
3144
3145 SpillSlotOffset -= SlotSize;
3146 CalleeSavedFrameSize += SlotSize;
3147
3148 int SlotIndex = MFI.CreateFixedSpillStackObject(Size: SlotSize, SPOffset: SpillSlotOffset);
3149 I.setFrameIdx(SlotIndex);
3150 }
3151
3152 // Adjust the offset of spill slot as we know the accurate callee saved frame
3153 // size.
3154 if (X86FI->getRestoreBasePointer()) {
3155 SpillSlotOffset -= SlotSize;
3156 CalleeSavedFrameSize += SlotSize;
3157
3158 MFI.CreateFixedSpillStackObject(Size: SlotSize, SPOffset: SpillSlotOffset);
3159 // TODO: saving the slot index is better?
3160 X86FI->setRestoreBasePointer(CalleeSavedFrameSize);
3161 }
3162 assert(X86FI->getNumCandidatesForPush2Pop2() % 2 == 0 &&
3163 "Expect even candidates for push2/pop2");
3164 if (X86FI->getNumCandidatesForPush2Pop2())
3165 ++NumFunctionUsingPush2Pop2;
3166 X86FI->setCalleeSavedFrameSize(CalleeSavedFrameSize);
3167 MFI.setCVBytesOfCalleeSavedRegisters(CalleeSavedFrameSize);
3168
3169 // Assign slots for XMMs.
3170 for (CalleeSavedInfo &I : llvm::reverse(C&: CSI)) {
3171 MCRegister Reg = I.getReg();
3172 if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg))
3173 continue;
3174
3175 const TargetRegisterClass *RC = getCalleeSavedSpillRC(Reg, STI, TRI: *TRI);
3176 unsigned Size = TRI->getSpillSize(RC: *RC);
3177 Align Alignment = TRI->getSpillAlign(RC: *RC);
3178 // ensure alignment
3179 assert(SpillSlotOffset < 0 && "SpillSlotOffset should always < 0 on X86");
3180 SpillSlotOffset = -alignTo(Size: -SpillSlotOffset, A: Alignment);
3181
3182 // spill into slot
3183 SpillSlotOffset -= Size;
3184 int SlotIndex = MFI.CreateFixedSpillStackObject(Size, SPOffset: SpillSlotOffset);
3185 I.setFrameIdx(SlotIndex);
3186 MFI.ensureMaxAlignment(Alignment);
3187
3188 // Save the start offset and size of XMM in stack frame for funclets.
3189 if (X86::VR128RegClass.contains(Reg)) {
3190 WinEHXMMSlotInfo[SlotIndex] = XMMCalleeSavedFrameSize;
3191 XMMCalleeSavedFrameSize += Size;
3192 }
3193 }
3194
3195 return true;
3196}
3197
3198bool X86FrameLowering::spillCalleeSavedRegisters(
3199 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
3200 ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
3201 DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
3202
3203 // Don't save CSRs in 32-bit EH funclets. The caller saves EBX, EBP, ESI, EDI
3204 // for us, and there are no XMM CSRs on Win32.
3205 if (MBB.isEHFuncletEntry() && STI.is32Bit() && STI.isOSWindows())
3206 return true;
3207
3208 // Push GPRs. It increases frame size.
3209 const MachineFunction &MF = *MBB.getParent();
3210 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
3211
3212 // Update LiveIn of the basic block and decide whether we can add a kill flag
3213 // to the use.
3214 auto UpdateLiveInCheckCanKill = [&](Register Reg) {
3215 const MachineRegisterInfo &MRI = MF.getRegInfo();
3216 // Do not set a kill flag on values that are also marked as live-in. This
3217 // happens with the @llvm-returnaddress intrinsic and with arguments
3218 // passed in callee saved registers.
3219 // Omitting the kill flags is conservatively correct even if the live-in
3220 // is not used after all.
3221 if (MRI.isLiveIn(Reg))
3222 return false;
3223 MBB.addLiveIn(PhysReg: Reg);
3224 // Check if any subregister is live-in
3225 for (MCRegAliasIterator AReg(Reg, TRI, false); AReg.isValid(); ++AReg)
3226 if (MRI.isLiveIn(Reg: *AReg))
3227 return false;
3228 return true;
3229 };
3230 auto UpdateLiveInGetKillRegState = [&](Register Reg) {
3231 return getKillRegState(B: UpdateLiveInCheckCanKill(Reg));
3232 };
3233
3234 for (auto RI = CSI.rbegin(), RE = CSI.rend(); RI != RE; ++RI) {
3235 MCRegister Reg = RI->getReg();
3236 if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg))
3237 continue;
3238
3239 if (X86FI->isCandidateForPush2Pop2(Reg)) {
3240 MCRegister Reg2 = (++RI)->getReg();
3241 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: getPUSH2Opcode(ST: STI)))
3242 .addReg(RegNo: Reg, Flags: UpdateLiveInGetKillRegState(Reg))
3243 .addReg(RegNo: Reg2, Flags: UpdateLiveInGetKillRegState(Reg2))
3244 .setMIFlag(MachineInstr::FrameSetup);
3245 } else {
3246 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: getPUSHOpcode(ST: STI)))
3247 .addReg(RegNo: Reg, Flags: UpdateLiveInGetKillRegState(Reg))
3248 .setMIFlag(MachineInstr::FrameSetup);
3249 }
3250 }
3251
3252 if (X86FI->getRestoreBasePointer()) {
3253 unsigned Opc = STI.is64Bit() ? X86::PUSH64r : X86::PUSH32r;
3254 Register BaseReg = this->TRI->getBaseRegister();
3255 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: Opc))
3256 .addReg(RegNo: BaseReg, Flags: getKillRegState(B: true))
3257 .setMIFlag(MachineInstr::FrameSetup);
3258 }
3259
3260 // Make XMM regs spilled. X86 does not have ability of push/pop XMM.
3261 // It can be done by spilling XMMs to stack frame.
3262 for (const CalleeSavedInfo &I : llvm::reverse(C&: CSI)) {
3263 MCRegister Reg = I.getReg();
3264 if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg))
3265 continue;
3266
3267 // Add the callee-saved register as live-in. It's killed at the spill.
3268 MBB.addLiveIn(PhysReg: Reg);
3269 const TargetRegisterClass *RC = getCalleeSavedSpillRC(Reg, STI, TRI: *TRI);
3270
3271 TII.storeRegToStackSlot(MBB, MI, SrcReg: Reg, isKill: true, FrameIndex: I.getFrameIdx(), RC, VReg: Register(),
3272 Flags: MachineInstr::FrameSetup);
3273 }
3274
3275 return true;
3276}
3277
3278void X86FrameLowering::emitCatchRetReturnValue(MachineBasicBlock &MBB,
3279 MachineBasicBlock::iterator MBBI,
3280 MachineInstr *CatchRet) const {
3281 // SEH shouldn't use catchret.
3282 assert(!isAsynchronousEHPersonality(classifyEHPersonality(
3283 MBB.getParent()->getFunction().getPersonalityFn())) &&
3284 "SEH should not use CATCHRET");
3285 const DebugLoc &DL = CatchRet->getDebugLoc();
3286 MachineBasicBlock *CatchRetTarget = CatchRet->getOperand(i: 0).getMBB();
3287
3288 // Fill EAX/RAX with the address of the target block.
3289 if (STI.is64Bit()) {
3290 // LEA64r CatchRetTarget(%rip), %rax
3291 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::LEA64r), DestReg: X86::RAX)
3292 .addReg(RegNo: X86::RIP)
3293 .addImm(Val: 0)
3294 .addReg(RegNo: 0)
3295 .addMBB(MBB: CatchRetTarget)
3296 .addReg(RegNo: 0);
3297 } else {
3298 // MOV32ri $CatchRetTarget, %eax
3299 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV32ri), DestReg: X86::EAX)
3300 .addMBB(MBB: CatchRetTarget);
3301 }
3302
3303 // Record that we've taken the address of CatchRetTarget and no longer just
3304 // reference it in a terminator.
3305 CatchRetTarget->setMachineBlockAddressTaken();
3306}
3307
3308bool X86FrameLowering::restoreCalleeSavedRegisters(
3309 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
3310 MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
3311 if (CSI.empty())
3312 return false;
3313
3314 if (MI != MBB.end() && isFuncletReturnInstr(MI&: *MI) && STI.isOSWindows()) {
3315 // Don't restore CSRs in 32-bit EH funclets. Matches
3316 // spillCalleeSavedRegisters.
3317 if (STI.is32Bit())
3318 return true;
3319 // Don't restore CSRs before an SEH catchret. SEH except blocks do not form
3320 // funclets. emitEpilogue transforms these to normal jumps.
3321 if (MI->getOpcode() == X86::CATCHRET) {
3322 const Function &F = MBB.getParent()->getFunction();
3323 bool IsSEH = isAsynchronousEHPersonality(
3324 Pers: classifyEHPersonality(Pers: F.getPersonalityFn()));
3325 if (IsSEH)
3326 return true;
3327 }
3328 }
3329
3330 DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
3331 MachineFunction &MF = *MBB.getParent();
3332 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
3333
3334 bool NeedsWin64CFI =
3335 isWin64Prologue(MF) && MF.getFunction().needsUnwindTableEntry();
3336 bool IsWin64UnwindV3 = NeedsWin64CFI && requireWinX64UnwindV3(MF);
3337
3338 // Reload XMMs from stack frame.
3339 for (const CalleeSavedInfo &I : CSI) {
3340 MCRegister Reg = I.getReg();
3341 if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg))
3342 continue;
3343
3344 const TargetRegisterClass *RC = getCalleeSavedSpillRC(Reg, STI, TRI: *TRI);
3345 TII.loadRegFromStackSlot(MBB, MI, DestReg: Reg, FrameIndex: I.getFrameIdx(), RC, VReg: Register(), SubReg: 0,
3346 Flags: MachineInstr::FrameDestroy);
3347 }
3348
3349 // Clear the stack slot for spill base pointer register.
3350 if (X86FI->getRestoreBasePointer()) {
3351 if (IsWin64UnwindV3)
3352 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_PushReg))
3353 .addImm(Val: this->TRI->getBaseRegister())
3354 .setMIFlag(MachineInstr::FrameDestroy);
3355 unsigned Opc = STI.is64Bit() ? X86::POP64r : X86::POP32r;
3356 Register BaseReg = this->TRI->getBaseRegister();
3357 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: BaseReg)
3358 .setMIFlag(MachineInstr::FrameDestroy);
3359 }
3360
3361 // POP GPRs.
3362 for (auto I = CSI.begin(), E = CSI.end(); I != E; ++I) {
3363 MCRegister Reg = I->getReg();
3364 if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg))
3365 continue;
3366
3367 if (X86FI->isCandidateForPush2Pop2(Reg)) {
3368 MCRegister Reg2 = (++I)->getReg();
3369 if (IsWin64UnwindV3) {
3370 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_Push2Regs))
3371 .addImm(Val: Reg)
3372 .addImm(Val: Reg2)
3373 .setMIFlag(MachineInstr::FrameDestroy);
3374 }
3375 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: getPOP2Opcode(ST: STI)), DestReg: Reg)
3376 .addReg(RegNo: Reg2, Flags: RegState::Define)
3377 .setMIFlag(MachineInstr::FrameDestroy);
3378 } else {
3379 if (IsWin64UnwindV3)
3380 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_PushReg))
3381 .addImm(Val: Reg)
3382 .setMIFlag(MachineInstr::FrameDestroy);
3383 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: getPOPOpcode(ST: STI)), DestReg: Reg)
3384 .setMIFlag(MachineInstr::FrameDestroy);
3385 }
3386 }
3387
3388 return true;
3389}
3390
3391void X86FrameLowering::determineCalleeSaves(MachineFunction &MF,
3392 BitVector &SavedRegs,
3393 RegScavenger *RS) const {
3394 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
3395
3396 // Spill the BasePtr if it's used.
3397 if (TRI->hasBasePointer(MF)) {
3398 Register BasePtr = TRI->getBaseRegister();
3399 if (STI.isTarget64BitILP32())
3400 BasePtr = getX86SubSuperRegister(Reg: BasePtr, Size: 64);
3401 SavedRegs.set(BasePtr);
3402 }
3403 if (STI.hasUserReservedRegisters()) {
3404 for (int Reg = SavedRegs.find_first(); Reg != -1;
3405 Reg = SavedRegs.find_next(Prev: Reg)) {
3406 if (STI.isRegisterReservedByUser(i: Reg)) {
3407 SavedRegs.reset(Idx: Reg);
3408 }
3409 }
3410 }
3411}
3412
3413static bool HasNestArgument(const MachineFunction *MF) {
3414 const Function &F = MF->getFunction();
3415 for (Function::const_arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E;
3416 I++) {
3417 if (I->hasNestAttr() && !I->use_empty())
3418 return true;
3419 }
3420 return false;
3421}
3422
3423/// GetScratchRegister - Get a temp register for performing work in the
3424/// segmented stack and the Erlang/HiPE stack prologue. Depending on platform
3425/// and the properties of the function either one or two registers will be
3426/// needed. Set primary to true for the first register, false for the second.
3427static unsigned GetScratchRegister(bool Is64Bit, bool IsLP64,
3428 const MachineFunction &MF, bool Primary) {
3429 CallingConv::ID CallingConvention = MF.getFunction().getCallingConv();
3430
3431 // Erlang stuff.
3432 if (CallingConvention == CallingConv::HiPE) {
3433 if (Is64Bit)
3434 return Primary ? X86::R14 : X86::R13;
3435 else
3436 return Primary ? X86::EBX : X86::EDI;
3437 }
3438
3439 if (Is64Bit) {
3440 if (IsLP64)
3441 return Primary ? X86::R11 : X86::R12;
3442 else
3443 return Primary ? X86::R11D : X86::R12D;
3444 }
3445
3446 bool IsNested = HasNestArgument(MF: &MF);
3447
3448 if (CallingConvention == CallingConv::X86_FastCall ||
3449 CallingConvention == CallingConv::Fast ||
3450 CallingConvention == CallingConv::Tail) {
3451 if (IsNested)
3452 report_fatal_error(reason: "Segmented stacks does not support fastcall with "
3453 "nested function.");
3454 return Primary ? X86::EAX : X86::ECX;
3455 }
3456 if (IsNested)
3457 return Primary ? X86::EDX : X86::EAX;
3458 return Primary ? X86::ECX : X86::EAX;
3459}
3460
3461// The stack limit in the TCB is set to this many bytes above the actual stack
3462// limit.
3463static const uint64_t kSplitStackAvailable = 256;
3464
3465void X86FrameLowering::adjustForSegmentedStacks(
3466 MachineFunction &MF, MachineBasicBlock &PrologueMBB) const {
3467 MachineFrameInfo &MFI = MF.getFrameInfo();
3468 uint64_t StackSize;
3469 unsigned TlsReg, TlsOffset;
3470 DebugLoc DL;
3471
3472 // To support shrink-wrapping we would need to insert the new blocks
3473 // at the right place and update the branches to PrologueMBB.
3474 assert(&(*MF.begin()) == &PrologueMBB && "Shrink-wrapping not supported yet");
3475
3476 unsigned ScratchReg = GetScratchRegister(Is64Bit, IsLP64, MF, Primary: true);
3477 assert(!MF.getRegInfo().isLiveIn(ScratchReg) &&
3478 "Scratch register is live-in");
3479
3480 if (MF.getFunction().isVarArg())
3481 report_fatal_error(reason: "Segmented stacks do not support vararg functions.");
3482 if (!STI.isTargetLinux() && !STI.isTargetDarwin() && !STI.isTargetWin32() &&
3483 !STI.isTargetWin64() && !STI.isTargetFreeBSD() &&
3484 !STI.isTargetDragonFly())
3485 report_fatal_error(reason: "Segmented stacks not supported on this platform.");
3486
3487 // Eventually StackSize will be calculated by a link-time pass; which will
3488 // also decide whether checking code needs to be injected into this particular
3489 // prologue.
3490 StackSize = MFI.getStackSize();
3491
3492 if (!MFI.needsSplitStackProlog())
3493 return;
3494
3495 MachineBasicBlock *allocMBB = MF.CreateMachineBasicBlock();
3496 MachineBasicBlock *checkMBB = MF.CreateMachineBasicBlock();
3497 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
3498 bool IsNested = false;
3499
3500 // We need to know if the function has a nest argument only in 64 bit mode.
3501 if (Is64Bit)
3502 IsNested = HasNestArgument(MF: &MF);
3503
3504 // The MOV R10, RAX needs to be in a different block, since the RET we emit in
3505 // allocMBB needs to be last (terminating) instruction.
3506
3507 for (const auto &LI : PrologueMBB.liveins()) {
3508 allocMBB->addLiveIn(RegMaskPair: LI);
3509 checkMBB->addLiveIn(RegMaskPair: LI);
3510 }
3511
3512 if (IsNested)
3513 allocMBB->addLiveIn(PhysReg: IsLP64 ? X86::R10 : X86::R10D);
3514
3515 MF.push_front(MBB: allocMBB);
3516 MF.push_front(MBB: checkMBB);
3517
3518 // When the frame size is less than 256 we just compare the stack
3519 // boundary directly to the value of the stack pointer, per gcc.
3520 bool CompareStackPointer = StackSize < kSplitStackAvailable;
3521
3522 // Read the limit off the current stacklet off the stack_guard location.
3523 if (Is64Bit) {
3524 if (STI.isTargetLinux()) {
3525 TlsReg = X86::FS;
3526 TlsOffset = IsLP64 ? 0x70 : 0x40;
3527 } else if (STI.isTargetDarwin()) {
3528 TlsReg = X86::GS;
3529 TlsOffset = 0x60 + 90 * 8; // See pthread_machdep.h. Steal TLS slot 90.
3530 } else if (STI.isTargetWin64()) {
3531 TlsReg = X86::GS;
3532 TlsOffset = 0x28; // pvArbitrary, reserved for application use
3533 } else if (STI.isTargetFreeBSD()) {
3534 TlsReg = X86::FS;
3535 TlsOffset = 0x18;
3536 } else if (STI.isTargetDragonFly()) {
3537 TlsReg = X86::FS;
3538 TlsOffset = 0x20; // use tls_tcb.tcb_segstack
3539 } else {
3540 report_fatal_error(reason: "Segmented stacks not supported on this platform.");
3541 }
3542
3543 if (CompareStackPointer)
3544 ScratchReg = IsLP64 ? X86::RSP : X86::ESP;
3545 else
3546 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: IsLP64 ? X86::LEA64r : X86::LEA64_32r),
3547 DestReg: ScratchReg)
3548 .addReg(RegNo: X86::RSP)
3549 .addImm(Val: 1)
3550 .addReg(RegNo: 0)
3551 .addImm(Val: -StackSize)
3552 .addReg(RegNo: 0);
3553
3554 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: IsLP64 ? X86::CMP64rm : X86::CMP32rm))
3555 .addReg(RegNo: ScratchReg)
3556 .addReg(RegNo: 0)
3557 .addImm(Val: 1)
3558 .addReg(RegNo: 0)
3559 .addImm(Val: TlsOffset)
3560 .addReg(RegNo: TlsReg);
3561 } else {
3562 if (STI.isTargetLinux()) {
3563 TlsReg = X86::GS;
3564 TlsOffset = 0x30;
3565 } else if (STI.isTargetDarwin()) {
3566 TlsReg = X86::GS;
3567 TlsOffset = 0x48 + 90 * 4;
3568 } else if (STI.isTargetWin32()) {
3569 TlsReg = X86::FS;
3570 TlsOffset = 0x14; // pvArbitrary, reserved for application use
3571 } else if (STI.isTargetDragonFly()) {
3572 TlsReg = X86::FS;
3573 TlsOffset = 0x10; // use tls_tcb.tcb_segstack
3574 } else if (STI.isTargetFreeBSD()) {
3575 report_fatal_error(reason: "Segmented stacks not supported on FreeBSD i386.");
3576 } else {
3577 report_fatal_error(reason: "Segmented stacks not supported on this platform.");
3578 }
3579
3580 if (CompareStackPointer)
3581 ScratchReg = X86::ESP;
3582 else
3583 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: X86::LEA32r), DestReg: ScratchReg)
3584 .addReg(RegNo: X86::ESP)
3585 .addImm(Val: 1)
3586 .addReg(RegNo: 0)
3587 .addImm(Val: -StackSize)
3588 .addReg(RegNo: 0);
3589
3590 if (STI.isTargetLinux() || STI.isTargetWin32() || STI.isTargetWin64() ||
3591 STI.isTargetDragonFly()) {
3592 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: X86::CMP32rm))
3593 .addReg(RegNo: ScratchReg)
3594 .addReg(RegNo: 0)
3595 .addImm(Val: 0)
3596 .addReg(RegNo: 0)
3597 .addImm(Val: TlsOffset)
3598 .addReg(RegNo: TlsReg);
3599 } else if (STI.isTargetDarwin()) {
3600
3601 // TlsOffset doesn't fit into a mod r/m byte so we need an extra register.
3602 unsigned ScratchReg2;
3603 bool SaveScratch2;
3604 if (CompareStackPointer) {
3605 // The primary scratch register is available for holding the TLS offset.
3606 ScratchReg2 = GetScratchRegister(Is64Bit, IsLP64, MF, Primary: true);
3607 SaveScratch2 = false;
3608 } else {
3609 // Need to use a second register to hold the TLS offset
3610 ScratchReg2 = GetScratchRegister(Is64Bit, IsLP64, MF, Primary: false);
3611
3612 // Unfortunately, with fastcc the second scratch register may hold an
3613 // argument.
3614 SaveScratch2 = MF.getRegInfo().isLiveIn(Reg: ScratchReg2);
3615 }
3616
3617 // If Scratch2 is live-in then it needs to be saved.
3618 assert((!MF.getRegInfo().isLiveIn(ScratchReg2) || SaveScratch2) &&
3619 "Scratch register is live-in and not saved");
3620
3621 if (SaveScratch2)
3622 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH32r))
3623 .addReg(RegNo: ScratchReg2, Flags: RegState::Kill);
3624
3625 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: X86::MOV32ri), DestReg: ScratchReg2)
3626 .addImm(Val: TlsOffset);
3627 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: X86::CMP32rm))
3628 .addReg(RegNo: ScratchReg)
3629 .addReg(RegNo: ScratchReg2)
3630 .addImm(Val: 1)
3631 .addReg(RegNo: 0)
3632 .addImm(Val: 0)
3633 .addReg(RegNo: TlsReg);
3634
3635 if (SaveScratch2)
3636 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: X86::POP32r), DestReg: ScratchReg2);
3637 }
3638 }
3639
3640 // This jump is taken if SP >= (Stacklet Limit + Stack Space required).
3641 // It jumps to normal execution of the function body.
3642 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
3643 .addMBB(MBB: &PrologueMBB)
3644 .addImm(Val: X86::COND_A);
3645
3646 // On 32 bit we first push the arguments size and then the frame size. On 64
3647 // bit, we pass the stack frame size in r10 and the argument size in r11.
3648 if (Is64Bit) {
3649 // Functions with nested arguments use R10, so it needs to be saved across
3650 // the call to _morestack
3651
3652 const unsigned RegAX = IsLP64 ? X86::RAX : X86::EAX;
3653 const unsigned Reg10 = IsLP64 ? X86::R10 : X86::R10D;
3654 const unsigned Reg11 = IsLP64 ? X86::R11 : X86::R11D;
3655 const unsigned MOVrr = IsLP64 ? X86::MOV64rr : X86::MOV32rr;
3656
3657 if (IsNested)
3658 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: MOVrr), DestReg: RegAX).addReg(RegNo: Reg10);
3659
3660 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: X86::getMOVriOpcode(Use64BitReg: IsLP64, Imm: StackSize)),
3661 DestReg: Reg10)
3662 .addImm(Val: StackSize);
3663 BuildMI(BB: allocMBB, MIMD: DL,
3664 MCID: TII.get(Opcode: X86::getMOVriOpcode(Use64BitReg: IsLP64, Imm: X86FI->getArgumentStackSize())),
3665 DestReg: Reg11)
3666 .addImm(Val: X86FI->getArgumentStackSize());
3667 } else {
3668 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH32i))
3669 .addImm(Val: X86FI->getArgumentStackSize());
3670 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH32i)).addImm(Val: StackSize);
3671 }
3672
3673 // __morestack is in libgcc
3674 if (Is64Bit && MF.getTarget().getCodeModel() == CodeModel::Large) {
3675 // Under the large code model, we cannot assume that __morestack lives
3676 // within 2^31 bytes of the call site, so we cannot use pc-relative
3677 // addressing. We cannot perform the call via a temporary register,
3678 // as the rax register may be used to store the static chain, and all
3679 // other suitable registers may be either callee-save or used for
3680 // parameter passing. We cannot use the stack at this point either
3681 // because __morestack manipulates the stack directly.
3682 //
3683 // To avoid these issues, perform an indirect call via a read-only memory
3684 // location containing the address.
3685 //
3686 // This solution is not perfect, as it assumes that the .rodata section
3687 // is laid out within 2^31 bytes of each function body, but this seems
3688 // to be sufficient for JIT.
3689 // FIXME: Add retpoline support and remove the error here..
3690 if (STI.useIndirectThunkCalls())
3691 report_fatal_error(reason: "Emitting morestack calls on 64-bit with the large "
3692 "code model and thunks not yet implemented.");
3693 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: X86::CALL64m))
3694 .addReg(RegNo: X86::RIP)
3695 .addImm(Val: 0)
3696 .addReg(RegNo: 0)
3697 .addExternalSymbol(FnName: "__morestack_addr")
3698 .addReg(RegNo: 0);
3699 } else {
3700 if (Is64Bit)
3701 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: X86::CALL64pcrel32))
3702 .addExternalSymbol(FnName: "__morestack");
3703 else
3704 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: X86::CALLpcrel32))
3705 .addExternalSymbol(FnName: "__morestack");
3706 }
3707
3708 if (IsNested)
3709 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: X86::MORESTACK_RET_RESTORE_R10));
3710 else
3711 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: X86::MORESTACK_RET));
3712
3713 allocMBB->addSuccessor(Succ: &PrologueMBB);
3714
3715 checkMBB->addSuccessor(Succ: allocMBB, Prob: BranchProbability::getZero());
3716 checkMBB->addSuccessor(Succ: &PrologueMBB, Prob: BranchProbability::getOne());
3717
3718#ifdef EXPENSIVE_CHECKS
3719 MF.verify();
3720#endif
3721}
3722
3723/// Lookup an ERTS parameter in the !hipe.literals named metadata node.
3724/// HiPE provides Erlang Runtime System-internal parameters, such as PCB offsets
3725/// to fields it needs, through a named metadata node "hipe.literals" containing
3726/// name-value pairs.
3727static unsigned getHiPELiteral(NamedMDNode *HiPELiteralsMD,
3728 const StringRef LiteralName) {
3729 for (int i = 0, e = HiPELiteralsMD->getNumOperands(); i != e; ++i) {
3730 MDNode *Node = HiPELiteralsMD->getOperand(i);
3731 if (Node->getNumOperands() != 2)
3732 continue;
3733 MDString *NodeName = dyn_cast<MDString>(Val: Node->getOperand(I: 0));
3734 ValueAsMetadata *NodeVal = dyn_cast<ValueAsMetadata>(Val: Node->getOperand(I: 1));
3735 if (!NodeName || !NodeVal)
3736 continue;
3737 ConstantInt *ValConst = dyn_cast_or_null<ConstantInt>(Val: NodeVal->getValue());
3738 if (ValConst && NodeName->getString() == LiteralName) {
3739 return ValConst->getZExtValue();
3740 }
3741 }
3742
3743 report_fatal_error(reason: "HiPE literal " + LiteralName +
3744 " required but not provided");
3745}
3746
3747// Return true if there are no non-ehpad successors to MBB and there are no
3748// non-meta instructions between MBBI and MBB.end().
3749static bool blockEndIsUnreachable(const MachineBasicBlock &MBB,
3750 MachineBasicBlock::const_iterator MBBI) {
3751 return llvm::all_of(
3752 Range: MBB.successors(),
3753 P: [](const MachineBasicBlock *Succ) { return Succ->isEHPad(); }) &&
3754 std::all_of(first: MBBI, last: MBB.end(), pred: [](const MachineInstr &MI) {
3755 return MI.isMetaInstruction();
3756 });
3757}
3758
3759/// Erlang programs may need a special prologue to handle the stack size they
3760/// might need at runtime. That is because Erlang/OTP does not implement a C
3761/// stack but uses a custom implementation of hybrid stack/heap architecture.
3762/// (for more information see Eric Stenman's Ph.D. thesis:
3763/// http://publications.uu.se/uu/fulltext/nbn_se_uu_diva-2688.pdf)
3764///
3765/// CheckStack:
3766/// temp0 = sp - MaxStack
3767/// if( temp0 < SP_LIMIT(P) ) goto IncStack else goto OldStart
3768/// OldStart:
3769/// ...
3770/// IncStack:
3771/// call inc_stack # doubles the stack space
3772/// temp0 = sp - MaxStack
3773/// if( temp0 < SP_LIMIT(P) ) goto IncStack else goto OldStart
3774void X86FrameLowering::adjustForHiPEPrologue(
3775 MachineFunction &MF, MachineBasicBlock &PrologueMBB) const {
3776 MachineFrameInfo &MFI = MF.getFrameInfo();
3777 DebugLoc DL;
3778
3779 // To support shrink-wrapping we would need to insert the new blocks
3780 // at the right place and update the branches to PrologueMBB.
3781 assert(&(*MF.begin()) == &PrologueMBB && "Shrink-wrapping not supported yet");
3782
3783 // HiPE-specific values
3784 NamedMDNode *HiPELiteralsMD =
3785 MF.getFunction().getParent()->getNamedMetadata(Name: "hipe.literals");
3786 if (!HiPELiteralsMD)
3787 report_fatal_error(
3788 reason: "Can't generate HiPE prologue without runtime parameters");
3789 const unsigned HipeLeafWords = getHiPELiteral(
3790 HiPELiteralsMD, LiteralName: Is64Bit ? "AMD64_LEAF_WORDS" : "X86_LEAF_WORDS");
3791 const unsigned CCRegisteredArgs = Is64Bit ? 6 : 5;
3792 const unsigned Guaranteed = HipeLeafWords * SlotSize;
3793 unsigned CallerStkArity = MF.getFunction().arg_size() > CCRegisteredArgs
3794 ? MF.getFunction().arg_size() - CCRegisteredArgs
3795 : 0;
3796 unsigned MaxStack = MFI.getStackSize() + CallerStkArity * SlotSize + SlotSize;
3797
3798 assert(STI.isTargetLinux() &&
3799 "HiPE prologue is only supported on Linux operating systems.");
3800
3801 // Compute the largest caller's frame that is needed to fit the callees'
3802 // frames. This 'MaxStack' is computed from:
3803 //
3804 // a) the fixed frame size, which is the space needed for all spilled temps,
3805 // b) outgoing on-stack parameter areas, and
3806 // c) the minimum stack space this function needs to make available for the
3807 // functions it calls (a tunable ABI property).
3808 if (MFI.hasCalls()) {
3809 unsigned MoreStackForCalls = 0;
3810
3811 for (auto &MBB : MF) {
3812 for (auto &MI : MBB) {
3813 if (!MI.isCall())
3814 continue;
3815
3816 // Get callee operand.
3817 const MachineOperand &MO = MI.getOperand(i: 0);
3818
3819 // Only take account of global function calls (no closures etc.).
3820 if (!MO.isGlobal())
3821 continue;
3822
3823 const Function *F = dyn_cast<Function>(Val: MO.getGlobal());
3824 if (!F)
3825 continue;
3826
3827 // Do not update 'MaxStack' for primitive and built-in functions
3828 // (encoded with names either starting with "erlang."/"bif_" or not
3829 // having a ".", such as a simple <Module>.<Function>.<Arity>, or an
3830 // "_", such as the BIF "suspend_0") as they are executed on another
3831 // stack.
3832 if (F->getName().contains(Other: "erlang.") || F->getName().contains(Other: "bif_") ||
3833 F->getName().find_first_of(Chars: "._") == StringRef::npos)
3834 continue;
3835
3836 unsigned CalleeStkArity = F->arg_size() > CCRegisteredArgs
3837 ? F->arg_size() - CCRegisteredArgs
3838 : 0;
3839 if (HipeLeafWords - 1 > CalleeStkArity)
3840 MoreStackForCalls =
3841 std::max(a: MoreStackForCalls,
3842 b: (HipeLeafWords - 1 - CalleeStkArity) * SlotSize);
3843 }
3844 }
3845 MaxStack += MoreStackForCalls;
3846 }
3847
3848 // If the stack frame needed is larger than the guaranteed then runtime checks
3849 // and calls to "inc_stack_0" BIF should be inserted in the assembly prologue.
3850 if (MaxStack > Guaranteed) {
3851 MachineBasicBlock *stackCheckMBB = MF.CreateMachineBasicBlock();
3852 MachineBasicBlock *incStackMBB = MF.CreateMachineBasicBlock();
3853
3854 for (const auto &LI : PrologueMBB.liveins()) {
3855 stackCheckMBB->addLiveIn(RegMaskPair: LI);
3856 incStackMBB->addLiveIn(RegMaskPair: LI);
3857 }
3858
3859 MF.push_front(MBB: incStackMBB);
3860 MF.push_front(MBB: stackCheckMBB);
3861
3862 unsigned ScratchReg, SPReg, PReg, SPLimitOffset;
3863 unsigned LEAop, CMPop, CALLop;
3864 SPLimitOffset = getHiPELiteral(HiPELiteralsMD, LiteralName: "P_NSP_LIMIT");
3865 if (Is64Bit) {
3866 SPReg = X86::RSP;
3867 PReg = X86::RBP;
3868 LEAop = X86::LEA64r;
3869 CMPop = X86::CMP64rm;
3870 CALLop = X86::CALL64pcrel32;
3871 } else {
3872 SPReg = X86::ESP;
3873 PReg = X86::EBP;
3874 LEAop = X86::LEA32r;
3875 CMPop = X86::CMP32rm;
3876 CALLop = X86::CALLpcrel32;
3877 }
3878
3879 ScratchReg = GetScratchRegister(Is64Bit, IsLP64, MF, Primary: true);
3880 assert(!MF.getRegInfo().isLiveIn(ScratchReg) &&
3881 "HiPE prologue scratch register is live-in");
3882
3883 // Create new MBB for StackCheck:
3884 addRegOffset(MIB: BuildMI(BB: stackCheckMBB, MIMD: DL, MCID: TII.get(Opcode: LEAop), DestReg: ScratchReg), Reg: SPReg,
3885 isKill: false, Offset: -MaxStack);
3886 // SPLimitOffset is in a fixed heap location (pointed by BP).
3887 addRegOffset(MIB: BuildMI(BB: stackCheckMBB, MIMD: DL, MCID: TII.get(Opcode: CMPop)).addReg(RegNo: ScratchReg),
3888 Reg: PReg, isKill: false, Offset: SPLimitOffset);
3889 BuildMI(BB: stackCheckMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
3890 .addMBB(MBB: &PrologueMBB)
3891 .addImm(Val: X86::COND_AE);
3892
3893 // Create new MBB for IncStack:
3894 BuildMI(BB: incStackMBB, MIMD: DL, MCID: TII.get(Opcode: CALLop)).addExternalSymbol(FnName: "inc_stack_0");
3895 addRegOffset(MIB: BuildMI(BB: incStackMBB, MIMD: DL, MCID: TII.get(Opcode: LEAop), DestReg: ScratchReg), Reg: SPReg,
3896 isKill: false, Offset: -MaxStack);
3897 addRegOffset(MIB: BuildMI(BB: incStackMBB, MIMD: DL, MCID: TII.get(Opcode: CMPop)).addReg(RegNo: ScratchReg),
3898 Reg: PReg, isKill: false, Offset: SPLimitOffset);
3899 BuildMI(BB: incStackMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
3900 .addMBB(MBB: incStackMBB)
3901 .addImm(Val: X86::COND_LE);
3902
3903 stackCheckMBB->addSuccessor(Succ: &PrologueMBB, Prob: {99, 100});
3904 stackCheckMBB->addSuccessor(Succ: incStackMBB, Prob: {1, 100});
3905 incStackMBB->addSuccessor(Succ: &PrologueMBB, Prob: {99, 100});
3906 incStackMBB->addSuccessor(Succ: incStackMBB, Prob: {1, 100});
3907 }
3908#ifdef EXPENSIVE_CHECKS
3909 MF.verify();
3910#endif
3911}
3912
3913bool X86FrameLowering::adjustStackWithPops(MachineBasicBlock &MBB,
3914 MachineBasicBlock::iterator MBBI,
3915 const DebugLoc &DL,
3916 int Offset) const {
3917 if (Offset <= 0)
3918 return false;
3919
3920 if (Offset % SlotSize)
3921 return false;
3922
3923 int NumPops = Offset / SlotSize;
3924 // This is only worth it if we have at most 2 pops.
3925 if (NumPops != 1 && NumPops != 2)
3926 return false;
3927
3928 // Handle only the trivial case where the adjustment directly follows
3929 // a call. This is the most common one, anyway.
3930 if (MBBI == MBB.begin())
3931 return false;
3932 MachineBasicBlock::iterator Prev = std::prev(x: MBBI);
3933 if (!Prev->isCall() || !Prev->getOperand(i: 1).isRegMask())
3934 return false;
3935
3936 unsigned Regs[2];
3937 unsigned FoundRegs = 0;
3938
3939 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3940 const MachineOperand &RegMask = Prev->getOperand(i: 1);
3941
3942 auto &RegClass =
3943 Is64Bit ? X86::GR64_NOREX_NOSPRegClass : X86::GR32_NOREX_NOSPRegClass;
3944 // Try to find up to NumPops free registers.
3945 for (auto Candidate : RegClass) {
3946 // Poor man's liveness:
3947 // Since we're immediately after a call, any register that is clobbered
3948 // by the call and not defined by it can be considered dead.
3949 if (!RegMask.clobbersPhysReg(PhysReg: Candidate))
3950 continue;
3951
3952 // Don't clobber reserved registers
3953 if (MRI.isReserved(PhysReg: Candidate))
3954 continue;
3955
3956 bool IsDef = false;
3957 for (const MachineOperand &MO : Prev->implicit_operands()) {
3958 if (MO.isReg() && MO.isDef() &&
3959 TRI->isSuperOrSubRegisterEq(RegA: MO.getReg(), RegB: Candidate)) {
3960 IsDef = true;
3961 break;
3962 }
3963 }
3964
3965 if (IsDef)
3966 continue;
3967
3968 Regs[FoundRegs++] = Candidate;
3969 if (FoundRegs == (unsigned)NumPops)
3970 break;
3971 }
3972
3973 if (FoundRegs == 0)
3974 return false;
3975
3976 // If we found only one free register, but need two, reuse the same one twice.
3977 while (FoundRegs < (unsigned)NumPops)
3978 Regs[FoundRegs++] = Regs[0];
3979
3980 for (int i = 0; i < NumPops; ++i)
3981 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: STI.is64Bit() ? X86::POP64r : X86::POP32r),
3982 DestReg: Regs[i]);
3983
3984 return true;
3985}
3986
3987MachineBasicBlock::iterator X86FrameLowering::eliminateCallFramePseudoInstr(
3988 MachineFunction &MF, MachineBasicBlock &MBB,
3989 MachineBasicBlock::iterator I) const {
3990 bool reserveCallFrame = hasReservedCallFrame(MF);
3991 unsigned Opcode = I->getOpcode();
3992 bool isDestroy = Opcode == TII.getCallFrameDestroyOpcode();
3993 DebugLoc DL = I->getDebugLoc(); // copy DebugLoc as I will be erased.
3994 uint64_t Amount = TII.getFrameSize(I: *I);
3995 uint64_t InternalAmt = (isDestroy || Amount) ? TII.getFrameAdjustment(I: *I) : 0;
3996 I = MBB.erase(I);
3997 auto InsertPos = skipDebugInstructionsForward(It: I, End: MBB.end());
3998
3999 // Try to avoid emitting dead SP adjustments if the block end is unreachable,
4000 // typically because the function is marked noreturn (abort, throw,
4001 // assert_fail, etc).
4002 if (isDestroy && blockEndIsUnreachable(MBB, MBBI: I))
4003 return I;
4004
4005 if (!reserveCallFrame) {
4006 // If the stack pointer can be changed after prologue, turn the
4007 // adjcallstackup instruction into a 'sub ESP, <amt>' and the
4008 // adjcallstackdown instruction into 'add ESP, <amt>'
4009
4010 // We need to keep the stack aligned properly. To do this, we round the
4011 // amount of space needed for the outgoing arguments up to the next
4012 // alignment boundary.
4013 Amount = alignTo(Size: Amount, A: getStackAlign());
4014
4015 const Function &F = MF.getFunction();
4016 bool WindowsCFI = MF.getTarget().getMCAsmInfo().usesWindowsCFI();
4017 bool DwarfCFI = !WindowsCFI && MF.needsFrameMoves();
4018
4019 // If we have any exception handlers in this function, and we adjust
4020 // the SP before calls, we may need to indicate this to the unwinder
4021 // using GNU_ARGS_SIZE. Note that this may be necessary even when
4022 // Amount == 0, because the preceding function may have set a non-0
4023 // GNU_ARGS_SIZE.
4024 // TODO: We don't need to reset this between subsequent functions,
4025 // if it didn't change.
4026 bool HasDwarfEHHandlers = !WindowsCFI && !MF.getLandingPads().empty();
4027
4028 if (HasDwarfEHHandlers && !isDestroy &&
4029 MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences())
4030 BuildCFI(MBB, MBBI: InsertPos, DL,
4031 CFIInst: MCCFIInstruction::createGnuArgsSize(L: nullptr, Size: Amount));
4032
4033 if (Amount == 0)
4034 return I;
4035
4036 // Factor out the amount that gets handled inside the sequence
4037 // (Pushes of argument for frame setup, callee pops for frame destroy)
4038 Amount -= InternalAmt;
4039
4040 // TODO: This is needed only if we require precise CFA.
4041 // If this is a callee-pop calling convention, emit a CFA adjust for
4042 // the amount the callee popped.
4043 if (isDestroy && InternalAmt && DwarfCFI && !hasFP(MF))
4044 BuildCFI(MBB, MBBI: InsertPos, DL,
4045 CFIInst: MCCFIInstruction::createAdjustCfaOffset(L: nullptr, Adjustment: -InternalAmt));
4046
4047 // Add Amount to SP to destroy a frame, or subtract to setup.
4048 int64_t StackAdjustment = isDestroy ? Amount : -Amount;
4049 int64_t CfaAdjustment = StackAdjustment;
4050
4051 if (StackAdjustment) {
4052 // Merge with any previous or following adjustment instruction. Note: the
4053 // instructions merged with here do not have CFI, so their stack
4054 // adjustments do not feed into CfaAdjustment
4055
4056 auto CalcCfaAdjust = [&CfaAdjustment](MachineBasicBlock::iterator PI,
4057 int64_t Offset) {
4058 CfaAdjustment += Offset;
4059 };
4060 auto CalcNewOffset = [&StackAdjustment](int64_t Offset) {
4061 return StackAdjustment + Offset;
4062 };
4063 StackAdjustment =
4064 mergeSPUpdates(MBB, MBBI&: InsertPos, FoundStackAdjust: CalcCfaAdjust, CalcNewOffset, doMergeWithPrevious: true);
4065 StackAdjustment =
4066 mergeSPUpdates(MBB, MBBI&: InsertPos, FoundStackAdjust: CalcCfaAdjust, CalcNewOffset, doMergeWithPrevious: false);
4067
4068 if (StackAdjustment) {
4069 if (!(F.hasMinSize() &&
4070 adjustStackWithPops(MBB, MBBI: InsertPos, DL, Offset: StackAdjustment)))
4071 BuildStackAdjustment(MBB, MBBI: InsertPos, DL, Offset: StackAdjustment,
4072 /*InEpilogue=*/false);
4073 }
4074 }
4075
4076 if (DwarfCFI && !hasFP(MF) && CfaAdjustment) {
4077 // If we don't have FP, but need to generate unwind information,
4078 // we need to set the correct CFA offset after the stack adjustment.
4079 // How much we adjust the CFA offset depends on whether we're emitting
4080 // CFI only for EH purposes or for debugging. EH only requires the CFA
4081 // offset to be correct at each call site, while for debugging we want
4082 // it to be more precise.
4083
4084 // TODO: When not using precise CFA, we also need to adjust for the
4085 // InternalAmt here.
4086 BuildCFI(
4087 MBB, MBBI: InsertPos, DL,
4088 CFIInst: MCCFIInstruction::createAdjustCfaOffset(L: nullptr, Adjustment: -CfaAdjustment));
4089 }
4090
4091 return I;
4092 }
4093
4094 if (InternalAmt) {
4095 MachineBasicBlock::iterator CI = I;
4096 MachineBasicBlock::iterator B = MBB.begin();
4097 while (CI != B && !std::prev(x: CI)->isCall())
4098 --CI;
4099 BuildStackAdjustment(MBB, MBBI: CI, DL, Offset: -InternalAmt, /*InEpilogue=*/false);
4100 }
4101
4102 return I;
4103}
4104
4105bool X86FrameLowering::canUseAsPrologue(const MachineBasicBlock &MBB) const {
4106 assert(MBB.getParent() && "Block is not attached to a function!");
4107 const MachineFunction &MF = *MBB.getParent();
4108 if (!MBB.isLiveIn(Reg: X86::EFLAGS))
4109 return true;
4110
4111 // If stack probes have to loop inline or call, that will clobber EFLAGS.
4112 // FIXME: we could allow cases that will use emitStackProbeInlineGenericBlock.
4113 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
4114 const X86TargetLowering &TLI = *STI.getTargetLowering();
4115 if (TLI.hasInlineStackProbe(MF) || TLI.hasStackProbeSymbol(MF))
4116 return false;
4117
4118 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
4119 return !TRI->hasStackRealignment(MF) && !X86FI->hasSwiftAsyncContext();
4120}
4121
4122bool X86FrameLowering::canUseAsEpilogue(const MachineBasicBlock &MBB) const {
4123 assert(MBB.getParent() && "Block is not attached to a function!");
4124
4125 // Win64 has strict requirements in terms of epilogue and we are
4126 // not taking a chance at messing with them.
4127 // I.e., unless this block is already an exit block, we can't use
4128 // it as an epilogue.
4129 if (STI.isTargetWin64() && !MBB.succ_empty() && !MBB.isReturnBlock())
4130 return false;
4131
4132 // Swift async context epilogue has a BTR instruction that clobbers parts of
4133 // EFLAGS.
4134 const MachineFunction &MF = *MBB.getParent();
4135 if (MF.getInfo<X86MachineFunctionInfo>()->hasSwiftAsyncContext())
4136 return !flagsNeedToBePreservedBeforeTheTerminators(MBB);
4137
4138 if (canUseLEAForSPInEpilogue(MF: *MBB.getParent()))
4139 return true;
4140
4141 // If we cannot use LEA to adjust SP, we may need to use ADD, which
4142 // clobbers the EFLAGS. Check that we do not need to preserve it,
4143 // otherwise, conservatively assume this is not
4144 // safe to insert the epilogue here.
4145 return !flagsNeedToBePreservedBeforeTheTerminators(MBB);
4146}
4147
4148bool X86FrameLowering::enableShrinkWrapping(const MachineFunction &MF) const {
4149 // If we may need to emit frameless compact unwind information, give
4150 // up as this is currently broken: PR25614.
4151 bool CompactUnwind =
4152 MF.getContext().getObjectFileInfo()->getCompactUnwindSection() != nullptr;
4153 return (MF.getFunction().hasFnAttribute(Kind: Attribute::NoUnwind) || hasFP(MF) ||
4154 !CompactUnwind) &&
4155 // The lowering of segmented stack and HiPE only support entry
4156 // blocks as prologue blocks: PR26107. This limitation may be
4157 // lifted if we fix:
4158 // - adjustForSegmentedStacks
4159 // - adjustForHiPEPrologue
4160 MF.getFunction().getCallingConv() != CallingConv::HiPE &&
4161 !MF.shouldSplitStack();
4162}
4163
4164MachineBasicBlock::iterator X86FrameLowering::restoreWin32EHStackPointers(
4165 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
4166 const DebugLoc &DL, bool RestoreSP) const {
4167 assert(STI.isTargetWindowsMSVC() && "funclets only supported in MSVC env");
4168 assert(STI.isTargetWin32() && "EBP/ESI restoration only required on win32");
4169 assert(STI.is32Bit() && !Uses64BitFramePtr &&
4170 "restoring EBP/ESI on non-32-bit target");
4171
4172 MachineFunction &MF = *MBB.getParent();
4173 Register FramePtr = TRI->getFrameRegister(MF);
4174 Register BasePtr = TRI->getBaseRegister();
4175 WinEHFuncInfo &FuncInfo = *MF.getWinEHFuncInfo();
4176 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
4177 MachineFrameInfo &MFI = MF.getFrameInfo();
4178
4179 // FIXME: Don't set FrameSetup flag in catchret case.
4180
4181 int FI = FuncInfo.EHRegNodeFrameIndex;
4182 int EHRegSize = MFI.getObjectSize(ObjectIdx: FI);
4183
4184 if (RestoreSP) {
4185 // MOV32rm -EHRegSize(%ebp), %esp
4186 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV32rm), DestReg: X86::ESP),
4187 Reg: X86::EBP, isKill: true, Offset: -EHRegSize)
4188 .setMIFlag(MachineInstr::FrameSetup);
4189 }
4190
4191 Register UsedReg;
4192 int EHRegOffset = getFrameIndexReference(MF, FI, FrameReg&: UsedReg).getFixed();
4193 int EndOffset = -EHRegOffset - EHRegSize;
4194 FuncInfo.EHRegNodeEndOffset = EndOffset;
4195
4196 if (UsedReg == FramePtr) {
4197 // ADD $offset, %ebp
4198 unsigned ADDri = getADDriOpcode(IsLP64: false);
4199 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: ADDri), DestReg: FramePtr)
4200 .addReg(RegNo: FramePtr)
4201 .addImm(Val: EndOffset)
4202 .setMIFlag(MachineInstr::FrameSetup)
4203 ->getOperand(i: 3)
4204 .setIsDead();
4205 assert(EndOffset >= 0 &&
4206 "end of registration object above normal EBP position!");
4207 } else if (UsedReg == BasePtr) {
4208 // LEA offset(%ebp), %esi
4209 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::LEA32r), DestReg: BasePtr),
4210 Reg: FramePtr, isKill: false, Offset: EndOffset)
4211 .setMIFlag(MachineInstr::FrameSetup);
4212 // MOV32rm SavedEBPOffset(%esi), %ebp
4213 assert(X86FI->getHasSEHFramePtrSave());
4214 int Offset =
4215 getFrameIndexReference(MF, FI: X86FI->getSEHFramePtrSaveIndex(), FrameReg&: UsedReg)
4216 .getFixed();
4217 assert(UsedReg == BasePtr);
4218 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV32rm), DestReg: FramePtr),
4219 Reg: UsedReg, isKill: true, Offset)
4220 .setMIFlag(MachineInstr::FrameSetup);
4221 } else {
4222 llvm_unreachable("32-bit frames with WinEH must use FramePtr or BasePtr");
4223 }
4224 return MBBI;
4225}
4226
4227int X86FrameLowering::getInitialCFAOffset(const MachineFunction &MF) const {
4228 return TRI->getSlotSize();
4229}
4230
4231Register
4232X86FrameLowering::getInitialCFARegister(const MachineFunction &MF) const {
4233 return StackPtr;
4234}
4235
4236TargetFrameLowering::DwarfFrameBase
4237X86FrameLowering::getDwarfFrameBase(const MachineFunction &MF) const {
4238 const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
4239 Register FrameRegister = RI->getFrameRegister(MF);
4240 if (getInitialCFARegister(MF) == FrameRegister &&
4241 MF.getInfo<X86MachineFunctionInfo>()->hasCFIAdjustCfa()) {
4242 DwarfFrameBase FrameBase;
4243 FrameBase.Kind = DwarfFrameBase::CFA;
4244 FrameBase.Location.Offset =
4245 -MF.getFrameInfo().getStackSize() - getInitialCFAOffset(MF);
4246 return FrameBase;
4247 }
4248
4249 return DwarfFrameBase{.Kind: DwarfFrameBase::Register, .Location: {.Reg: FrameRegister}};
4250}
4251
4252namespace {
4253// Struct used by orderFrameObjects to help sort the stack objects.
4254struct X86FrameSortingObject {
4255 bool IsValid = false; // true if we care about this Object.
4256 unsigned ObjectIndex = 0; // Index of Object into MFI list.
4257 unsigned ObjectSize = 0; // Size of Object in bytes.
4258 Align ObjectAlignment = Align(1); // Alignment of Object in bytes.
4259 unsigned ObjectNumUses = 0; // Object static number of uses.
4260};
4261
4262// The comparison function we use for std::sort to order our local
4263// stack symbols. The current algorithm is to use an estimated
4264// "density". This takes into consideration the size and number of
4265// uses each object has in order to roughly minimize code size.
4266// So, for example, an object of size 16B that is referenced 5 times
4267// will get higher priority than 4 4B objects referenced 1 time each.
4268// It's not perfect and we may be able to squeeze a few more bytes out of
4269// it (for example : 0(esp) requires fewer bytes, symbols allocated at the
4270// fringe end can have special consideration, given their size is less
4271// important, etc.), but the algorithmic complexity grows too much to be
4272// worth the extra gains we get. This gets us pretty close.
4273// The final order leaves us with objects with highest priority going
4274// at the end of our list.
4275struct X86FrameSortingComparator {
4276 inline bool operator()(const X86FrameSortingObject &A,
4277 const X86FrameSortingObject &B) const {
4278 uint64_t DensityAScaled, DensityBScaled;
4279
4280 // For consistency in our comparison, all invalid objects are placed
4281 // at the end. This also allows us to stop walking when we hit the
4282 // first invalid item after it's all sorted.
4283 if (!A.IsValid)
4284 return false;
4285 if (!B.IsValid)
4286 return true;
4287
4288 // The density is calculated by doing :
4289 // (double)DensityA = A.ObjectNumUses / A.ObjectSize
4290 // (double)DensityB = B.ObjectNumUses / B.ObjectSize
4291 // Since this approach may cause inconsistencies in
4292 // the floating point <, >, == comparisons, depending on the floating
4293 // point model with which the compiler was built, we're going
4294 // to scale both sides by multiplying with
4295 // A.ObjectSize * B.ObjectSize. This ends up factoring away
4296 // the division and, with it, the need for any floating point
4297 // arithmetic.
4298 DensityAScaled = static_cast<uint64_t>(A.ObjectNumUses) *
4299 static_cast<uint64_t>(B.ObjectSize);
4300 DensityBScaled = static_cast<uint64_t>(B.ObjectNumUses) *
4301 static_cast<uint64_t>(A.ObjectSize);
4302
4303 // If the two densities are equal, prioritize highest alignment
4304 // objects. This allows for similar alignment objects
4305 // to be packed together (given the same density).
4306 // There's room for improvement here, also, since we can pack
4307 // similar alignment (different density) objects next to each
4308 // other to save padding. This will also require further
4309 // complexity/iterations, and the overall gain isn't worth it,
4310 // in general. Something to keep in mind, though.
4311 if (DensityAScaled == DensityBScaled)
4312 return A.ObjectAlignment < B.ObjectAlignment;
4313
4314 return DensityAScaled < DensityBScaled;
4315 }
4316};
4317} // namespace
4318
4319// Order the symbols in the local stack.
4320// We want to place the local stack objects in some sort of sensible order.
4321// The heuristic we use is to try and pack them according to static number
4322// of uses and size of object in order to minimize code size.
4323void X86FrameLowering::orderFrameObjects(
4324 const MachineFunction &MF, SmallVectorImpl<int> &ObjectsToAllocate) const {
4325 const MachineFrameInfo &MFI = MF.getFrameInfo();
4326
4327 // Don't waste time if there's nothing to do.
4328 if (ObjectsToAllocate.empty())
4329 return;
4330
4331 // Create an array of all MFI objects. We won't need all of these
4332 // objects, but we're going to create a full array of them to make
4333 // it easier to index into when we're counting "uses" down below.
4334 // We want to be able to easily/cheaply access an object by simply
4335 // indexing into it, instead of having to search for it every time.
4336 std::vector<X86FrameSortingObject> SortingObjects(MFI.getObjectIndexEnd());
4337
4338 // Walk the objects we care about and mark them as such in our working
4339 // struct.
4340 for (auto &Obj : ObjectsToAllocate) {
4341 SortingObjects[Obj].IsValid = true;
4342 SortingObjects[Obj].ObjectIndex = Obj;
4343 SortingObjects[Obj].ObjectAlignment = MFI.getObjectAlign(ObjectIdx: Obj);
4344 // Set the size.
4345 int ObjectSize = MFI.getObjectSize(ObjectIdx: Obj);
4346 if (ObjectSize == 0)
4347 // Variable size. Just use 4.
4348 SortingObjects[Obj].ObjectSize = 4;
4349 else
4350 SortingObjects[Obj].ObjectSize = ObjectSize;
4351 }
4352
4353 // Count the number of uses for each object.
4354 for (auto &MBB : MF) {
4355 for (auto &MI : MBB) {
4356 if (MI.isDebugInstr())
4357 continue;
4358 for (const MachineOperand &MO : MI.operands()) {
4359 // Check to see if it's a local stack symbol.
4360 if (!MO.isFI())
4361 continue;
4362 int Index = MO.getIndex();
4363 // Check to see if it falls within our range, and is tagged
4364 // to require ordering.
4365 if (Index >= 0 && Index < MFI.getObjectIndexEnd() &&
4366 SortingObjects[Index].IsValid)
4367 SortingObjects[Index].ObjectNumUses++;
4368 }
4369 }
4370 }
4371
4372 // Sort the objects using X86FrameSortingAlgorithm (see its comment for
4373 // info).
4374 llvm::stable_sort(Range&: SortingObjects, C: X86FrameSortingComparator());
4375
4376 // Now modify the original list to represent the final order that
4377 // we want. The order will depend on whether we're going to access them
4378 // from the stack pointer or the frame pointer. For SP, the list should
4379 // end up with the END containing objects that we want with smaller offsets.
4380 // For FP, it should be flipped.
4381 int i = 0;
4382 for (auto &Obj : SortingObjects) {
4383 // All invalid items are sorted at the end, so it's safe to stop.
4384 if (!Obj.IsValid)
4385 break;
4386 ObjectsToAllocate[i++] = Obj.ObjectIndex;
4387 }
4388
4389 // Flip it if we're accessing off of the FP.
4390 if (!TRI->hasStackRealignment(MF) && hasFP(MF))
4391 std::reverse(first: ObjectsToAllocate.begin(), last: ObjectsToAllocate.end());
4392}
4393
4394unsigned
4395X86FrameLowering::getWinEHParentFrameOffset(const MachineFunction &MF) const {
4396 // RDX, the parent frame pointer, is homed into 16(%rsp) in the prologue.
4397 unsigned Offset = 16;
4398 // RBP is immediately pushed.
4399 Offset += SlotSize;
4400 // All callee-saved registers are then pushed.
4401 Offset += MF.getInfo<X86MachineFunctionInfo>()->getCalleeSavedFrameSize();
4402 // Every funclet allocates enough stack space for the largest outgoing call.
4403 Offset += getWinEHFuncletFrameSize(MF);
4404 return Offset;
4405}
4406
4407void X86FrameLowering::processFunctionBeforeFrameFinalized(
4408 MachineFunction &MF, RegScavenger *RS) const {
4409 // Mark the function as not having WinCFI. We will set it back to true in
4410 // emitPrologue if it gets called and emits CFI.
4411 MF.setHasWinCFI(false);
4412
4413 MachineFrameInfo &MFI = MF.getFrameInfo();
4414 // If the frame is big enough that we might need to scavenge a register to
4415 // handle huge offsets, reserve a stack slot for that now.
4416 if (!isInt<32>(x: MFI.estimateStackSize(MF))) {
4417 int FI = MFI.CreateStackObject(Size: SlotSize, Alignment: Align(SlotSize), isSpillSlot: false);
4418 RS->addScavengingFrameIndex(FI);
4419 }
4420
4421 // If we are using Windows x64 CFI, ensure that the stack is always 8 byte
4422 // aligned. The format doesn't support misaligned stack adjustments.
4423 if (MF.getTarget().getMCAsmInfo().usesWindowsCFI())
4424 MF.getFrameInfo().ensureMaxAlignment(Alignment: Align(SlotSize));
4425
4426 // If this function isn't doing Win64-style C++ EH, we don't need to do
4427 // anything.
4428 if (STI.is64Bit() && MF.hasEHFunclets() &&
4429 classifyEHPersonality(Pers: MF.getFunction().getPersonalityFn()) ==
4430 EHPersonality::MSVC_CXX) {
4431 adjustFrameForMsvcCxxEh(MF);
4432 }
4433}
4434
4435void X86FrameLowering::adjustFrameForMsvcCxxEh(MachineFunction &MF) const {
4436 // Win64 C++ EH needs to allocate the UnwindHelp object at some fixed offset
4437 // relative to RSP after the prologue. Find the offset of the last fixed
4438 // object, so that we can allocate a slot immediately following it. If there
4439 // were no fixed objects, use offset -SlotSize, which is immediately after the
4440 // return address. Fixed objects have negative frame indices.
4441 MachineFrameInfo &MFI = MF.getFrameInfo();
4442 WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo();
4443 int64_t MinFixedObjOffset = -SlotSize;
4444 for (int I = MFI.getObjectIndexBegin(); I < 0; ++I)
4445 MinFixedObjOffset = std::min(a: MinFixedObjOffset, b: MFI.getObjectOffset(ObjectIdx: I));
4446
4447 for (WinEHTryBlockMapEntry &TBME : EHInfo.TryBlockMap) {
4448 for (WinEHHandlerType &H : TBME.HandlerArray) {
4449 int FrameIndex = H.CatchObj.FrameIndex;
4450 if ((FrameIndex != INT_MAX) && MFI.getObjectOffset(ObjectIdx: FrameIndex) == 0) {
4451 // Ensure alignment.
4452 unsigned Align = MFI.getObjectAlign(ObjectIdx: FrameIndex).value();
4453 MinFixedObjOffset -= std::abs(i: MinFixedObjOffset) % Align;
4454 MinFixedObjOffset -= MFI.getObjectSize(ObjectIdx: FrameIndex);
4455 MFI.setObjectOffset(ObjectIdx: FrameIndex, SPOffset: MinFixedObjOffset);
4456 }
4457 }
4458 }
4459
4460 // Ensure alignment.
4461 MinFixedObjOffset -= std::abs(i: MinFixedObjOffset) % 8;
4462 int64_t UnwindHelpOffset = MinFixedObjOffset - SlotSize;
4463 int UnwindHelpFI =
4464 MFI.CreateFixedObject(Size: SlotSize, SPOffset: UnwindHelpOffset, /*IsImmutable=*/false);
4465 EHInfo.UnwindHelpFrameIdx = UnwindHelpFI;
4466
4467 // Store -2 into UnwindHelp on function entry. We have to scan forwards past
4468 // other frame setup instructions.
4469 MachineBasicBlock &MBB = MF.front();
4470 auto MBBI = MBB.begin();
4471 while (MBBI != MBB.end() && MBBI->getFlag(Flag: MachineInstr::FrameSetup))
4472 ++MBBI;
4473
4474 DebugLoc DL = MBB.findDebugLoc(MBBI);
4475 addFrameReference(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64mi32)),
4476 FI: UnwindHelpFI)
4477 .addImm(Val: -2);
4478}
4479
4480void X86FrameLowering::processFunctionBeforeFrameIndicesReplaced(
4481 MachineFunction &MF, RegScavenger *RS) const {
4482 auto *X86FI = MF.getInfo<X86MachineFunctionInfo>();
4483
4484 if (STI.is32Bit() && MF.hasEHFunclets())
4485 restoreWinEHStackPointersInParent(MF);
4486 // We have emitted prolog and epilog. Don't need stack pointer saving
4487 // instruction any more.
4488 if (MachineInstr *MI = X86FI->getStackPtrSaveMI()) {
4489 MI->eraseFromParent();
4490 X86FI->setStackPtrSaveMI(nullptr);
4491 }
4492}
4493
4494void X86FrameLowering::restoreWinEHStackPointersInParent(
4495 MachineFunction &MF) const {
4496 // 32-bit functions have to restore stack pointers when control is transferred
4497 // back to the parent function. These blocks are identified as eh pads that
4498 // are not funclet entries.
4499 bool IsSEH = isAsynchronousEHPersonality(
4500 Pers: classifyEHPersonality(Pers: MF.getFunction().getPersonalityFn()));
4501 for (MachineBasicBlock &MBB : MF) {
4502 bool NeedsRestore = MBB.isEHPad() && !MBB.isEHFuncletEntry();
4503 if (NeedsRestore)
4504 restoreWin32EHStackPointers(MBB, MBBI: MBB.begin(), DL: DebugLoc(),
4505 /*RestoreSP=*/IsSEH);
4506 }
4507}
4508
4509// Compute the alignment gap between current SP after spilling FP/BP and the
4510// next properly aligned stack offset.
4511static int computeFPBPAlignmentGap(MachineFunction &MF,
4512 const TargetRegisterClass *RC,
4513 unsigned NumSpilledRegs) {
4514 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
4515 unsigned AllocSize = TRI->getSpillSize(RC: *RC) * NumSpilledRegs;
4516 Align StackAlign = MF.getSubtarget().getFrameLowering()->getStackAlign();
4517 unsigned AlignedSize = alignTo(Size: AllocSize, A: StackAlign);
4518 return AlignedSize - AllocSize;
4519}
4520
4521void X86FrameLowering::spillFPBPUsingSP(MachineFunction &MF,
4522 MachineBasicBlock::iterator BeforeMI,
4523 Register FP, Register BP,
4524 int SPAdjust) const {
4525 assert(FP.isValid() || BP.isValid());
4526
4527 MachineBasicBlock *MBB = BeforeMI->getParent();
4528 DebugLoc DL = BeforeMI->getDebugLoc();
4529
4530 // Spill FP.
4531 if (FP.isValid()) {
4532 BuildMI(BB&: *MBB, I: BeforeMI, MIMD: DL,
4533 MCID: TII.get(Opcode: getPUSHOpcode(ST: MF.getSubtarget<X86Subtarget>())))
4534 .addReg(RegNo: FP);
4535 }
4536
4537 // Spill BP.
4538 if (BP.isValid()) {
4539 BuildMI(BB&: *MBB, I: BeforeMI, MIMD: DL,
4540 MCID: TII.get(Opcode: getPUSHOpcode(ST: MF.getSubtarget<X86Subtarget>())))
4541 .addReg(RegNo: BP);
4542 }
4543
4544 // Make sure SP is aligned.
4545 if (SPAdjust)
4546 emitSPUpdate(MBB&: *MBB, MBBI&: BeforeMI, DL, NumBytes: -SPAdjust, InEpilogue: false);
4547
4548 // Emit unwinding information.
4549 if (FP.isValid() && needsDwarfCFI(MF)) {
4550 // Emit .cfi_remember_state to remember old frame.
4551 unsigned CFIIndex =
4552 MF.addFrameInst(Inst: MCCFIInstruction::createRememberState(L: nullptr));
4553 BuildMI(BB&: *MBB, I: BeforeMI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::CFI_INSTRUCTION))
4554 .addCFIIndex(CFIIndex);
4555
4556 // Setup new CFA value with DW_CFA_def_cfa_expression:
4557 // DW_OP_breg7+offset, DW_OP_deref, DW_OP_consts 16, DW_OP_plus
4558 SmallString<64> CfaExpr;
4559 uint8_t buffer[16];
4560 int Offset = SPAdjust;
4561 if (BP.isValid())
4562 Offset += TRI->getSpillSize(RC: *TRI->getMinimalPhysRegClass(Reg: BP));
4563 // If BeforeMI is a frame setup instruction, we need to adjust the position
4564 // and offset of the new cfi instruction.
4565 if (TII.isFrameSetup(I: *BeforeMI)) {
4566 Offset += alignTo(Size: TII.getFrameSize(I: *BeforeMI), A: getStackAlign());
4567 BeforeMI = std::next(x: BeforeMI);
4568 }
4569 Register StackPtr = TRI->getStackRegister();
4570 if (STI.isTarget64BitILP32())
4571 StackPtr = Register(getX86SubSuperRegister(Reg: StackPtr, Size: 64));
4572 unsigned DwarfStackPtr = TRI->getDwarfRegNum(Reg: StackPtr, isEH: true);
4573 CfaExpr.push_back(Elt: (uint8_t)(dwarf::DW_OP_breg0 + DwarfStackPtr));
4574 CfaExpr.append(in_start: buffer, in_end: buffer + encodeSLEB128(Value: Offset, p: buffer));
4575 CfaExpr.push_back(Elt: dwarf::DW_OP_deref);
4576 CfaExpr.push_back(Elt: dwarf::DW_OP_consts);
4577 CfaExpr.append(in_start: buffer, in_end: buffer + encodeSLEB128(Value: SlotSize * 2, p: buffer));
4578 CfaExpr.push_back(Elt: (uint8_t)dwarf::DW_OP_plus);
4579
4580 SmallString<64> DefCfaExpr;
4581 DefCfaExpr.push_back(Elt: dwarf::DW_CFA_def_cfa_expression);
4582 DefCfaExpr.append(in_start: buffer, in_end: buffer + encodeSLEB128(Value: CfaExpr.size(), p: buffer));
4583 DefCfaExpr.append(RHS: CfaExpr.str());
4584 BuildCFI(MBB&: *MBB, MBBI: BeforeMI, DL,
4585 CFIInst: MCCFIInstruction::createEscape(L: nullptr, Vals: DefCfaExpr.str()),
4586 Flag: MachineInstr::FrameSetup);
4587 }
4588}
4589
4590void X86FrameLowering::restoreFPBPUsingSP(MachineFunction &MF,
4591 MachineBasicBlock::iterator AfterMI,
4592 Register FP, Register BP,
4593 int SPAdjust) const {
4594 assert(FP.isValid() || BP.isValid());
4595
4596 // Adjust SP so it points to spilled FP or BP.
4597 MachineBasicBlock *MBB = AfterMI->getParent();
4598 MachineBasicBlock::iterator Pos = std::next(x: AfterMI);
4599 DebugLoc DL = AfterMI->getDebugLoc();
4600 if (SPAdjust)
4601 emitSPUpdate(MBB&: *MBB, MBBI&: Pos, DL, NumBytes: SPAdjust, InEpilogue: false);
4602
4603 // Restore BP.
4604 if (BP.isValid()) {
4605 BuildMI(BB&: *MBB, I: Pos, MIMD: DL,
4606 MCID: TII.get(Opcode: getPOPOpcode(ST: MF.getSubtarget<X86Subtarget>())), DestReg: BP);
4607 }
4608
4609 // Restore FP.
4610 if (FP.isValid()) {
4611 BuildMI(BB&: *MBB, I: Pos, MIMD: DL,
4612 MCID: TII.get(Opcode: getPOPOpcode(ST: MF.getSubtarget<X86Subtarget>())), DestReg: FP);
4613
4614 // Emit unwinding information.
4615 if (needsDwarfCFI(MF)) {
4616 // Restore original frame with .cfi_restore_state.
4617 unsigned CFIIndex =
4618 MF.addFrameInst(Inst: MCCFIInstruction::createRestoreState(L: nullptr));
4619 BuildMI(BB&: *MBB, I: Pos, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::CFI_INSTRUCTION))
4620 .addCFIIndex(CFIIndex);
4621 }
4622 }
4623}
4624
4625void X86FrameLowering::saveAndRestoreFPBPUsingSP(
4626 MachineFunction &MF, MachineBasicBlock::iterator BeforeMI,
4627 MachineBasicBlock::iterator AfterMI, bool SpillFP, bool SpillBP) const {
4628 assert(SpillFP || SpillBP);
4629
4630 Register FP, BP;
4631 const TargetRegisterClass *RC;
4632 unsigned NumRegs = 0;
4633
4634 if (SpillFP) {
4635 FP = TRI->getFrameRegister(MF);
4636 if (STI.isTarget64BitILP32())
4637 FP = Register(getX86SubSuperRegister(Reg: FP, Size: 64));
4638 RC = TRI->getMinimalPhysRegClass(Reg: FP);
4639 ++NumRegs;
4640 }
4641 if (SpillBP) {
4642 BP = TRI->getBaseRegister();
4643 if (STI.isTarget64BitILP32())
4644 BP = Register(getX86SubSuperRegister(Reg: BP, Size: 64));
4645 RC = TRI->getMinimalPhysRegClass(Reg: BP);
4646 ++NumRegs;
4647 }
4648 int SPAdjust = computeFPBPAlignmentGap(MF, RC, NumSpilledRegs: NumRegs);
4649
4650 spillFPBPUsingSP(MF, BeforeMI, FP, BP, SPAdjust);
4651 restoreFPBPUsingSP(MF, AfterMI, FP, BP, SPAdjust);
4652}
4653
4654bool X86FrameLowering::skipSpillFPBP(
4655 MachineFunction &MF, MachineBasicBlock::reverse_iterator &MI) const {
4656 if (MI->getOpcode() == X86::LCMPXCHG16B_SAVE_RBX) {
4657 // The pseudo instruction LCMPXCHG16B_SAVE_RBX is generated in the form
4658 // SaveRbx = COPY RBX
4659 // SaveRbx = LCMPXCHG16B_SAVE_RBX ..., SaveRbx, implicit-def rbx
4660 // And later LCMPXCHG16B_SAVE_RBX is expanded to restore RBX from SaveRbx.
4661 // We should skip this instruction sequence.
4662 int FI;
4663 Register Reg;
4664 while (!(MI->getOpcode() == TargetOpcode::COPY &&
4665 MI->getOperand(i: 1).getReg() == X86::RBX) &&
4666 !((Reg = TII.isStoreToStackSlot(MI: *MI, FrameIndex&: FI)) && Reg == X86::RBX))
4667 ++MI;
4668 return true;
4669 }
4670 return false;
4671}
4672
4673static bool isFPBPAccess(const MachineInstr &MI, Register FP, Register BP,
4674 const TargetRegisterInfo *TRI, bool &AccessFP,
4675 bool &AccessBP) {
4676 AccessFP = AccessBP = false;
4677 if (FP) {
4678 if (MI.findRegisterUseOperandIdx(Reg: FP, TRI, isKill: false) != -1 ||
4679 MI.findRegisterDefOperandIdx(Reg: FP, TRI, isDead: false, Overlap: true) != -1)
4680 AccessFP = true;
4681 }
4682 if (BP) {
4683 if (MI.findRegisterUseOperandIdx(Reg: BP, TRI, isKill: false) != -1 ||
4684 MI.findRegisterDefOperandIdx(Reg: BP, TRI, isDead: false, Overlap: true) != -1)
4685 AccessBP = true;
4686 }
4687 return AccessFP || AccessBP;
4688}
4689
4690// Invoke instruction has been lowered to normal function call. We try to figure
4691// out if MI comes from Invoke.
4692// Do we have any better method?
4693static bool isInvoke(const MachineInstr &MI, bool InsideEHLabels) {
4694 if (!MI.isCall())
4695 return false;
4696 if (InsideEHLabels)
4697 return true;
4698
4699 const MachineBasicBlock *MBB = MI.getParent();
4700 if (!MBB->hasEHPadSuccessor())
4701 return false;
4702
4703 // Check if there is another call instruction from MI to the end of MBB.
4704 MachineBasicBlock::const_iterator MBBI = MI, ME = MBB->end();
4705 for (++MBBI; MBBI != ME; ++MBBI)
4706 if (MBBI->isCall())
4707 return false;
4708 return true;
4709}
4710
4711/// Given the live range of FP or BP (DefMI, KillMI), check if there is any
4712/// interfered stack access in the range, usually generated by register spill.
4713void X86FrameLowering::checkInterferedAccess(
4714 MachineFunction &MF, MachineBasicBlock::reverse_iterator DefMI,
4715 MachineBasicBlock::reverse_iterator KillMI, bool SpillFP,
4716 bool SpillBP) const {
4717 if (DefMI == KillMI)
4718 return;
4719 if (TRI->hasBasePointer(MF)) {
4720 if (!SpillBP)
4721 return;
4722 } else {
4723 if (!SpillFP)
4724 return;
4725 }
4726
4727 auto MI = KillMI;
4728 while (MI != DefMI) {
4729 if (any_of(Range: MI->operands(),
4730 P: [](const MachineOperand &MO) { return MO.isFI(); }))
4731 MF.getContext().reportError(L: SMLoc(),
4732 Msg: "Interference usage of base pointer/frame "
4733 "pointer.");
4734 MI++;
4735 }
4736}
4737
4738/// If a function uses base pointer and the base pointer is clobbered by inline
4739/// asm, RA doesn't detect this case, and after the inline asm, the base pointer
4740/// contains garbage value.
4741/// For example if a 32b x86 function uses base pointer esi, and esi is
4742/// clobbered by following inline asm
4743/// asm("rep movsb" : "+D"(ptr), "+S"(x), "+c"(c)::"memory");
4744/// We need to save esi before the asm and restore it after the asm.
4745///
4746/// The problem can also occur to frame pointer if there is a function call, and
4747/// the callee uses a different calling convention and clobbers the fp.
4748///
4749/// Because normal frame objects (spill slots) are accessed through fp/bp
4750/// register, so we can't spill fp/bp to normal spill slots.
4751///
4752/// FIXME: There are 2 possible enhancements:
4753/// 1. In many cases there are different physical registers not clobbered by
4754/// inline asm, we can use one of them as base pointer. Or use a virtual
4755/// register as base pointer and let RA allocate a physical register to it.
4756/// 2. If there is no other instructions access stack with fp/bp from the
4757/// inline asm to the epilog, and no cfi requirement for a correct fp, we can
4758/// skip the save and restore operations.
4759void X86FrameLowering::spillFPBP(MachineFunction &MF) const {
4760 Register FP, BP;
4761 const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering();
4762 if (TFI.hasFP(MF))
4763 FP = TRI->getFrameRegister(MF);
4764 if (TRI->hasBasePointer(MF))
4765 BP = TRI->getBaseRegister();
4766
4767 // Currently only inline asm and function call can clobbers fp/bp. So we can
4768 // do some quick test and return early.
4769 if (!MF.hasInlineAsm()) {
4770 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
4771 if (!X86FI->getFPClobberedByCall())
4772 FP = 0;
4773 if (!X86FI->getBPClobberedByCall())
4774 BP = 0;
4775 }
4776 if (!FP && !BP)
4777 return;
4778
4779 for (MachineBasicBlock &MBB : MF) {
4780 bool InsideEHLabels = false;
4781 auto MI = MBB.rbegin(), ME = MBB.rend();
4782 auto TermMI = MBB.getFirstTerminator();
4783 if (TermMI == MBB.begin())
4784 continue;
4785 MI = *(std::prev(x: TermMI));
4786
4787 while (MI != ME) {
4788 // Skip frame setup/destroy instructions.
4789 // Skip Invoke (call inside try block) instructions.
4790 // Skip instructions handled by target.
4791 if (MI->getFlag(Flag: MachineInstr::MIFlag::FrameSetup) ||
4792 MI->getFlag(Flag: MachineInstr::MIFlag::FrameDestroy) ||
4793 isInvoke(MI: *MI, InsideEHLabels) || skipSpillFPBP(MF, MI)) {
4794 ++MI;
4795 continue;
4796 }
4797
4798 if (MI->getOpcode() == TargetOpcode::EH_LABEL) {
4799 InsideEHLabels = !InsideEHLabels;
4800 ++MI;
4801 continue;
4802 }
4803
4804 bool AccessFP, AccessBP;
4805 // Check if fp or bp is used in MI.
4806 if (!isFPBPAccess(MI: *MI, FP, BP, TRI, AccessFP, AccessBP)) {
4807 ++MI;
4808 continue;
4809 }
4810
4811 // Look for the range [DefMI, KillMI] in which fp or bp is defined and
4812 // used.
4813 bool FPLive = false, BPLive = false;
4814 bool SpillFP = false, SpillBP = false;
4815 auto DefMI = MI, KillMI = MI;
4816 do {
4817 SpillFP |= AccessFP;
4818 SpillBP |= AccessBP;
4819
4820 // Maintain FPLive and BPLive.
4821 if (FPLive && MI->findRegisterDefOperandIdx(Reg: FP, TRI, isDead: false, Overlap: true) != -1)
4822 FPLive = false;
4823 if (FP && MI->findRegisterUseOperandIdx(Reg: FP, TRI, isKill: false) != -1)
4824 FPLive = true;
4825 if (BPLive && MI->findRegisterDefOperandIdx(Reg: BP, TRI, isDead: false, Overlap: true) != -1)
4826 BPLive = false;
4827 if (BP && MI->findRegisterUseOperandIdx(Reg: BP, TRI, isKill: false) != -1)
4828 BPLive = true;
4829
4830 DefMI = MI++;
4831 } while ((MI != ME) &&
4832 (FPLive || BPLive ||
4833 isFPBPAccess(MI: *MI, FP, BP, TRI, AccessFP, AccessBP)));
4834
4835 // Don't need to save/restore if FP is accessed through llvm.frameaddress.
4836 if (FPLive && !SpillBP)
4837 continue;
4838
4839 // If the bp is clobbered by a call, we should save and restore outside of
4840 // the frame setup instructions.
4841 if (KillMI->isCall() && DefMI != ME) {
4842 auto FrameSetup = std::next(x: DefMI);
4843 // Look for frame setup instruction toward the start of the BB.
4844 // If we reach another call instruction, it means no frame setup
4845 // instruction for the current call instruction.
4846 while (FrameSetup != ME && !TII.isFrameSetup(I: *FrameSetup) &&
4847 !FrameSetup->isCall())
4848 ++FrameSetup;
4849 // If a frame setup instruction is found, we need to find out the
4850 // corresponding frame destroy instruction.
4851 if (FrameSetup != ME && TII.isFrameSetup(I: *FrameSetup) &&
4852 (TII.getFrameSize(I: *FrameSetup) ||
4853 TII.getFrameAdjustment(I: *FrameSetup))) {
4854 while (!TII.isFrameInstr(I: *KillMI))
4855 --KillMI;
4856 DefMI = FrameSetup;
4857 MI = DefMI;
4858 ++MI;
4859 }
4860 }
4861
4862 checkInterferedAccess(MF, DefMI, KillMI, SpillFP, SpillBP);
4863
4864 // Call target function to spill and restore FP and BP registers.
4865 saveAndRestoreFPBPUsingSP(MF, BeforeMI: &(*DefMI), AfterMI: &(*KillMI), SpillFP, SpillBP);
4866 }
4867 }
4868}
4869