1//===-- X86FrameLowering.cpp - X86 Frame Information ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the X86 implementation of TargetFrameLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "X86FrameLowering.h"
14#include "MCTargetDesc/X86MCTargetDesc.h"
15#include "X86.h"
16#include "X86InstrBuilder.h"
17#include "X86InstrInfo.h"
18#include "X86MachineFunctionInfo.h"
19#include "X86Subtarget.h"
20#include "X86TargetMachine.h"
21#include "llvm/ADT/Statistic.h"
22#include "llvm/CodeGen/LivePhysRegs.h"
23#include "llvm/CodeGen/MachineFrameInfo.h"
24#include "llvm/CodeGen/MachineFunction.h"
25#include "llvm/CodeGen/MachineInstrBuilder.h"
26#include "llvm/CodeGen/MachineModuleInfo.h"
27#include "llvm/CodeGen/MachineRegisterInfo.h"
28#include "llvm/CodeGen/RegisterScavenging.h"
29#include "llvm/CodeGen/WinEHFuncInfo.h"
30#include "llvm/IR/DataLayout.h"
31#include "llvm/IR/EHPersonalities.h"
32#include "llvm/IR/Function.h"
33#include "llvm/IR/Module.h"
34#include "llvm/MC/MCAsmInfo.h"
35#include "llvm/MC/MCObjectFileInfo.h"
36#include "llvm/MC/MCSymbol.h"
37#include "llvm/Support/LEB128.h"
38#include "llvm/Target/TargetOptions.h"
39#include <cstdlib>
40
41#define DEBUG_TYPE "x86-fl"
42
43STATISTIC(NumFrameLoopProbe, "Number of loop stack probes used in prologue");
44STATISTIC(NumFrameExtraProbe,
45 "Number of extra stack probes generated in prologue");
46STATISTIC(NumFunctionUsingPush2Pop2, "Number of functions using push2/pop2");
47
48using namespace llvm;
49
50bool llvm::requireWinX64UnwindV3(const MachineFunction &MF) {
51 const Function &Fn = MF.getFunction();
52
53 // Whole module is in V3 mode.
54 if (Fn.getParent()->getWinX64EHUnwindMode() == WinX64EHUnwindMode::V3)
55 return true;
56
57 // Otherwise promote a function that may use EGPR (R16-R31), which V1/V2
58 // unwind codes cannot encode. The per-function "+egpr" feature is the signal,
59 // so an auto-dispatch APX clone gets V3 while the baseline clone stays on the
60 // module default. We conservatively promote any egpr function rather than
61 // checking for an actual EGPR save, keeping this a cheap query. (PUSH2/POP2
62 // does not need V3: V1/V2 describe a PUSH2 as two SEH_PushReg codes.)
63 return Fn.needsUnwindTableEntry() &&
64 MF.getSubtarget<X86Subtarget>().hasEGPR();
65}
66
67static const TargetRegisterClass *
68getCalleeSavedSpillRC(MCRegister Reg, const X86Subtarget &STI,
69 const TargetRegisterInfo &TRI) {
70 if (X86::VK16RegClass.contains(Reg))
71 return STI.hasBWI() ? &X86::VK64RegClass : &X86::VK16RegClass;
72 return TRI.getMinimalPhysRegClass(Reg);
73}
74
75X86FrameLowering::X86FrameLowering(const X86Subtarget &STI,
76 MaybeAlign StackAlignOverride)
77 : TargetFrameLowering(StackGrowsDown, StackAlignOverride.valueOrOne(),
78 STI.is64Bit() ? -8 : -4),
79 STI(STI), TII(*STI.getInstrInfo()), TRI(STI.getRegisterInfo()) {
80 // Cache a bunch of frame-related predicates for this subtarget.
81 SlotSize = TRI->getSlotSize();
82 assert(SlotSize == 4 || SlotSize == 8);
83 Is64Bit = STI.is64Bit();
84 IsLP64 = STI.isTarget64BitLP64();
85 // standard x86_64 uses 64-bit frame/stack pointers, x32 - 32-bit.
86 Uses64BitFramePtr = STI.isTarget64BitLP64();
87 StackPtr = TRI->getStackRegister();
88}
89
90bool X86FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
91 return !MF.getFrameInfo().hasVarSizedObjects() &&
92 !MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences() &&
93 !MF.getInfo<X86MachineFunctionInfo>()->hasPreallocatedCall();
94}
95
96/// canSimplifyCallFramePseudos - If there is a reserved call frame, the
97/// call frame pseudos can be simplified. Having a FP, as in the default
98/// implementation, is not sufficient here since we can't always use it.
99/// Use a more nuanced condition.
100bool X86FrameLowering::canSimplifyCallFramePseudos(
101 const MachineFunction &MF) const {
102 return hasReservedCallFrame(MF) ||
103 MF.getInfo<X86MachineFunctionInfo>()->hasPreallocatedCall() ||
104 (hasFP(MF) && !TRI->hasStackRealignment(MF)) ||
105 TRI->hasBasePointer(MF);
106}
107
108// needsFrameIndexResolution - Do we need to perform FI resolution for
109// this function. Normally, this is required only when the function
110// has any stack objects. However, FI resolution actually has another job,
111// not apparent from the title - it resolves callframesetup/destroy
112// that were not simplified earlier.
113// So, this is required for x86 functions that have push sequences even
114// when there are no stack objects.
115bool X86FrameLowering::needsFrameIndexResolution(
116 const MachineFunction &MF) const {
117 return MF.getFrameInfo().hasStackObjects() ||
118 MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences();
119}
120
121/// hasFPImpl - Return true if the specified function should have a dedicated
122/// frame pointer register. This is true if the function has variable sized
123/// allocas or if frame pointer elimination is disabled.
124bool X86FrameLowering::hasFPImpl(const MachineFunction &MF) const {
125 const MachineFrameInfo &MFI = MF.getFrameInfo();
126 return (MF.disableFramePointerElim() || TRI->hasStackRealignment(MF) ||
127 MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken() ||
128 MFI.hasOpaqueSPAdjustment() ||
129 MF.getInfo<X86MachineFunctionInfo>()->getForceFramePointer() ||
130 MF.getInfo<X86MachineFunctionInfo>()->hasPreallocatedCall() ||
131 MF.callsUnwindInit() || MF.hasEHFunclets() || MF.callsEHReturn() ||
132 MFI.hasStackMap() || MFI.hasPatchPoint() ||
133 (MFI.hasCopyImplyingStackAdjustment() && MF.usesWindowsUnwindInfo()));
134}
135
136static unsigned getSUBriOpcode(bool IsLP64) {
137 return IsLP64 ? X86::SUB64ri32 : X86::SUB32ri;
138}
139
140static unsigned getADDriOpcode(bool IsLP64) {
141 return IsLP64 ? X86::ADD64ri32 : X86::ADD32ri;
142}
143
144static unsigned getSUBrrOpcode(bool IsLP64) {
145 return IsLP64 ? X86::SUB64rr : X86::SUB32rr;
146}
147
148static unsigned getADDrrOpcode(bool IsLP64) {
149 return IsLP64 ? X86::ADD64rr : X86::ADD32rr;
150}
151
152static unsigned getANDriOpcode(bool IsLP64, int64_t Imm) {
153 return IsLP64 ? X86::AND64ri32 : X86::AND32ri;
154}
155
156static unsigned getLEArOpcode(bool IsLP64) {
157 return IsLP64 ? X86::LEA64r : X86::LEA32r;
158}
159
160// Push-Pop Acceleration (PPX) hint is used to indicate that the POP reads the
161// value written by the PUSH from the stack. The processor tracks these marked
162// instructions internally and fast-forwards register data between matching PUSH
163// and POP instructions, without going through memory or through the training
164// loop of the Fast Store Forwarding Predictor (FSFP). Instead, a more efficient
165// memory-renaming optimization can be used.
166//
167// The PPX hint is purely a performance hint. Instructions with this hint have
168// the same functional semantics as those without. PPX hints set by the
169// compiler that violate the balancing rule may turn off the PPX optimization,
170// but they will not affect program semantics.
171//
172// Hence, PPX is used for balanced spill/reloads (Exceptions and setjmp/longjmp
173// are not considered).
174//
175// PUSH2 and POP2 are instructions for (respectively) pushing/popping 2
176// GPRs at a time to/from the stack.
177static unsigned getPUSHOpcode(const X86Subtarget &ST) {
178 return ST.is64Bit() ? (ST.hasPPX() ? X86::PUSHP64r : X86::PUSH64r)
179 : X86::PUSH32r;
180}
181static unsigned getPOPOpcode(const X86Subtarget &ST) {
182 return ST.is64Bit() ? (ST.hasPPX() ? X86::POPP64r : X86::POP64r)
183 : X86::POP32r;
184}
185static unsigned getPUSH2Opcode(const X86Subtarget &ST) {
186 return ST.hasPPX() ? X86::PUSH2P : X86::PUSH2;
187}
188static unsigned getPOP2Opcode(const X86Subtarget &ST) {
189 return ST.hasPPX() ? X86::POP2P : X86::POP2;
190}
191
192static bool isEAXLiveIn(MachineBasicBlock &MBB) {
193 for (MachineBasicBlock::RegisterMaskPair RegMask : MBB.liveins()) {
194 MCRegister Reg = RegMask.PhysReg;
195
196 if (Reg == X86::RAX || Reg == X86::EAX || Reg == X86::AX ||
197 Reg == X86::AH || Reg == X86::AL)
198 return true;
199 }
200
201 return false;
202}
203
204/// Check if the flags need to be preserved before the terminators.
205/// This would be the case, if the eflags is live-in of the region
206/// composed by the terminators or live-out of that region, without
207/// being defined by a terminator.
208static bool
209flagsNeedToBePreservedBeforeTheTerminators(const MachineBasicBlock &MBB) {
210 for (const MachineInstr &MI : MBB.terminators()) {
211 bool BreakNext = false;
212 for (const MachineOperand &MO : MI.operands()) {
213 if (!MO.isReg())
214 continue;
215 Register Reg = MO.getReg();
216 if (Reg != X86::EFLAGS)
217 continue;
218
219 // This terminator needs an eflags that is not defined
220 // by a previous another terminator:
221 // EFLAGS is live-in of the region composed by the terminators.
222 if (!MO.isDef())
223 return true;
224 // This terminator defines the eflags, i.e., we don't need to preserve it.
225 // However, we still need to check this specific terminator does not
226 // read a live-in value.
227 BreakNext = true;
228 }
229 // We found a definition of the eflags, no need to preserve them.
230 if (BreakNext)
231 return false;
232 }
233
234 // None of the terminators use or define the eflags.
235 // Check if they are live-out, that would imply we need to preserve them.
236 for (const MachineBasicBlock *Succ : MBB.successors())
237 if (Succ->isLiveIn(Reg: X86::EFLAGS))
238 return true;
239
240 return false;
241}
242
243constexpr uint64_t MaxSPChunk = (1ULL << 31) - 1;
244
245/// emitSPUpdate - Emit a series of instructions to increment / decrement the
246/// stack pointer by a constant value.
247void X86FrameLowering::emitSPUpdate(MachineBasicBlock &MBB,
248 MachineBasicBlock::iterator &MBBI,
249 const DebugLoc &DL, int64_t NumBytes,
250 bool InEpilogue) const {
251 bool isSub = NumBytes < 0;
252 uint64_t Offset = isSub ? -NumBytes : NumBytes;
253 MachineInstr::MIFlag Flag =
254 isSub ? MachineInstr::FrameSetup : MachineInstr::FrameDestroy;
255
256 if (!Uses64BitFramePtr && !isUInt<32>(x: Offset)) {
257 // We're being asked to adjust a 32-bit stack pointer by 4 GiB or more.
258 // This might be unreachable code, so don't complain now; just trap if
259 // it's reached at runtime.
260 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::TRAP));
261 return;
262 }
263
264 MachineFunction &MF = *MBB.getParent();
265 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
266 const X86TargetLowering &TLI = *STI.getTargetLowering();
267 const bool EmitInlineStackProbe = TLI.hasInlineStackProbe(MF);
268
269 // It's ok to not take into account large chunks when probing, as the
270 // allocation is split in smaller chunks anyway.
271 if (EmitInlineStackProbe && !InEpilogue) {
272
273 // This pseudo-instruction is going to be expanded, potentially using a
274 // loop, by inlineStackProbe().
275 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::STACKALLOC_W_PROBING)).addImm(Val: Offset);
276 return;
277 } else if (Offset > MaxSPChunk) {
278 // Rather than emit a long series of instructions for large offsets,
279 // load the offset into a register and do one sub/add
280 unsigned Reg = 0;
281 unsigned Rax = (unsigned)(Uses64BitFramePtr ? X86::RAX : X86::EAX);
282
283 if (isSub && !isEAXLiveIn(MBB))
284 Reg = Rax;
285 else
286 Reg = getX86SubSuperRegister(Reg: TRI->findDeadCallerSavedReg(MBB, MBBI),
287 Size: Uses64BitFramePtr ? 64 : 32);
288
289 unsigned AddSubRROpc = isSub ? getSUBrrOpcode(IsLP64: Uses64BitFramePtr)
290 : getADDrrOpcode(IsLP64: Uses64BitFramePtr);
291 if (Reg) {
292 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
293 MCID: TII.get(Opcode: X86::getMOVriOpcode(Use64BitReg: Uses64BitFramePtr, Imm: Offset)), DestReg: Reg)
294 .addImm(Val: Offset)
295 .setMIFlag(Flag);
296 MachineInstr *MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AddSubRROpc), DestReg: StackPtr)
297 .addReg(RegNo: StackPtr)
298 .addReg(RegNo: Reg);
299 MI->getOperand(i: 3).setIsDead(); // The EFLAGS implicit def is dead.
300 return;
301 } else if (Offset > 8 * MaxSPChunk) {
302 // If we would need more than 8 add or sub instructions (a >16GB stack
303 // frame), it's worth spilling RAX to materialize this immediate.
304 // pushq %rax
305 // movabsq +-$Offset+-SlotSize, %rax
306 // addq %rsp, %rax
307 // xchg %rax, (%rsp)
308 // movq (%rsp), %rsp
309 assert(Uses64BitFramePtr && "can't have 32-bit 16GB stack frame");
310 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH64r))
311 .addReg(RegNo: Rax, Flags: RegState::Kill)
312 .setMIFlag(Flag);
313 // Subtract is not commutative, so negate the offset and always use add.
314 // Subtract 8 less and add 8 more to account for the PUSH we just did.
315 if (isSub)
316 Offset = -(Offset - SlotSize);
317 else
318 Offset = Offset + SlotSize;
319 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
320 MCID: TII.get(Opcode: X86::getMOVriOpcode(Use64BitReg: Uses64BitFramePtr, Imm: Offset)), DestReg: Rax)
321 .addImm(Val: Offset)
322 .setMIFlag(Flag);
323 MachineInstr *MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::ADD64rr), DestReg: Rax)
324 .addReg(RegNo: Rax)
325 .addReg(RegNo: StackPtr);
326 MI->getOperand(i: 3).setIsDead(); // The EFLAGS implicit def is dead.
327 // Exchange the new SP in RAX with the top of the stack.
328 addRegOffset(
329 MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::XCHG64rm), DestReg: Rax).addReg(RegNo: Rax),
330 Reg: StackPtr, isKill: false, Offset: 0);
331 // Load new SP from the top of the stack into RSP.
332 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64rm), DestReg: StackPtr),
333 Reg: StackPtr, isKill: false, Offset: 0);
334 return;
335 }
336 }
337
338 while (Offset) {
339 if (Offset == SlotSize) {
340 // Use push / pop for slot sized adjustments as a size optimization. We
341 // need to find a dead register when using pop.
342 unsigned Reg = isSub ? (unsigned)(Is64Bit ? X86::RAX : X86::EAX)
343 : TRI->findDeadCallerSavedReg(MBB, MBBI);
344 if (Reg) {
345 unsigned Opc = isSub ? (Is64Bit ? X86::PUSH64r : X86::PUSH32r)
346 : (Is64Bit ? X86::POP64r : X86::POP32r);
347 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opc))
348 .addReg(RegNo: Reg, Flags: getDefRegState(B: !isSub) | getUndefRegState(B: isSub))
349 .setMIFlag(Flag);
350 return;
351 }
352 }
353
354 uint64_t ThisVal = std::min(a: Offset, b: MaxSPChunk);
355
356 BuildStackAdjustment(MBB, MBBI, DL, Offset: isSub ? -ThisVal : ThisVal, InEpilogue)
357 .setMIFlag(Flag);
358
359 Offset -= ThisVal;
360 }
361}
362
363MachineInstrBuilder X86FrameLowering::BuildStackAdjustment(
364 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
365 const DebugLoc &DL, int64_t Offset, bool InEpilogue) const {
366 assert(Offset != 0 && "zero offset stack adjustment requested");
367
368 // On Atom, using LEA to adjust SP is preferred, but using it in the epilogue
369 // is tricky.
370 bool UseLEA;
371 if (!InEpilogue) {
372 // Check if inserting the prologue at the beginning
373 // of MBB would require to use LEA operations.
374 // We need to use LEA operations if EFLAGS is live in, because
375 // it means an instruction will read it before it gets defined.
376 UseLEA = STI.useLeaForSP() || MBB.isLiveIn(Reg: X86::EFLAGS);
377 } else {
378 // If we can use LEA for SP but we shouldn't, check that none
379 // of the terminators uses the eflags. Otherwise we will insert
380 // a ADD that will redefine the eflags and break the condition.
381 // Alternatively, we could move the ADD, but this may not be possible
382 // and is an optimization anyway.
383 UseLEA = canUseLEAForSPInEpilogue(MF: *MBB.getParent());
384 if (UseLEA && !STI.useLeaForSP())
385 UseLEA = flagsNeedToBePreservedBeforeTheTerminators(MBB);
386 // If that assert breaks, that means we do not do the right thing
387 // in canUseAsEpilogue.
388 assert((UseLEA || !flagsNeedToBePreservedBeforeTheTerminators(MBB)) &&
389 "We shouldn't have allowed this insertion point");
390 }
391
392 MachineInstrBuilder MI;
393 // Use an NF (no-flags) variant as a smaller replacement for LEA when EFLAGS
394 // must be preserved (i.e. only when we would otherwise emit LEA). If EFLAGS
395 // is dead we prefer the plain SUB/ADD, which is shorter than the EVEX-encoded
396 // NF form. The NF stack-adjust opcodes below are 64-bit (SUB64ri32_NF/
397 // ADD64ri32_NF), so don't use them for the x32 ABI where the stack pointer is
398 // 32-bit. NF cannot reach a Win64 epilogue (which never uses LEA for the SP
399 // adjustment unless it has a frame pointer, and that path doesn't go through
400 // here), so the Windows epilogue unwinder never sees an undisassemblable NF
401 // add/sub.
402 bool UseNF = UseLEA && STI.hasNF() && Uses64BitFramePtr;
403 bool IsSub = Offset < 0;
404 uint64_t AbsOffset = IsSub ? -Offset : Offset;
405 if (UseNF) {
406 const unsigned Opc = IsSub ? X86::SUB64ri32_NF : X86::ADD64ri32_NF;
407 MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: StackPtr)
408 .addReg(RegNo: StackPtr)
409 .addImm(Val: AbsOffset);
410 // NF instructions define no EFLAGS, so there is nothing to mark dead.
411 } else if (UseLEA) {
412 MI = addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
413 MCID: TII.get(Opcode: getLEArOpcode(IsLP64: Uses64BitFramePtr)),
414 DestReg: StackPtr),
415 Reg: StackPtr, isKill: false, Offset);
416 } else {
417 unsigned Opc = IsSub ? getSUBriOpcode(IsLP64: Uses64BitFramePtr)
418 : getADDriOpcode(IsLP64: Uses64BitFramePtr);
419 int64_t Imm = AbsOffset;
420 // Prefer `add rsp, -128` over `sub rsp, 128` (and vice versa in the
421 // epilogue): 128 is the one magnitude whose negation fits the
422 // sign-extended 8-bit immediate while the value itself does not, so the
423 // flipped operation is three bytes shorter. EFLAGS is dead here (this
424 // branch clobbers it anyway). Windows CFI epilogues keep the canonical
425 // ADD: v1 unwind info describes no epilogues, so the unwinder detects
426 // one by disassembling forward for `add rsp, imm` (prologues are
427 // delimited by SizeOfProlog and never disassembled). Unwind v2/v3 do
428 // describe epilogues, but X86WinEHUnwindV2 expects the ADD spelling
429 // too.
430 if (AbsOffset == 128 &&
431 !(InEpilogue && MBB.getParent()->usesWindowsUnwindInfo())) {
432 Opc = IsSub ? getADDriOpcode(IsLP64: Uses64BitFramePtr)
433 : getSUBriOpcode(IsLP64: Uses64BitFramePtr);
434 Imm = -128;
435 }
436 MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: StackPtr)
437 .addReg(RegNo: StackPtr)
438 .addImm(Val: Imm);
439 MI->getOperand(i: 3).setIsDead(); // The EFLAGS implicit def is dead.
440 }
441 return MI;
442}
443
444template <typename FoundT, typename CalcT>
445int64_t X86FrameLowering::mergeSPUpdates(MachineBasicBlock &MBB,
446 MachineBasicBlock::iterator &MBBI,
447 FoundT FoundStackAdjust,
448 CalcT CalcNewOffset,
449 bool doMergeWithPrevious) const {
450 if ((doMergeWithPrevious && MBBI == MBB.begin()) ||
451 (!doMergeWithPrevious && MBBI == MBB.end()))
452 return CalcNewOffset(0);
453
454 MachineBasicBlock::iterator PI = doMergeWithPrevious ? std::prev(x: MBBI) : MBBI;
455
456 PI = skipDebugInstructionsBackward(It: PI, Begin: MBB.begin());
457 // It is assumed that ADD/SUB/LEA instruction is succeded by one CFI
458 // instruction, and that there are no DBG_VALUE or other instructions between
459 // ADD/SUB/LEA and its corresponding CFI instruction.
460 /* TODO: Add support for the case where there are multiple CFI instructions
461 below the ADD/SUB/LEA, e.g.:
462 ...
463 add
464 cfi_def_cfa_offset
465 cfi_offset
466 ...
467 */
468 if (doMergeWithPrevious && PI != MBB.begin() && PI->isCFIInstruction())
469 PI = std::prev(x: PI);
470
471 int64_t Offset = 0;
472 for (;;) {
473 unsigned Opc = PI->getOpcode();
474
475 if ((Opc == X86::ADD64ri32 || Opc == X86::ADD32ri ||
476 Opc == X86::ADD64ri32_NF) &&
477 PI->getOperand(i: 0).getReg() == StackPtr) {
478 assert(PI->getOperand(1).getReg() == StackPtr);
479 Offset = PI->getOperand(i: 2).getImm();
480 } else if ((Opc == X86::LEA32r || Opc == X86::LEA64_32r) &&
481 PI->getOperand(i: 0).getReg() == StackPtr &&
482 PI->getOperand(i: 1).getReg() == StackPtr &&
483 PI->getOperand(i: 2).getImm() == 1 &&
484 PI->getOperand(i: 3).getReg() == X86::NoRegister &&
485 PI->getOperand(i: 5).getReg() == X86::NoRegister) {
486 // For LEAs we have: def = lea SP, FI, noreg, Offset, noreg.
487 Offset = PI->getOperand(i: 4).getImm();
488 } else if ((Opc == X86::SUB64ri32 || Opc == X86::SUB32ri ||
489 Opc == X86::SUB64ri32_NF) &&
490 PI->getOperand(i: 0).getReg() == StackPtr) {
491 assert(PI->getOperand(1).getReg() == StackPtr);
492 Offset = -PI->getOperand(i: 2).getImm();
493 } else
494 return CalcNewOffset(0);
495
496 FoundStackAdjust(PI, Offset);
497 if ((uint64_t)std::abs(i: (int64_t)CalcNewOffset(Offset)) < MaxSPChunk)
498 break;
499
500 if (doMergeWithPrevious ? (PI == MBB.begin()) : (PI == MBB.end()))
501 return CalcNewOffset(0);
502
503 PI = doMergeWithPrevious ? std::prev(x: PI) : std::next(x: PI);
504 }
505
506 PI = MBB.erase(I: PI);
507 if (PI != MBB.end() && PI->isCFIInstruction()) {
508 auto CIs = MBB.getParent()->getFrameInstructions();
509 MCCFIInstruction CI = CIs[PI->getOperand(i: 0).getCFIIndex()];
510 if (CI.getOperation() == MCCFIInstruction::OpDefCfaOffset ||
511 CI.getOperation() == MCCFIInstruction::OpAdjustCfaOffset)
512 PI = MBB.erase(I: PI);
513 }
514 if (!doMergeWithPrevious)
515 MBBI = skipDebugInstructionsForward(It: PI, End: MBB.end());
516
517 return CalcNewOffset(Offset);
518}
519
520int64_t X86FrameLowering::mergeSPAdd(MachineBasicBlock &MBB,
521 MachineBasicBlock::iterator &MBBI,
522 int64_t AddOffset,
523 bool doMergeWithPrevious) const {
524 return mergeSPUpdates(
525 MBB, MBBI, CalcNewOffset: [AddOffset](int64_t Offset) { return AddOffset + Offset; },
526 doMergeWithPrevious);
527}
528
529void X86FrameLowering::BuildCFI(MachineBasicBlock &MBB,
530 MachineBasicBlock::iterator MBBI,
531 const DebugLoc &DL,
532 const MCCFIInstruction &CFIInst,
533 MachineInstr::MIFlag Flag) const {
534 MachineFunction &MF = *MBB.getParent();
535 unsigned CFIIndex = MF.addFrameInst(Inst: CFIInst);
536
537 if (CFIInst.getOperation() == MCCFIInstruction::OpAdjustCfaOffset)
538 MF.getInfo<X86MachineFunctionInfo>()->setHasCFIAdjustCfa(true);
539
540 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::CFI_INSTRUCTION))
541 .addCFIIndex(CFIIndex)
542 .setMIFlag(Flag);
543}
544
545/// Emits Dwarf Info specifying offsets of callee saved registers and
546/// frame pointer. This is called only when basic block sections are enabled.
547void X86FrameLowering::emitCalleeSavedFrameMovesFullCFA(
548 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
549 MachineFunction &MF = *MBB.getParent();
550 if (!hasFP(MF)) {
551 emitCalleeSavedFrameMoves(MBB, MBBI, DL: DebugLoc{}, IsPrologue: true);
552 return;
553 }
554 const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
555 const Register FramePtr = TRI->getFrameRegister(MF);
556 const Register MachineFramePtr =
557 STI.isTarget64BitILP32() ? Register(getX86SubSuperRegister(Reg: FramePtr, Size: 64))
558 : FramePtr;
559 unsigned DwarfReg = MRI->getDwarfRegNum(Reg: MachineFramePtr, isEH: true);
560 // Offset = space for return address + size of the frame pointer itself.
561 int64_t Offset = (Is64Bit ? 8 : 4) + (Uses64BitFramePtr ? 8 : 4);
562 BuildCFI(MBB, MBBI, DL: DebugLoc{},
563 CFIInst: MCCFIInstruction::createOffset(L: nullptr, Register: DwarfReg, Offset: -Offset));
564 emitCalleeSavedFrameMoves(MBB, MBBI, DL: DebugLoc{}, IsPrologue: true);
565}
566
567void X86FrameLowering::emitCalleeSavedFrameMoves(
568 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
569 const DebugLoc &DL, bool IsPrologue) const {
570 MachineFunction &MF = *MBB.getParent();
571 MachineFrameInfo &MFI = MF.getFrameInfo();
572 const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
573 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
574
575 // Add callee saved registers to move list.
576 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
577
578 // Calculate offsets.
579 for (const CalleeSavedInfo &I : CSI) {
580 int64_t Offset = MFI.getObjectOffset(ObjectIdx: I.getFrameIdx());
581 MCRegister Reg = I.getReg();
582 unsigned DwarfReg = MRI->getDwarfRegNum(Reg, isEH: true);
583
584 if (IsPrologue) {
585 if (X86FI->getStackPtrSaveMI()) {
586 // +2*SlotSize because there is return address and ebp at the bottom
587 // of the stack.
588 // | retaddr |
589 // | ebp |
590 // | |<--ebp
591 Offset += 2 * SlotSize;
592 SmallString<64> CfaExpr;
593 CfaExpr.push_back(Elt: dwarf::DW_CFA_expression);
594 uint8_t buffer[16];
595 CfaExpr.append(in_start: buffer, in_end: buffer + encodeULEB128(Value: DwarfReg, p: buffer));
596 CfaExpr.push_back(Elt: 2);
597 Register FramePtr = TRI->getFrameRegister(MF);
598 const Register MachineFramePtr =
599 STI.isTarget64BitILP32()
600 ? Register(getX86SubSuperRegister(Reg: FramePtr, Size: 64))
601 : FramePtr;
602 unsigned DwarfFramePtr = MRI->getDwarfRegNum(Reg: MachineFramePtr, isEH: true);
603 CfaExpr.push_back(Elt: (uint8_t)(dwarf::DW_OP_breg0 + DwarfFramePtr));
604 CfaExpr.append(in_start: buffer, in_end: buffer + encodeSLEB128(Value: Offset, p: buffer));
605 BuildCFI(MBB, MBBI, DL,
606 CFIInst: MCCFIInstruction::createEscape(L: nullptr, Vals: CfaExpr.str()),
607 Flag: MachineInstr::FrameSetup);
608 } else {
609 BuildCFI(MBB, MBBI, DL,
610 CFIInst: MCCFIInstruction::createOffset(L: nullptr, Register: DwarfReg, Offset));
611 }
612 } else {
613 BuildCFI(MBB, MBBI, DL,
614 CFIInst: MCCFIInstruction::createRestore(L: nullptr, Register: DwarfReg));
615 }
616 }
617 if (auto *MI = X86FI->getStackPtrSaveMI()) {
618 int FI = MI->getOperand(i: 1).getIndex();
619 int64_t Offset = MFI.getObjectOffset(ObjectIdx: FI) + 2 * SlotSize;
620 SmallString<64> CfaExpr;
621 Register FramePtr = TRI->getFrameRegister(MF);
622 const Register MachineFramePtr =
623 STI.isTarget64BitILP32()
624 ? Register(getX86SubSuperRegister(Reg: FramePtr, Size: 64))
625 : FramePtr;
626 unsigned DwarfFramePtr = MRI->getDwarfRegNum(Reg: MachineFramePtr, isEH: true);
627 CfaExpr.push_back(Elt: (uint8_t)(dwarf::DW_OP_breg0 + DwarfFramePtr));
628 uint8_t buffer[16];
629 CfaExpr.append(in_start: buffer, in_end: buffer + encodeSLEB128(Value: Offset, p: buffer));
630 CfaExpr.push_back(Elt: dwarf::DW_OP_deref);
631
632 SmallString<64> DefCfaExpr;
633 DefCfaExpr.push_back(Elt: dwarf::DW_CFA_def_cfa_expression);
634 DefCfaExpr.append(in_start: buffer, in_end: buffer + encodeSLEB128(Value: CfaExpr.size(), p: buffer));
635 DefCfaExpr.append(RHS: CfaExpr.str());
636 // DW_CFA_def_cfa_expression: DW_OP_breg5 offset, DW_OP_deref
637 BuildCFI(MBB, MBBI, DL,
638 CFIInst: MCCFIInstruction::createEscape(L: nullptr, Vals: DefCfaExpr.str()),
639 Flag: MachineInstr::FrameSetup);
640 }
641}
642
643void X86FrameLowering::emitZeroCallUsedRegs(BitVector RegsToZero,
644 MachineBasicBlock &MBB,
645 RegScavenger *) const {
646 const MachineFunction &MF = *MBB.getParent();
647
648 // Insertion point.
649 MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
650
651 // Fake a debug loc.
652 DebugLoc DL;
653 if (MBBI != MBB.end())
654 DL = MBBI->getDebugLoc();
655
656 // Zero out FP stack if referenced. Do this outside of the loop below so that
657 // it's done only once.
658 for (MCRegister Reg : RegsToZero.set_bits()) {
659 if (!X86::RFP80RegClass.contains(Reg))
660 continue;
661
662 // Do not push zeros over x87 return values. X86FloatingPoint records
663 // returned values as implicit ST0/ST1 uses on the return instruction.
664 unsigned NumFPRegs = 8;
665 if (MBBI->hasRegisterImplicitUseOperand(Reg: X86::ST0))
666 --NumFPRegs;
667 if (MBBI->hasRegisterImplicitUseOperand(Reg: X86::ST1))
668 --NumFPRegs;
669
670 for (unsigned i = 0; i != NumFPRegs; ++i)
671 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::LD_F0));
672
673 for (unsigned i = 0; i != NumFPRegs; ++i)
674 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::ST_FPrr)).addReg(RegNo: X86::ST0);
675 break;
676 }
677
678 // For GPRs, we only care to clear out the 32-bit register.
679 BitVector GPRsToZero(TRI->getNumRegs());
680 for (MCRegister Reg : RegsToZero.set_bits())
681 if (TRI->isGeneralPurposeRegister(MF, Reg)) {
682 GPRsToZero.set(getX86SubSuperRegister(Reg, Size: 32));
683 RegsToZero.reset(Idx: Reg);
684 }
685
686 // Zero out the GPRs first.
687 for (MCRegister Reg : GPRsToZero.set_bits())
688 TII.buildClearRegister(Reg, MBB, Iter: MBBI, DL);
689
690 // Coalesce the aliasing XMM/YMM/ZMM views of each vector register so a lane
691 // is cleared only once, mirroring the GPR handling above.
692 auto getVectorClearReg = [&](MCRegister Reg) -> MCRegister {
693 if (!X86::VR128RegClass.contains(Reg) &&
694 !X86::VR128XRegClass.contains(Reg) &&
695 !X86::VR256RegClass.contains(Reg) &&
696 !X86::VR256XRegClass.contains(Reg) && !X86::VR512RegClass.contains(Reg))
697 return MCRegister();
698
699 // Clearing the XMM zeroes the whole lane. XMM0-15 use the compact VEX form;
700 // XMM16-31 are EVEX-only, reachable only via the ZMM form.
701 MCRegister Xmm = TRI->getSubReg(Reg, Idx: X86::sub_xmm);
702 if (!Xmm)
703 Xmm = Reg;
704 if (X86::VR128RegClass.contains(Reg: Xmm))
705 return Xmm;
706 MCRegister Zmm =
707 TRI->getMatchingSuperReg(Reg: Xmm, SubIdx: X86::sub_xmm, RC: &X86::VR512RegClass);
708 assert(Zmm && "XMM16-31 must have an enclosing ZMM to clear through");
709 return Zmm;
710 };
711
712 BitVector VecRegsToZero(TRI->getNumRegs());
713 for (MCRegister Reg : RegsToZero.set_bits())
714 if (MCRegister Clear = getVectorClearReg(Reg)) {
715 VecRegsToZero.set(Clear.id());
716 RegsToZero.reset(Idx: Reg);
717 }
718
719 for (MCRegister Reg : VecRegsToZero.set_bits())
720 TII.buildClearRegister(Reg, MBB, Iter: MBBI, DL);
721
722 // Zero out the remaining registers (e.g. mask registers).
723 for (MCRegister Reg : RegsToZero.set_bits())
724 TII.buildClearRegister(Reg, MBB, Iter: MBBI, DL);
725}
726
727void X86FrameLowering::emitStackProbe(
728 MachineFunction &MF, MachineBasicBlock &MBB,
729 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog,
730 std::optional<MachineFunction::DebugInstrOperandPair> InstrNum) const {
731 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
732 if (STI.isTargetWindowsCoreCLR()) {
733 if (InProlog) {
734 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::STACKALLOC_W_PROBING))
735 .addImm(Val: 0 /* no explicit stack size */);
736 } else {
737 emitStackProbeInline(MF, MBB, MBBI, DL, InProlog: false);
738 }
739 } else {
740 emitStackProbeCall(MF, MBB, MBBI, DL, InProlog, InstrNum);
741 }
742}
743
744bool X86FrameLowering::stackProbeFunctionModifiesSP() const {
745 return STI.isOSWindows() && !STI.isTargetWin64();
746}
747
748void X86FrameLowering::inlineStackProbe(MachineFunction &MF,
749 MachineBasicBlock &PrologMBB) const {
750 auto Where = llvm::find_if(Range&: PrologMBB, P: [](MachineInstr &MI) {
751 return MI.getOpcode() == X86::STACKALLOC_W_PROBING;
752 });
753 if (Where != PrologMBB.end()) {
754 DebugLoc DL = PrologMBB.findDebugLoc(MBBI: Where);
755 emitStackProbeInline(MF, MBB&: PrologMBB, MBBI: Where, DL, InProlog: true);
756 Where->eraseFromParent();
757 }
758}
759
760void X86FrameLowering::emitStackProbeInline(MachineFunction &MF,
761 MachineBasicBlock &MBB,
762 MachineBasicBlock::iterator MBBI,
763 const DebugLoc &DL,
764 bool InProlog) const {
765 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
766 if (STI.isTargetWindowsCoreCLR() && STI.is64Bit())
767 emitStackProbeInlineWindowsCoreCLR64(MF, MBB, MBBI, DL, InProlog);
768 else
769 emitStackProbeInlineGeneric(MF, MBB, MBBI, DL, InProlog);
770}
771
772void X86FrameLowering::emitStackProbeInlineGeneric(
773 MachineFunction &MF, MachineBasicBlock &MBB,
774 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog) const {
775 MachineInstr &AllocWithProbe = *MBBI;
776 uint64_t Offset = AllocWithProbe.getOperand(i: 0).getImm();
777
778 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
779 const X86TargetLowering &TLI = *STI.getTargetLowering();
780 assert(!(STI.is64Bit() && STI.isTargetWindowsCoreCLR()) &&
781 "different expansion expected for CoreCLR 64 bit");
782
783 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
784 uint64_t ProbeChunk = StackProbeSize * 8;
785
786 uint64_t MaxAlign =
787 TRI->hasStackRealignment(MF) ? calculateMaxStackAlign(MF) : 0;
788
789 // Synthesize a loop or unroll it, depending on the number of iterations.
790 // BuildStackAlignAND ensures that only MaxAlign % StackProbeSize bits left
791 // between the unaligned rsp and current rsp.
792 if (Offset > ProbeChunk) {
793 emitStackProbeInlineGenericLoop(MF, MBB, MBBI, DL, Offset,
794 Align: MaxAlign % StackProbeSize);
795 } else {
796 emitStackProbeInlineGenericBlock(MF, MBB, MBBI, DL, Offset,
797 Align: MaxAlign % StackProbeSize);
798 }
799}
800
801void X86FrameLowering::emitStackProbeInlineGenericBlock(
802 MachineFunction &MF, MachineBasicBlock &MBB,
803 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, uint64_t Offset,
804 uint64_t AlignOffset) const {
805
806 // The code below is allocating one StackProbeSize at a time,
807 // then inserting a "probe" (write of 0), in order to hit any guard pages.
808 // A probe is only needed every StackProbeSize (no skipping entire pages).
809 //
810 // StackProbeSize (PAGE_SIZE)
811 // / \
812 // |----------|----------|------> rsp decreasing
813 // ^ ^ ^
814 // | | |
815 // CurrentOffset Offset (Offset + AlignOffset)
816 //
817 // Offset: size we are being asked to allocate on the stack, in bytes.
818 // AlignOffset: extra amount to allocate (< StackProbeSize),
819 // needed to satisfy any alignment for calling conventions, etc.
820
821 const bool NeedsDwarfCFI = needsDwarfCFI(MF);
822 const bool HasFP = hasFP(MF);
823 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
824 const X86TargetLowering &TLI = *STI.getTargetLowering();
825 const unsigned MovMIOpc = Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
826 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
827
828 uint64_t CurrentOffset = 0;
829
830 assert(AlignOffset < StackProbeSize);
831
832 // If the offset is so small it fits within a page, there's nothing to do.
833 if (StackProbeSize < Offset + AlignOffset) {
834
835 uint64_t StackAdjustment = StackProbeSize - AlignOffset;
836 BuildStackAdjustment(MBB, MBBI, DL, Offset: -StackAdjustment, /*InEpilogue=*/false)
837 .setMIFlag(MachineInstr::FrameSetup);
838 if (!HasFP && NeedsDwarfCFI) {
839 BuildCFI(
840 MBB, MBBI, DL,
841 CFIInst: MCCFIInstruction::createAdjustCfaOffset(L: nullptr, Adjustment: StackAdjustment));
842 }
843
844 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: MovMIOpc))
845 .setMIFlag(MachineInstr::FrameSetup),
846 Reg: StackPtr, isKill: false, Offset: 0)
847 .addImm(Val: 0)
848 .setMIFlag(MachineInstr::FrameSetup);
849 NumFrameExtraProbe++;
850 CurrentOffset = StackProbeSize - AlignOffset;
851 }
852
853 // For the next N - 1 pages, just probe.
854 while (CurrentOffset + StackProbeSize <= Offset) {
855 BuildStackAdjustment(MBB, MBBI, DL, Offset: -StackProbeSize, /*InEpilogue=*/false)
856 .setMIFlag(MachineInstr::FrameSetup);
857
858 if (!HasFP && NeedsDwarfCFI) {
859 BuildCFI(
860 MBB, MBBI, DL,
861 CFIInst: MCCFIInstruction::createAdjustCfaOffset(L: nullptr, Adjustment: StackProbeSize));
862 }
863 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: MovMIOpc))
864 .setMIFlag(MachineInstr::FrameSetup),
865 Reg: StackPtr, isKill: false, Offset: 0)
866 .addImm(Val: 0)
867 .setMIFlag(MachineInstr::FrameSetup);
868 NumFrameExtraProbe++;
869 CurrentOffset += StackProbeSize;
870 }
871
872 const uint64_t ChunkSize = Offset - CurrentOffset;
873 // We may not require a tail allocation. For example, this can happen
874 // if Offset is a multiple of StackProbeSize, and AlignOffset = 0.
875 if (ChunkSize == 0)
876 return;
877 else if (ChunkSize == SlotSize) {
878 // Use push for slot sized adjustments as a size optimization,
879 // like emitSPUpdate does when not probing.
880 unsigned Reg = Is64Bit ? X86::RAX : X86::EAX;
881 unsigned Opc = Is64Bit ? X86::PUSH64r : X86::PUSH32r;
882 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opc))
883 .addReg(RegNo: Reg, Flags: RegState::Undef)
884 .setMIFlag(MachineInstr::FrameSetup);
885 } else {
886 assert(!(ChunkSize > StackProbeSize) &&
887 "Tail allocation greater than page size");
888 BuildStackAdjustment(MBB, MBBI, DL, Offset: -ChunkSize, /*InEpilogue=*/false)
889 .setMIFlag(MachineInstr::FrameSetup);
890 }
891 // No need to adjust Dwarf CFA offset here, the last position of the stack has
892 // been defined
893}
894
895void X86FrameLowering::emitStackProbeInlineGenericLoop(
896 MachineFunction &MF, MachineBasicBlock &MBB,
897 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, uint64_t Offset,
898 uint64_t AlignOffset) const {
899 assert(Offset && "null offset");
900
901 assert(MBB.computeRegisterLiveness(TRI, X86::EFLAGS, MBBI) !=
902 MachineBasicBlock::LQR_Live &&
903 "Inline stack probe loop will clobber live EFLAGS.");
904
905 const bool NeedsDwarfCFI = needsDwarfCFI(MF);
906 const bool HasFP = hasFP(MF);
907 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
908 const X86TargetLowering &TLI = *STI.getTargetLowering();
909 const unsigned MovMIOpc = Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
910 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
911
912 if (AlignOffset) {
913 if (AlignOffset < StackProbeSize) {
914 // Perform a first smaller allocation followed by a probe.
915 BuildStackAdjustment(MBB, MBBI, DL, Offset: -AlignOffset, /*InEpilogue=*/false)
916 .setMIFlag(MachineInstr::FrameSetup);
917
918 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: MovMIOpc))
919 .setMIFlag(MachineInstr::FrameSetup),
920 Reg: StackPtr, isKill: false, Offset: 0)
921 .addImm(Val: 0)
922 .setMIFlag(MachineInstr::FrameSetup);
923 NumFrameExtraProbe++;
924 Offset -= AlignOffset;
925 }
926 }
927
928 // Synthesize a loop
929 NumFrameLoopProbe++;
930 const BasicBlock *LLVM_BB = MBB.getBasicBlock();
931
932 MachineBasicBlock *testMBB = MF.CreateMachineBasicBlock(BB: LLVM_BB);
933 MachineBasicBlock *tailMBB = MF.CreateMachineBasicBlock(BB: LLVM_BB);
934
935 MachineFunction::iterator MBBIter = ++MBB.getIterator();
936 MF.insert(MBBI: MBBIter, MBB: testMBB);
937 MF.insert(MBBI: MBBIter, MBB: tailMBB);
938
939 Register FinalStackProbed = Uses64BitFramePtr ? X86::R11
940 : Is64Bit ? X86::R11D
941 : X86::EAX;
942
943 // save loop bound
944 {
945 const uint64_t BoundOffset = alignDown(Value: Offset, Align: StackProbeSize);
946
947 // Can we calculate the loop bound using SUB with a 32-bit immediate?
948 // Note that the immediate gets sign-extended when used with a 64-bit
949 // register, so in that case we only have 31 bits to work with.
950 bool canUseSub =
951 Uses64BitFramePtr ? isUInt<31>(x: BoundOffset) : isUInt<32>(x: BoundOffset);
952
953 if (canUseSub) {
954 const unsigned SUBOpc = getSUBriOpcode(IsLP64: Uses64BitFramePtr);
955
956 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: FinalStackProbed)
957 .addReg(RegNo: StackPtr)
958 .setMIFlag(MachineInstr::FrameSetup);
959 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: SUBOpc), DestReg: FinalStackProbed)
960 .addReg(RegNo: FinalStackProbed)
961 .addImm(Val: BoundOffset)
962 .setMIFlag(MachineInstr::FrameSetup)
963 .setOperandDead(3); // implicit-def $eflags
964 } else if (Uses64BitFramePtr) {
965 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64ri), DestReg: FinalStackProbed)
966 .addImm(Val: -BoundOffset)
967 .setMIFlag(MachineInstr::FrameSetup);
968 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::ADD64rr), DestReg: FinalStackProbed)
969 .addReg(RegNo: FinalStackProbed)
970 .addReg(RegNo: StackPtr)
971 .setMIFlag(MachineInstr::FrameSetup)
972 .setOperandDead(3); // implicit-def $eflags
973 } else {
974 llvm_unreachable("Offset too large for 32-bit stack pointer");
975 }
976
977 // while in the loop, use loop-invariant reg for CFI,
978 // instead of the stack pointer, which changes during the loop
979 if (!HasFP && NeedsDwarfCFI) {
980 // x32 uses the same DWARF register numbers as x86-64,
981 // so there isn't a register number for r11d, we must use r11 instead
982 const Register DwarfFinalStackProbed =
983 STI.isTarget64BitILP32()
984 ? Register(getX86SubSuperRegister(Reg: FinalStackProbed, Size: 64))
985 : FinalStackProbed;
986
987 BuildCFI(MBB, MBBI, DL,
988 CFIInst: MCCFIInstruction::createDefCfaRegister(
989 L: nullptr, Register: TRI->getDwarfRegNum(Reg: DwarfFinalStackProbed, isEH: true)));
990 BuildCFI(MBB, MBBI, DL,
991 CFIInst: MCCFIInstruction::createAdjustCfaOffset(L: nullptr, Adjustment: BoundOffset));
992 }
993 }
994
995 // allocate a page
996 BuildStackAdjustment(MBB&: *testMBB, MBBI: testMBB->end(), DL, Offset: -StackProbeSize,
997 /*InEpilogue=*/false)
998 .setMIFlag(MachineInstr::FrameSetup);
999
1000 // touch the page
1001 addRegOffset(MIB: BuildMI(BB: testMBB, MIMD: DL, MCID: TII.get(Opcode: MovMIOpc))
1002 .setMIFlag(MachineInstr::FrameSetup),
1003 Reg: StackPtr, isKill: false, Offset: 0)
1004 .addImm(Val: 0)
1005 .setMIFlag(MachineInstr::FrameSetup);
1006
1007 // cmp with stack pointer bound
1008 BuildMI(BB: testMBB, MIMD: DL, MCID: TII.get(Opcode: Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
1009 .addReg(RegNo: StackPtr)
1010 .addReg(RegNo: FinalStackProbed)
1011 .setMIFlag(MachineInstr::FrameSetup);
1012
1013 // jump
1014 BuildMI(BB: testMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
1015 .addMBB(MBB: testMBB)
1016 .addImm(Val: X86::COND_NE)
1017 .setMIFlag(MachineInstr::FrameSetup);
1018 testMBB->addSuccessor(Succ: testMBB);
1019 testMBB->addSuccessor(Succ: tailMBB);
1020
1021 // BB management
1022 tailMBB->splice(Where: tailMBB->end(), Other: &MBB, From: MBBI, To: MBB.end());
1023 tailMBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
1024 MBB.addSuccessor(Succ: testMBB);
1025
1026 // handle tail
1027 const uint64_t TailOffset = Offset % StackProbeSize;
1028 MachineBasicBlock::iterator TailMBBIter = tailMBB->begin();
1029 if (TailOffset) {
1030 BuildStackAdjustment(MBB&: *tailMBB, MBBI: TailMBBIter, DL, Offset: -TailOffset,
1031 /*InEpilogue=*/false)
1032 .setMIFlag(MachineInstr::FrameSetup);
1033 }
1034
1035 // after the loop, switch back to stack pointer for CFI
1036 if (!HasFP && NeedsDwarfCFI) {
1037 // x32 uses the same DWARF register numbers as x86-64,
1038 // so there isn't a register number for esp, we must use rsp instead
1039 const Register DwarfStackPtr =
1040 STI.isTarget64BitILP32()
1041 ? Register(getX86SubSuperRegister(Reg: StackPtr, Size: 64))
1042 : Register(StackPtr);
1043
1044 BuildCFI(MBB&: *tailMBB, MBBI: TailMBBIter, DL,
1045 CFIInst: MCCFIInstruction::createDefCfaRegister(
1046 L: nullptr, Register: TRI->getDwarfRegNum(Reg: DwarfStackPtr, isEH: true)));
1047 }
1048
1049 // Update Live In information
1050 fullyRecomputeLiveIns(MBBs: {tailMBB, testMBB});
1051}
1052
1053void X86FrameLowering::emitStackProbeInlineWindowsCoreCLR64(
1054 MachineFunction &MF, MachineBasicBlock &MBB,
1055 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog) const {
1056 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
1057 assert(STI.is64Bit() && "different expansion needed for 32 bit");
1058 assert(STI.isTargetWindowsCoreCLR() && "custom expansion expects CoreCLR");
1059 const TargetInstrInfo &TII = *STI.getInstrInfo();
1060 const BasicBlock *LLVM_BB = MBB.getBasicBlock();
1061
1062 assert(MBB.computeRegisterLiveness(TRI, X86::EFLAGS, MBBI) !=
1063 MachineBasicBlock::LQR_Live &&
1064 "Inline stack probe loop will clobber live EFLAGS.");
1065
1066 // RAX contains the number of bytes of desired stack adjustment.
1067 // The handling here assumes this value has already been updated so as to
1068 // maintain stack alignment.
1069 //
1070 // We need to exit with RSP modified by this amount and execute suitable
1071 // page touches to notify the OS that we're growing the stack responsibly.
1072 // All stack probing must be done without modifying RSP.
1073 //
1074 // MBB:
1075 // SizeReg = RAX;
1076 // ZeroReg = 0
1077 // CopyReg = RSP
1078 // Flags, TestReg = CopyReg - SizeReg
1079 // FinalReg = !Flags.Ovf ? TestReg : ZeroReg
1080 // LimitReg = gs magic thread env access
1081 // if FinalReg >= LimitReg goto ContinueMBB
1082 // RoundBB:
1083 // RoundReg = page address of FinalReg
1084 // LoopMBB:
1085 // LoopReg = PHI(LimitReg,ProbeReg)
1086 // ProbeReg = LoopReg - PageSize
1087 // [ProbeReg] = 0
1088 // if (ProbeReg > RoundReg) goto LoopMBB
1089 // ContinueMBB:
1090 // RSP = RSP - RAX
1091 // [rest of original MBB]
1092
1093 // Set up the new basic blocks
1094 MachineBasicBlock *RoundMBB = MF.CreateMachineBasicBlock(BB: LLVM_BB);
1095 MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(BB: LLVM_BB);
1096 MachineBasicBlock *ContinueMBB = MF.CreateMachineBasicBlock(BB: LLVM_BB);
1097
1098 MachineFunction::iterator MBBIter = std::next(x: MBB.getIterator());
1099 MF.insert(MBBI: MBBIter, MBB: RoundMBB);
1100 MF.insert(MBBI: MBBIter, MBB: LoopMBB);
1101 MF.insert(MBBI: MBBIter, MBB: ContinueMBB);
1102
1103 // Split MBB and move the tail portion down to ContinueMBB.
1104 MachineBasicBlock::iterator BeforeMBBI = std::prev(x: MBBI);
1105 ContinueMBB->splice(Where: ContinueMBB->begin(), Other: &MBB, From: MBBI, To: MBB.end());
1106 ContinueMBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
1107
1108 // Some useful constants
1109 const int64_t ThreadEnvironmentStackLimit = 0x10;
1110 const int64_t PageSize = 0x1000;
1111 const int64_t PageMask = ~(PageSize - 1);
1112
1113 // Registers we need. For the normal case we use virtual
1114 // registers. For the prolog expansion we use RAX, RCX and RDX.
1115 MachineRegisterInfo &MRI = MF.getRegInfo();
1116 const TargetRegisterClass *RegClass = &X86::GR64RegClass;
1117 const Register
1118 SizeReg = InProlog ? X86::RAX : MRI.createVirtualRegister(RegClass),
1119 ZeroReg = InProlog ? X86::RCX : MRI.createVirtualRegister(RegClass),
1120 CopyReg = InProlog ? X86::RDX : MRI.createVirtualRegister(RegClass),
1121 TestReg = InProlog ? X86::RDX : MRI.createVirtualRegister(RegClass),
1122 FinalReg = InProlog ? X86::RDX : MRI.createVirtualRegister(RegClass),
1123 RoundedReg = InProlog ? X86::RDX : MRI.createVirtualRegister(RegClass),
1124 LimitReg = InProlog ? X86::RCX : MRI.createVirtualRegister(RegClass),
1125 JoinReg = InProlog ? X86::RCX : MRI.createVirtualRegister(RegClass),
1126 ProbeReg = InProlog ? X86::RCX : MRI.createVirtualRegister(RegClass);
1127
1128 // SP-relative offsets where we can save RCX and RDX.
1129 int64_t RCXShadowSlot = 0;
1130 int64_t RDXShadowSlot = 0;
1131
1132 // If inlining in the prolog, save RCX and RDX.
1133 if (InProlog) {
1134 // Compute the offsets. We need to account for things already
1135 // pushed onto the stack at this point: return address, frame
1136 // pointer (if used), and callee saves.
1137 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
1138 const int64_t CalleeSaveSize = X86FI->getCalleeSavedFrameSize();
1139 const bool HasFP = hasFP(MF);
1140
1141 // Check if we need to spill RCX and/or RDX.
1142 // Here we assume that no earlier prologue instruction changes RCX and/or
1143 // RDX, so checking the block live-ins is enough.
1144 const bool IsRCXLiveIn = MBB.isLiveIn(Reg: X86::RCX);
1145 const bool IsRDXLiveIn = MBB.isLiveIn(Reg: X86::RDX);
1146 int64_t InitSlot = 8 + CalleeSaveSize + (HasFP ? 8 : 0);
1147 // Assign the initial slot to both registers, then change RDX's slot if both
1148 // need to be spilled.
1149 if (IsRCXLiveIn)
1150 RCXShadowSlot = InitSlot;
1151 if (IsRDXLiveIn)
1152 RDXShadowSlot = InitSlot;
1153 if (IsRDXLiveIn && IsRCXLiveIn)
1154 RDXShadowSlot += 8;
1155 // Emit the saves if needed.
1156 if (IsRCXLiveIn)
1157 addRegOffset(MIB: BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64mr)), Reg: X86::RSP, isKill: false,
1158 Offset: RCXShadowSlot)
1159 .addReg(RegNo: X86::RCX);
1160 if (IsRDXLiveIn)
1161 addRegOffset(MIB: BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64mr)), Reg: X86::RSP, isKill: false,
1162 Offset: RDXShadowSlot)
1163 .addReg(RegNo: X86::RDX);
1164 } else {
1165 // Not in the prolog. Copy RAX to a virtual reg.
1166 BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64rr), DestReg: SizeReg).addReg(RegNo: X86::RAX);
1167 }
1168
1169 // Add code to MBB to check for overflow and set the new target stack pointer
1170 // to zero if so.
1171 BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::XOR64rr), DestReg: ZeroReg)
1172 .addReg(RegNo: ZeroReg, Flags: RegState::Undef)
1173 .addReg(RegNo: ZeroReg, Flags: RegState::Undef)
1174 .setOperandDead(3); // implicit-def $eflags
1175 BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64rr), DestReg: CopyReg).addReg(RegNo: X86::RSP);
1176 BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::SUB64rr), DestReg: TestReg)
1177 .addReg(RegNo: CopyReg)
1178 .addReg(RegNo: SizeReg);
1179 BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::CMOV64rr), DestReg: FinalReg)
1180 .addReg(RegNo: TestReg)
1181 .addReg(RegNo: ZeroReg)
1182 .addImm(Val: X86::COND_B);
1183
1184 // FinalReg now holds final stack pointer value, or zero if
1185 // allocation would overflow. Compare against the current stack
1186 // limit from the thread environment block. Note this limit is the
1187 // lowest touched page on the stack, not the point at which the OS
1188 // will cause an overflow exception, so this is just an optimization
1189 // to avoid unnecessarily touching pages that are below the current
1190 // SP but already committed to the stack by the OS.
1191 BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64rm), DestReg: LimitReg)
1192 .addReg(RegNo: 0)
1193 .addImm(Val: 1)
1194 .addReg(RegNo: 0)
1195 .addImm(Val: ThreadEnvironmentStackLimit)
1196 .addReg(RegNo: X86::GS);
1197 BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::CMP64rr)).addReg(RegNo: FinalReg).addReg(RegNo: LimitReg);
1198 // Jump if the desired stack pointer is at or above the stack limit.
1199 BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
1200 .addMBB(MBB: ContinueMBB)
1201 .addImm(Val: X86::COND_AE);
1202
1203 // Add code to roundMBB to round the final stack pointer to a page boundary.
1204 if (InProlog)
1205 RoundMBB->addLiveIn(PhysReg: FinalReg);
1206 BuildMI(BB: RoundMBB, MIMD: DL, MCID: TII.get(Opcode: X86::AND64ri32), DestReg: RoundedReg)
1207 .addReg(RegNo: FinalReg)
1208 .addImm(Val: PageMask)
1209 .setOperandDead(3); // implicit-def $eflags
1210 BuildMI(BB: RoundMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JMP_1)).addMBB(MBB: LoopMBB);
1211
1212 // LimitReg now holds the current stack limit, RoundedReg page-rounded
1213 // final RSP value. Add code to loopMBB to decrement LimitReg page-by-page
1214 // and probe until we reach RoundedReg.
1215 if (!InProlog) {
1216 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII.get(Opcode: X86::PHI), DestReg: JoinReg)
1217 .addReg(RegNo: LimitReg)
1218 .addMBB(MBB: RoundMBB)
1219 .addReg(RegNo: ProbeReg)
1220 .addMBB(MBB: LoopMBB);
1221 }
1222
1223 if (InProlog)
1224 LoopMBB->addLiveIn(PhysReg: JoinReg);
1225 addRegOffset(MIB: BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII.get(Opcode: X86::LEA64r), DestReg: ProbeReg), Reg: JoinReg,
1226 isKill: false, Offset: -PageSize);
1227
1228 // Probe by storing a byte onto the stack.
1229 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII.get(Opcode: X86::MOV8mi))
1230 .addReg(RegNo: ProbeReg)
1231 .addImm(Val: 1)
1232 .addReg(RegNo: 0)
1233 .addImm(Val: 0)
1234 .addReg(RegNo: 0)
1235 .addImm(Val: 0);
1236
1237 if (InProlog)
1238 LoopMBB->addLiveIn(PhysReg: RoundedReg);
1239 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII.get(Opcode: X86::CMP64rr))
1240 .addReg(RegNo: RoundedReg)
1241 .addReg(RegNo: ProbeReg);
1242 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
1243 .addMBB(MBB: LoopMBB)
1244 .addImm(Val: X86::COND_NE);
1245
1246 MachineBasicBlock::iterator ContinueMBBI = ContinueMBB->getFirstNonPHI();
1247
1248 // If in prolog, restore RDX and RCX.
1249 if (InProlog) {
1250 if (RCXShadowSlot) // It means we spilled RCX in the prologue.
1251 addRegOffset(MIB: BuildMI(BB&: *ContinueMBB, I: ContinueMBBI, MIMD: DL,
1252 MCID: TII.get(Opcode: X86::MOV64rm), DestReg: X86::RCX),
1253 Reg: X86::RSP, isKill: false, Offset: RCXShadowSlot);
1254 if (RDXShadowSlot) // It means we spilled RDX in the prologue.
1255 addRegOffset(MIB: BuildMI(BB&: *ContinueMBB, I: ContinueMBBI, MIMD: DL,
1256 MCID: TII.get(Opcode: X86::MOV64rm), DestReg: X86::RDX),
1257 Reg: X86::RSP, isKill: false, Offset: RDXShadowSlot);
1258 }
1259
1260 // Now that the probing is done, add code to continueMBB to update
1261 // the stack pointer for real.
1262 BuildMI(BB&: *ContinueMBB, I: ContinueMBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SUB64rr), DestReg: X86::RSP)
1263 .addReg(RegNo: X86::RSP)
1264 .addReg(RegNo: SizeReg)
1265 .setOperandDead(3); // implicit-def $eflags
1266
1267 // Add the control flow edges we need.
1268 MBB.addSuccessor(Succ: ContinueMBB);
1269 MBB.addSuccessor(Succ: RoundMBB);
1270 RoundMBB->addSuccessor(Succ: LoopMBB);
1271 LoopMBB->addSuccessor(Succ: ContinueMBB);
1272 LoopMBB->addSuccessor(Succ: LoopMBB);
1273
1274 if (InProlog) {
1275 LivePhysRegs LiveRegs;
1276 computeAndAddLiveIns(LiveRegs, MBB&: *ContinueMBB);
1277 }
1278
1279 // Mark all the instructions added to the prolog as frame setup.
1280 if (InProlog) {
1281 for (++BeforeMBBI; BeforeMBBI != MBB.end(); ++BeforeMBBI) {
1282 BeforeMBBI->setFlag(MachineInstr::FrameSetup);
1283 }
1284 for (MachineInstr &MI : *RoundMBB) {
1285 MI.setFlag(MachineInstr::FrameSetup);
1286 }
1287 for (MachineInstr &MI : *LoopMBB) {
1288 MI.setFlag(MachineInstr::FrameSetup);
1289 }
1290 for (MachineInstr &MI :
1291 llvm::make_range(x: ContinueMBB->begin(), y: ContinueMBBI)) {
1292 MI.setFlag(MachineInstr::FrameSetup);
1293 }
1294 }
1295}
1296
1297void X86FrameLowering::emitStackProbeCall(
1298 MachineFunction &MF, MachineBasicBlock &MBB,
1299 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog,
1300 std::optional<MachineFunction::DebugInstrOperandPair> InstrNum) const {
1301 bool IsLargeCodeModel = MF.getTarget().getCodeModel() == CodeModel::Large;
1302
1303 // FIXME: Add indirect thunk support and remove this.
1304 if (Is64Bit && IsLargeCodeModel && STI.useIndirectThunkCalls())
1305 report_fatal_error(reason: "Emitting stack probe calls on 64-bit with the large "
1306 "code model and indirect thunks not yet implemented.");
1307
1308 assert(MBB.computeRegisterLiveness(TRI, X86::EFLAGS, MBBI) !=
1309 MachineBasicBlock::LQR_Live &&
1310 "Stack probe calls will clobber live EFLAGS.");
1311
1312 unsigned CallOp;
1313 if (Is64Bit)
1314 CallOp = IsLargeCodeModel ? X86::CALL64r : X86::CALL64pcrel32;
1315 else
1316 CallOp = X86::CALLpcrel32;
1317
1318 StringRef Symbol = STI.getTargetLowering()->getStackProbeSymbolName(MF);
1319
1320 MachineInstrBuilder CI;
1321 MachineBasicBlock::iterator ExpansionMBBI = std::prev(x: MBBI);
1322
1323 // All current stack probes take AX and SP as input, clobber flags, and
1324 // preserve all registers. x86_64 probes leave RSP unmodified.
1325 if (Is64Bit && MF.getTarget().getCodeModel() == CodeModel::Large) {
1326 // For the large code model, we have to call through a register. Use R11,
1327 // as it is scratch in all supported calling conventions.
1328 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64ri), DestReg: X86::R11)
1329 .addExternalSymbol(FnName: MF.createExternalSymbolName(Name: Symbol));
1330 CI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: CallOp)).addReg(RegNo: X86::R11);
1331 } else {
1332 CI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: CallOp))
1333 .addExternalSymbol(FnName: MF.createExternalSymbolName(Name: Symbol));
1334 }
1335
1336 unsigned AX = Uses64BitFramePtr ? X86::RAX : X86::EAX;
1337 unsigned SP = Uses64BitFramePtr ? X86::RSP : X86::ESP;
1338 bool EmitSPSub = STI.isTargetWin64() || !STI.isOSWindows();
1339
1340 CI.addReg(RegNo: AX, Flags: RegState::Implicit)
1341 .addReg(RegNo: SP, Flags: RegState::Implicit)
1342 .addReg(RegNo: AX, Flags: RegState::Define | RegState::Implicit |
1343 getDeadRegState(B: !EmitSPSub))
1344 .addReg(RegNo: SP, Flags: RegState::Define | RegState::Implicit)
1345 .addReg(RegNo: X86::EFLAGS,
1346 Flags: RegState::Define | RegState::Implicit | RegState::Dead);
1347
1348 MachineInstr *ModInst = CI;
1349 if (EmitSPSub) {
1350 // MSVC x32's _chkstk and cygwin/mingw's _alloca adjust %esp themselves.
1351 // MSVC x64's __chkstk and cygwin/mingw's ___chkstk_ms do not adjust %rsp
1352 // themselves. They also does not clobber %rax so we can reuse it when
1353 // adjusting %rsp.
1354 // All other platforms do not specify a particular ABI for the stack probe
1355 // function, so we arbitrarily define it to not adjust %esp/%rsp itself.
1356 ModInst =
1357 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: getSUBrrOpcode(IsLP64: Uses64BitFramePtr)), DestReg: SP)
1358 .addReg(RegNo: SP)
1359 .addReg(RegNo: AX)
1360 .setOperandDead(3); // implicit-def $eflags
1361 }
1362
1363 // DebugInfo variable locations -- if there's an instruction number for the
1364 // allocation (i.e., DYN_ALLOC_*), substitute it for the instruction that
1365 // modifies SP.
1366 if (InstrNum) {
1367 if (EmitSPSub) {
1368 // Label destination operand of the subtract.
1369 MF.makeDebugValueSubstitution(*InstrNum,
1370 {ModInst->getDebugInstrNum(), 0});
1371 } else {
1372 // Label the call. The operand number is the penultimate operand, zero
1373 // based.
1374 unsigned SPDefOperand = ModInst->getNumOperands() - 2;
1375 MF.makeDebugValueSubstitution(
1376 *InstrNum, {ModInst->getDebugInstrNum(), SPDefOperand});
1377 }
1378 }
1379
1380 if (InProlog) {
1381 // Apply the frame setup flag to all inserted instrs.
1382 for (++ExpansionMBBI; ExpansionMBBI != MBBI; ++ExpansionMBBI)
1383 ExpansionMBBI->setFlag(MachineInstr::FrameSetup);
1384 }
1385}
1386
1387static unsigned calculateSetFPREG(uint64_t SPAdjust) {
1388 // Win64 ABI has a less restrictive limitation of 240; 128 works equally well
1389 // and might require smaller successive adjustments.
1390 const uint64_t Win64MaxSEHOffset = 128;
1391 uint64_t SEHFrameOffset = std::min(a: SPAdjust, b: Win64MaxSEHOffset);
1392 // Win64 ABI requires 16-byte alignment for the UWOP_SET_FPREG opcode.
1393 return SEHFrameOffset & -16;
1394}
1395
1396// If we're forcing a stack realignment we can't rely on just the frame
1397// info, we need to know the ABI stack alignment as well in case we
1398// have a call out. Otherwise just make sure we have some alignment - we'll
1399// go with the minimum SlotSize.
1400uint64_t
1401X86FrameLowering::calculateMaxStackAlign(const MachineFunction &MF) const {
1402 const MachineFrameInfo &MFI = MF.getFrameInfo();
1403 Align MaxAlign = MFI.getMaxAlign(); // Desired stack alignment.
1404 Align StackAlign = getStackAlign();
1405 bool HasRealign = MF.getFunction().hasFnAttribute(Kind: "stackrealign");
1406 if (HasRealign) {
1407 if (MFI.hasCalls())
1408 MaxAlign = (StackAlign > MaxAlign) ? StackAlign : MaxAlign;
1409 else if (MaxAlign < SlotSize)
1410 MaxAlign = Align(SlotSize);
1411 }
1412
1413 if (!Is64Bit && MF.getFunction().getCallingConv() == CallingConv::X86_INTR) {
1414 if (HasRealign)
1415 MaxAlign = (MaxAlign > 16) ? MaxAlign : Align(16);
1416 else
1417 MaxAlign = Align(16);
1418 }
1419 return MaxAlign.value();
1420}
1421
1422void X86FrameLowering::BuildStackAlignAND(MachineBasicBlock &MBB,
1423 MachineBasicBlock::iterator MBBI,
1424 const DebugLoc &DL, Register Reg,
1425 uint64_t MaxAlign) const {
1426 uint64_t Val = -MaxAlign;
1427 unsigned AndOp = getANDriOpcode(IsLP64: Uses64BitFramePtr, Imm: Val);
1428
1429 MachineFunction &MF = *MBB.getParent();
1430 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
1431 const X86TargetLowering &TLI = *STI.getTargetLowering();
1432 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
1433 const bool EmitInlineStackProbe = TLI.hasInlineStackProbe(MF);
1434
1435 // We want to make sure that (in worst case) less than StackProbeSize bytes
1436 // are not probed after the AND. This assumption is used in
1437 // emitStackProbeInlineGeneric.
1438 if (Reg == StackPtr && EmitInlineStackProbe && MaxAlign >= StackProbeSize) {
1439 {
1440 NumFrameLoopProbe++;
1441 MachineBasicBlock *entryMBB =
1442 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
1443 MachineBasicBlock *headMBB =
1444 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
1445 MachineBasicBlock *bodyMBB =
1446 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
1447 MachineBasicBlock *footMBB =
1448 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
1449
1450 MachineFunction::iterator MBBIter = MBB.getIterator();
1451 MF.insert(MBBI: MBBIter, MBB: entryMBB);
1452 MF.insert(MBBI: MBBIter, MBB: headMBB);
1453 MF.insert(MBBI: MBBIter, MBB: bodyMBB);
1454 MF.insert(MBBI: MBBIter, MBB: footMBB);
1455 const unsigned MovMIOpc = Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
1456 Register FinalStackProbed = Uses64BitFramePtr ? X86::R11
1457 : Is64Bit ? X86::R11D
1458 : X86::EAX;
1459
1460 // Setup entry block
1461 {
1462
1463 entryMBB->splice(Where: entryMBB->end(), Other: &MBB, From: MBB.begin(), To: MBBI);
1464 BuildMI(BB: entryMBB, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: FinalStackProbed)
1465 .addReg(RegNo: StackPtr)
1466 .setMIFlag(MachineInstr::FrameSetup);
1467 MachineInstr *MI =
1468 BuildMI(BB: entryMBB, MIMD: DL, MCID: TII.get(Opcode: AndOp), DestReg: FinalStackProbed)
1469 .addReg(RegNo: FinalStackProbed)
1470 .addImm(Val)
1471 .setMIFlag(MachineInstr::FrameSetup);
1472
1473 // The EFLAGS implicit def is dead.
1474 MI->getOperand(i: 3).setIsDead();
1475
1476 BuildMI(BB: entryMBB, MIMD: DL,
1477 MCID: TII.get(Opcode: Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
1478 .addReg(RegNo: FinalStackProbed)
1479 .addReg(RegNo: StackPtr)
1480 .setMIFlag(MachineInstr::FrameSetup);
1481 BuildMI(BB: entryMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
1482 .addMBB(MBB: &MBB)
1483 .addImm(Val: X86::COND_E)
1484 .setMIFlag(MachineInstr::FrameSetup);
1485 entryMBB->addSuccessor(Succ: headMBB);
1486 entryMBB->addSuccessor(Succ: &MBB);
1487 }
1488
1489 // Loop entry block
1490
1491 {
1492 const unsigned SUBOpc = getSUBriOpcode(IsLP64: Uses64BitFramePtr);
1493 BuildMI(BB: headMBB, MIMD: DL, MCID: TII.get(Opcode: SUBOpc), DestReg: StackPtr)
1494 .addReg(RegNo: StackPtr)
1495 .addImm(Val: StackProbeSize)
1496 .setMIFlag(MachineInstr::FrameSetup)
1497 .setOperandDead(3); // implicit-def $eflags
1498
1499 BuildMI(BB: headMBB, MIMD: DL,
1500 MCID: TII.get(Opcode: Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
1501 .addReg(RegNo: StackPtr)
1502 .addReg(RegNo: FinalStackProbed)
1503 .setMIFlag(MachineInstr::FrameSetup);
1504
1505 // jump to the footer if StackPtr < FinalStackProbed
1506 BuildMI(BB: headMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
1507 .addMBB(MBB: footMBB)
1508 .addImm(Val: X86::COND_B)
1509 .setMIFlag(MachineInstr::FrameSetup);
1510
1511 headMBB->addSuccessor(Succ: bodyMBB);
1512 headMBB->addSuccessor(Succ: footMBB);
1513 }
1514
1515 // setup loop body
1516 {
1517 addRegOffset(MIB: BuildMI(BB: bodyMBB, MIMD: DL, MCID: TII.get(Opcode: MovMIOpc))
1518 .setMIFlag(MachineInstr::FrameSetup),
1519 Reg: StackPtr, isKill: false, Offset: 0)
1520 .addImm(Val: 0)
1521 .setMIFlag(MachineInstr::FrameSetup);
1522
1523 const unsigned SUBOpc = getSUBriOpcode(IsLP64: Uses64BitFramePtr);
1524 BuildMI(BB: bodyMBB, MIMD: DL, MCID: TII.get(Opcode: SUBOpc), DestReg: StackPtr)
1525 .addReg(RegNo: StackPtr)
1526 .addImm(Val: StackProbeSize)
1527 .setMIFlag(MachineInstr::FrameSetup)
1528 .setOperandDead(3); // implicit-def $eflags
1529
1530 // cmp with stack pointer bound
1531 BuildMI(BB: bodyMBB, MIMD: DL,
1532 MCID: TII.get(Opcode: Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
1533 .addReg(RegNo: FinalStackProbed)
1534 .addReg(RegNo: StackPtr)
1535 .setMIFlag(MachineInstr::FrameSetup);
1536
1537 // jump back while FinalStackProbed < StackPtr
1538 BuildMI(BB: bodyMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
1539 .addMBB(MBB: bodyMBB)
1540 .addImm(Val: X86::COND_B)
1541 .setMIFlag(MachineInstr::FrameSetup);
1542 bodyMBB->addSuccessor(Succ: bodyMBB);
1543 bodyMBB->addSuccessor(Succ: footMBB);
1544 }
1545
1546 // setup loop footer
1547 {
1548 BuildMI(BB: footMBB, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: StackPtr)
1549 .addReg(RegNo: FinalStackProbed)
1550 .setMIFlag(MachineInstr::FrameSetup);
1551 addRegOffset(MIB: BuildMI(BB: footMBB, MIMD: DL, MCID: TII.get(Opcode: MovMIOpc))
1552 .setMIFlag(MachineInstr::FrameSetup),
1553 Reg: StackPtr, isKill: false, Offset: 0)
1554 .addImm(Val: 0)
1555 .setMIFlag(MachineInstr::FrameSetup);
1556 footMBB->addSuccessor(Succ: &MBB);
1557 }
1558
1559 fullyRecomputeLiveIns(MBBs: {footMBB, bodyMBB, headMBB, &MBB});
1560 }
1561 } else {
1562 MachineInstr *MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AndOp), DestReg: Reg)
1563 .addReg(RegNo: Reg)
1564 .addImm(Val)
1565 .setMIFlag(MachineInstr::FrameSetup);
1566
1567 // The EFLAGS implicit def is dead.
1568 MI->getOperand(i: 3).setIsDead();
1569 }
1570}
1571
1572bool X86FrameLowering::has128ByteRedZone(const MachineFunction &MF) const {
1573 // x86-64 (non Win64) has a 128 byte red zone which is guaranteed not to be
1574 // clobbered by any interrupt handler.
1575 assert(&STI == &MF.getSubtarget<X86Subtarget>() &&
1576 "MF used frame lowering for wrong subtarget");
1577 const Function &Fn = MF.getFunction();
1578 const bool IsWin64CC = STI.isCallingConvWin64(CC: Fn.getCallingConv());
1579 return Is64Bit && !IsWin64CC && !Fn.hasFnAttribute(Kind: Attribute::NoRedZone);
1580}
1581
1582bool X86FrameLowering::needsDwarfCFI(const MachineFunction &MF) const {
1583 return !MF.usesWindowsUnwindInfo() && MF.needsFrameMoves();
1584}
1585
1586/// Return true if an opcode is part of the REP group of instructions
1587static bool isOpcodeRep(unsigned Opcode) {
1588 switch (Opcode) {
1589 case X86::REPNE_PREFIX:
1590 case X86::REP_MOVSB_32:
1591 case X86::REP_MOVSB_64:
1592 case X86::REP_MOVSD_32:
1593 case X86::REP_MOVSD_64:
1594 case X86::REP_MOVSQ_32:
1595 case X86::REP_MOVSQ_64:
1596 case X86::REP_MOVSW_32:
1597 case X86::REP_MOVSW_64:
1598 case X86::REP_PREFIX:
1599 case X86::REP_STOSB_32:
1600 case X86::REP_STOSB_64:
1601 case X86::REP_STOSD_32:
1602 case X86::REP_STOSD_64:
1603 case X86::REP_STOSQ_32:
1604 case X86::REP_STOSQ_64:
1605 case X86::REP_STOSW_32:
1606 case X86::REP_STOSW_64:
1607 return true;
1608 default:
1609 break;
1610 }
1611 return false;
1612}
1613
1614/// Returns the number of bytes between the end of the fixed and callee-save
1615/// area and the first local object, which PEI leaves as padding when it aligns
1616/// the local objects relative to the incoming stack pointer.
1617static uint64_t getUnusedLocalAreaPadding(const MachineFrameInfo &MFI) {
1618 int64_t FixedEnd = 0;
1619 int64_t LocalsTop = std::numeric_limits<int64_t>::max();
1620 for (int I : seq(Begin: MFI.getObjectIndexBegin(), End: MFI.getObjectIndexEnd())) {
1621 if (MFI.isDeadObjectIndex(ObjectIdx: I) || MFI.isVariableSizedObjectIndex(ObjectIdx: I) ||
1622 MFI.getStackID(ObjectIdx: I) != TargetStackID::Default)
1623 continue;
1624
1625 int64_t ObjOffset = MFI.getObjectOffset(ObjectIdx: I);
1626 int64_t ObjSize = MFI.getObjectSize(ObjectIdx: I);
1627 // Offsets are negative, measured from the incoming stack pointer.
1628 if (MFI.isFixedObjectIndex(ObjectIdx: I))
1629 FixedEnd = std::max(a: FixedEnd, b: -ObjOffset);
1630 else
1631 LocalsTop = std::min<int64_t>(a: LocalsTop, b: -ObjOffset - ObjSize);
1632 }
1633 if (LocalsTop == std::numeric_limits<int64_t>::max())
1634 return 0;
1635
1636 assert(LocalsTop >= FixedEnd && "Local object overlaps the fixed area");
1637 return LocalsTop - FixedEnd;
1638}
1639
1640/// emitPrologue - Push callee-saved registers onto the stack, which
1641/// automatically adjust the stack pointer. Adjust the stack pointer to allocate
1642/// space for local variables. Also emit labels used by the exception handler to
1643/// generate the exception handling frames.
1644
1645/*
1646 Here's a gist of what gets emitted:
1647
1648 ; Establish frame pointer, if needed
1649 [if needs FP]
1650 push %rbp
1651 .cfi_def_cfa_offset 16
1652 .cfi_offset %rbp, -16
1653 .seh_pushreg %rpb
1654 mov %rsp, %rbp
1655 .cfi_def_cfa_register %rbp
1656
1657 ; Spill general-purpose registers
1658 [for all callee-saved GPRs]
1659 pushq %<reg>
1660 [if not needs FP]
1661 .cfi_def_cfa_offset (offset from RETADDR)
1662 .seh_pushreg %<reg>
1663
1664 ; If the required stack alignment > default stack alignment
1665 ; rsp needs to be re-aligned. This creates a "re-alignment gap"
1666 ; of unknown size in the stack frame.
1667 [if stack needs re-alignment]
1668 and $MASK, %rsp
1669
1670 ; Allocate space for locals
1671 [if target is Windows and allocated space > 4096 bytes]
1672 ; Windows needs special care for allocations larger
1673 ; than one page.
1674 mov $NNN, %rax
1675 call ___chkstk_ms/___chkstk
1676 sub %rax, %rsp
1677 [else]
1678 sub $NNN, %rsp
1679
1680 [if needs FP]
1681 .seh_stackalloc (size of XMM spill slots)
1682 .seh_setframe %rbp, SEHFrameOffset ; = size of all spill slots
1683 [else]
1684 .seh_stackalloc NNN
1685
1686 ; Spill XMMs
1687 ; Note, that while only Windows 64 ABI specifies XMMs as callee-preserved,
1688 ; they may get spilled on any platform, if the current function
1689 ; calls @llvm.eh.unwind.init
1690 [if needs FP]
1691 [for all callee-saved XMM registers]
1692 movaps %<xmm reg>, -MMM(%rbp)
1693 [for all callee-saved XMM registers]
1694 .seh_savexmm %<xmm reg>, (-MMM + SEHFrameOffset)
1695 ; i.e. the offset relative to (%rbp - SEHFrameOffset)
1696 [else]
1697 [for all callee-saved XMM registers]
1698 movaps %<xmm reg>, KKK(%rsp)
1699 [for all callee-saved XMM registers]
1700 .seh_savexmm %<xmm reg>, KKK
1701
1702 .seh_endprologue
1703
1704 [if needs base pointer]
1705 mov %rsp, %rbx
1706 [if needs to restore base pointer]
1707 mov %rsp, -MMM(%rbp)
1708
1709 ; Emit CFI info
1710 [if needs FP]
1711 [for all callee-saved registers]
1712 .cfi_offset %<reg>, (offset from %rbp)
1713 [else]
1714 .cfi_def_cfa_offset (offset from RETADDR)
1715 [for all callee-saved registers]
1716 .cfi_offset %<reg>, (offset from %rsp)
1717
1718 Notes:
1719 - .seh directives are emitted only for Windows 64 ABI
1720 - .cv_fpo directives are emitted on win32 when emitting CodeView
1721 - .cfi directives are emitted for all other ABIs
1722 - for 32-bit code, substitute %e?? registers for %r??
1723*/
1724
1725void X86FrameLowering::emitPrologue(MachineFunction &MF,
1726 MachineBasicBlock &MBB) const {
1727 assert(&STI == &MF.getSubtarget<X86Subtarget>() &&
1728 "MF used frame lowering for wrong subtarget");
1729 MachineBasicBlock::iterator MBBI = MBB.begin();
1730 MachineFrameInfo &MFI = MF.getFrameInfo();
1731 const Function &Fn = MF.getFunction();
1732 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
1733 uint64_t MaxAlign = calculateMaxStackAlign(MF); // Desired stack alignment.
1734 uint64_t StackSize = MFI.getStackSize(); // Number of bytes to allocate.
1735 bool IsFunclet = MBB.isEHFuncletEntry();
1736 EHPersonality Personality = EHPersonality::Unknown;
1737 if (Fn.hasPersonalityFn())
1738 Personality = classifyEHPersonality(Pers: Fn.getPersonalityFn());
1739 bool FnHasClrFunclet =
1740 MF.hasEHFunclets() && Personality == EHPersonality::CoreCLR;
1741 bool IsClrFunclet = IsFunclet && FnHasClrFunclet;
1742 bool HasFP = hasFP(MF);
1743 bool IsWin64Prologue = MF.usesWindowsUnwindInfo();
1744 bool NeedsWin64CFI = IsWin64Prologue && Fn.needsUnwindTableEntry();
1745 // FIXME: Emit FPO data for EH funclets.
1746 bool NeedsWinFPO = !IsFunclet && STI.isTargetWin32() &&
1747 MF.getFunction().getParent()->getCodeViewFlag();
1748 bool NeedsWinCFI = NeedsWin64CFI || NeedsWinFPO;
1749 bool NeedsDwarfCFI = needsDwarfCFI(MF);
1750 bool IsWin64UnwindV3 = NeedsWin64CFI && requireWinX64UnwindV3(MF);
1751 Register FramePtr = TRI->getFrameRegister(MF);
1752 const Register MachineFramePtr =
1753 STI.isTarget64BitILP32() ? Register(getX86SubSuperRegister(Reg: FramePtr, Size: 64))
1754 : FramePtr;
1755 Register BasePtr = TRI->getBaseRegister();
1756 bool HasWinCFI = false;
1757
1758 // Helpers to emit Windows x64 unwind SEH pseudos with the correct placement.
1759 // V1/V2: pseudo goes after the real instruction.
1760 // V3: pseudo goes before the real instruction.
1761 // Usage:
1762 // EmitSEHBefore([&]{ BuildMI(...SEH_PushReg...); });
1763 // BuildMI(... real instruction ...);
1764 // EmitSEHAfter([&]{ BuildMI(...SEH_PushReg...); });
1765 auto EmitSEHBefore = [&](auto EmitFn) {
1766 if (NeedsWinCFI && IsWin64UnwindV3) {
1767 HasWinCFI = true;
1768 EmitFn();
1769 }
1770 };
1771 auto EmitSEHAfter = [&](auto EmitFn) {
1772 if (NeedsWinCFI && !IsWin64UnwindV3) {
1773 HasWinCFI = true;
1774 EmitFn();
1775 }
1776 };
1777
1778 // Debug location must be unknown since the first debug location is used
1779 // to determine the end of the prologue.
1780 DebugLoc DL;
1781 Register ArgBaseReg;
1782
1783 // Emit extra prolog for argument stack slot reference.
1784 if (auto *MI = X86FI->getStackPtrSaveMI()) {
1785 // MI is lea instruction that created in X86ArgumentStackSlotPass.
1786 // Creat extra prolog for stack realignment.
1787 ArgBaseReg = MI->getOperand(i: 0).getReg();
1788 // leal 4(%esp), %basereg
1789 // .cfi_def_cfa %basereg, 0
1790 // andl $-128, %esp
1791 // pushl -4(%basereg)
1792 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Is64Bit ? X86::LEA64r : X86::LEA32r),
1793 DestReg: ArgBaseReg)
1794 .addUse(RegNo: StackPtr)
1795 .addImm(Val: 1)
1796 .addUse(RegNo: X86::NoRegister)
1797 .addImm(Val: SlotSize)
1798 .addUse(RegNo: X86::NoRegister)
1799 .setMIFlag(MachineInstr::FrameSetup);
1800 if (NeedsDwarfCFI) {
1801 // .cfi_def_cfa %basereg, 0
1802 unsigned DwarfStackPtr = TRI->getDwarfRegNum(Reg: ArgBaseReg, isEH: true);
1803 BuildCFI(MBB, MBBI, DL,
1804 CFIInst: MCCFIInstruction::cfiDefCfa(L: nullptr, Register: DwarfStackPtr, Offset: 0),
1805 Flag: MachineInstr::FrameSetup);
1806 }
1807 BuildStackAlignAND(MBB, MBBI, DL, Reg: StackPtr, MaxAlign);
1808 int64_t Offset = -(int64_t)SlotSize;
1809 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Is64Bit ? X86::PUSH64rmm : X86::PUSH32rmm))
1810 .addReg(RegNo: ArgBaseReg)
1811 .addImm(Val: 1)
1812 .addReg(RegNo: X86::NoRegister)
1813 .addImm(Val: Offset)
1814 .addReg(RegNo: X86::NoRegister)
1815 .setMIFlag(MachineInstr::FrameSetup);
1816 }
1817
1818 // Space reserved for stack-based arguments when making a (ABI-guaranteed)
1819 // tail call.
1820 unsigned TailCallArgReserveSize = -X86FI->getTCReturnAddrDelta();
1821 if (TailCallArgReserveSize && IsWin64Prologue)
1822 report_fatal_error(reason: "Can't handle guaranteed tail call under win64 yet");
1823
1824 const bool EmitStackProbeCall =
1825 STI.getTargetLowering()->hasStackProbeSymbol(MF);
1826 unsigned StackProbeSize = STI.getTargetLowering()->getStackProbeSize(MF);
1827
1828 if (HasFP && X86FI->hasSwiftAsyncContext()) {
1829 switch (MF.getTarget().Options.SwiftAsyncFramePointer) {
1830 case SwiftAsyncFramePointerMode::DeploymentBased:
1831 if (STI.swiftAsyncContextIsDynamicallySet()) {
1832 // The special symbol below is absolute and has a *value* suitable to be
1833 // combined with the frame pointer directly.
1834 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::OR64rm), DestReg: MachineFramePtr)
1835 .addUse(RegNo: MachineFramePtr)
1836 .addUse(RegNo: X86::RIP)
1837 .addImm(Val: 1)
1838 .addUse(RegNo: X86::NoRegister)
1839 .addExternalSymbol(FnName: "swift_async_extendedFramePointerFlags",
1840 TargetFlags: X86II::MO_GOTPCREL)
1841 .addUse(RegNo: X86::NoRegister);
1842 break;
1843 }
1844 [[fallthrough]];
1845
1846 case SwiftAsyncFramePointerMode::Always:
1847 assert(
1848 !IsWin64Prologue &&
1849 "win64 prologue does not set the bit 60 in the saved frame pointer");
1850 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::BTS64ri8), DestReg: MachineFramePtr)
1851 .addUse(RegNo: MachineFramePtr)
1852 .addImm(Val: 60)
1853 .setMIFlag(MachineInstr::FrameSetup);
1854 break;
1855
1856 case SwiftAsyncFramePointerMode::Never:
1857 break;
1858 }
1859 }
1860
1861 // Re-align the stack on 64-bit if the x86-interrupt calling convention is
1862 // used and an error code was pushed, since the x86-64 ABI requires a 16-byte
1863 // stack alignment.
1864 if (Fn.getCallingConv() == CallingConv::X86_INTR && Is64Bit &&
1865 Fn.arg_size() == 2) {
1866 // Update the stack pointer by pushing a register. This is the instruction
1867 // emitted that would be end up being emitted by a call to `emitSPUpdate`.
1868 // Hard-coding the update to a push avoids emitting a second
1869 // `STACKALLOC_W_PROBING` instruction in the save block: We know that stack
1870 // probing isn't needed anyways for an 8-byte update.
1871 // Pushing a register leaves us in a similar situation to a regular
1872 // function call where we know that the address at (rsp-8) is writeable.
1873 // That way we avoid any off-by-ones with stack probing for additional
1874 // stack pointer updates later on.
1875 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH64r))
1876 .addReg(RegNo: X86::RAX, Flags: RegState::Undef)
1877 .setMIFlag(MachineInstr::FrameSetup);
1878 }
1879
1880 // If this is x86-64 and the Red Zone is not disabled, if we are a leaf
1881 // function, and use up to 128 bytes of stack space, don't have a frame
1882 // pointer, calls, or dynamic alloca then we do not need to adjust the
1883 // stack pointer (we fit in the Red Zone). We also check that we don't
1884 // push and pop from the stack.
1885 if (has128ByteRedZone(MF) && !TRI->hasStackRealignment(MF) &&
1886 !MFI.hasVarSizedObjects() && // No dynamic alloca.
1887 !MFI.adjustsStack() && // No calls.
1888 !EmitStackProbeCall && // No stack probes.
1889 !MFI.hasCopyImplyingStackAdjustment() && // Don't push and pop.
1890 !MF.shouldSplitStack()) { // Regular stack
1891 uint64_t MinSize =
1892 X86FI->getCalleeSavedFrameSize() - X86FI->getTCReturnAddrDelta();
1893 if (HasFP)
1894 MinSize += SlotSize;
1895 X86FI->setUsesRedZone(MinSize > 0 || StackSize > 0);
1896 StackSize = std::max(a: MinSize, b: StackSize > 128 ? StackSize - 128 : 0);
1897 MFI.setStackSize(StackSize);
1898 }
1899
1900 // Insert stack pointer adjustment for later moving of return addr. Only
1901 // applies to tail call optimized functions where the callee argument stack
1902 // size is bigger than the callers.
1903 if (TailCallArgReserveSize != 0) {
1904 BuildStackAdjustment(MBB, MBBI, DL, Offset: -(int)TailCallArgReserveSize,
1905 /*InEpilogue=*/false)
1906 .setMIFlag(MachineInstr::FrameSetup);
1907 }
1908
1909 // Mapping for machine moves:
1910 //
1911 // DST: VirtualFP AND
1912 // SRC: VirtualFP => DW_CFA_def_cfa_offset
1913 // ELSE => DW_CFA_def_cfa
1914 //
1915 // SRC: VirtualFP AND
1916 // DST: Register => DW_CFA_def_cfa_register
1917 //
1918 // ELSE
1919 // OFFSET < 0 => DW_CFA_offset_extended_sf
1920 // REG < 64 => DW_CFA_offset + Reg
1921 // ELSE => DW_CFA_offset_extended
1922
1923 uint64_t NumBytes = 0;
1924 int stackGrowth = -SlotSize;
1925
1926 // Find the funclet establisher parameter
1927 MCRegister Establisher;
1928 if (IsClrFunclet)
1929 Establisher = Uses64BitFramePtr ? X86::RCX : X86::ECX;
1930 else if (IsFunclet)
1931 Establisher = Uses64BitFramePtr ? X86::RDX : X86::EDX;
1932
1933 if (IsWin64Prologue && IsFunclet && !IsClrFunclet) {
1934 // Immediately spill establisher into the home slot.
1935 // The runtime cares about this.
1936 // MOV64mr %rdx, 16(%rsp)
1937 unsigned MOVmr = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr;
1938 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: MOVmr)), Reg: StackPtr, isKill: true, Offset: 16)
1939 .addReg(RegNo: Establisher)
1940 .setMIFlag(MachineInstr::FrameSetup);
1941 MBB.addLiveIn(PhysReg: Establisher);
1942 }
1943
1944 if (HasFP) {
1945 assert(MF.getRegInfo().isReserved(MachineFramePtr) && "FP reserved");
1946
1947 // Calculate required stack adjustment.
1948 uint64_t FrameSize = StackSize - SlotSize;
1949 NumBytes =
1950 FrameSize - (X86FI->getCalleeSavedFrameSize() + TailCallArgReserveSize);
1951
1952 // Callee-saved registers are pushed on stack before the stack is realigned,
1953 // and the realignment itself already provides the local objects' alignment,
1954 // so leave out the padding PEI put in front of them for it.
1955 if (TRI->hasStackRealignment(MF) && !IsWin64Prologue) {
1956 uint64_t Padding = getUnusedLocalAreaPadding(MFI);
1957 assert(Padding <= NumBytes && "Padding exceeds the local area");
1958 NumBytes = alignTo(Value: NumBytes - Padding, Align: MaxAlign);
1959 }
1960
1961 // Save EBP/RBP into the appropriate stack slot.
1962 auto EmitSEHPushFramePtr = [&]() {
1963 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_PushReg))
1964 .addImm(Val: FramePtr)
1965 .setMIFlag(MachineInstr::FrameSetup);
1966 };
1967 EmitSEHBefore(EmitSEHPushFramePtr);
1968 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
1969 MCID: TII.get(Opcode: getPUSHOpcode(ST: MF.getSubtarget<X86Subtarget>())))
1970 .addReg(RegNo: MachineFramePtr, Flags: RegState::Kill)
1971 .setMIFlag(MachineInstr::FrameSetup);
1972 EmitSEHAfter(EmitSEHPushFramePtr);
1973
1974 if (NeedsDwarfCFI && !ArgBaseReg.isValid()) {
1975 // Mark the place where EBP/RBP was saved.
1976 // Define the current CFA rule to use the provided offset.
1977 assert(StackSize);
1978 BuildCFI(MBB, MBBI, DL,
1979 CFIInst: MCCFIInstruction::cfiDefCfaOffset(
1980 L: nullptr, Offset: -2 * stackGrowth + (int)TailCallArgReserveSize),
1981 Flag: MachineInstr::FrameSetup);
1982
1983 // Change the rule for the FramePtr to be an "offset" rule.
1984 unsigned DwarfFramePtr = TRI->getDwarfRegNum(Reg: MachineFramePtr, isEH: true);
1985 BuildCFI(MBB, MBBI, DL,
1986 CFIInst: MCCFIInstruction::createOffset(L: nullptr, Register: DwarfFramePtr,
1987 Offset: 2 * stackGrowth -
1988 (int)TailCallArgReserveSize),
1989 Flag: MachineInstr::FrameSetup);
1990 }
1991
1992 if (!IsFunclet) {
1993 if (X86FI->hasSwiftAsyncContext()) {
1994 assert(!IsWin64Prologue &&
1995 "win64 prologue does not store async context right below rbp");
1996 const auto &Attrs = MF.getFunction().getAttributes();
1997
1998 // Before we update the live frame pointer we have to ensure there's a
1999 // valid (or null) asynchronous context in its slot just before FP in
2000 // the frame record, so store it now.
2001 auto EmitSEHPushR14 = [&]() {
2002 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_PushReg))
2003 .addImm(Val: X86::R14)
2004 .setMIFlag(MachineInstr::FrameSetup);
2005 };
2006 EmitSEHBefore(EmitSEHPushR14);
2007 if (Attrs.hasAttrSomewhere(Kind: Attribute::SwiftAsync)) {
2008 // We have an initial context in r14, store it just before the frame
2009 // pointer.
2010 MBB.addLiveIn(PhysReg: X86::R14);
2011 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH64r))
2012 .addReg(RegNo: X86::R14)
2013 .setMIFlag(MachineInstr::FrameSetup);
2014 } else {
2015 // No initial context, store null so that there's no pointer that
2016 // could be misused.
2017 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH64i32))
2018 .addImm(Val: 0)
2019 .setMIFlag(MachineInstr::FrameSetup);
2020 }
2021
2022 // Update CFA offset for the async-context push.
2023 if (NeedsDwarfCFI && !ArgBaseReg.isValid()) {
2024 BuildCFI(
2025 MBB, MBBI, DL,
2026 CFIInst: MCCFIInstruction::createAdjustCfaOffset(L: nullptr, Adjustment: -stackGrowth),
2027 Flag: MachineInstr::FrameSetup);
2028 }
2029
2030 EmitSEHAfter(EmitSEHPushR14);
2031
2032 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::LEA64r), DestReg: FramePtr)
2033 .addUse(RegNo: X86::RSP)
2034 .addImm(Val: 1)
2035 .addUse(RegNo: X86::NoRegister)
2036 .addImm(Val: 8)
2037 .addUse(RegNo: X86::NoRegister)
2038 .setMIFlag(MachineInstr::FrameSetup);
2039
2040 // Switch to an FP-relative CFA before adjusting RSP below.
2041 if (NeedsDwarfCFI && !ArgBaseReg.isValid()) {
2042 unsigned DwarfFramePtr = TRI->getDwarfRegNum(Reg: MachineFramePtr, isEH: true);
2043 BuildCFI(MBB, MBBI, DL,
2044 CFIInst: MCCFIInstruction::cfiDefCfa(L: nullptr, Register: DwarfFramePtr,
2045 Offset: -2 * stackGrowth +
2046 (int)TailCallArgReserveSize),
2047 Flag: MachineInstr::FrameSetup);
2048 }
2049
2050 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SUB64ri32), DestReg: X86::RSP)
2051 .addUse(RegNo: X86::RSP)
2052 .addImm(Val: 8)
2053 .setMIFlag(MachineInstr::FrameSetup);
2054 }
2055
2056 if (!IsWin64Prologue && !IsFunclet) {
2057 // Update EBP with the new base value.
2058 if (!X86FI->hasSwiftAsyncContext())
2059 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
2060 MCID: TII.get(Opcode: Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr),
2061 DestReg: FramePtr)
2062 .addReg(RegNo: StackPtr)
2063 .setMIFlag(MachineInstr::FrameSetup);
2064
2065 if (NeedsDwarfCFI) {
2066 if (ArgBaseReg.isValid()) {
2067 SmallString<64> CfaExpr;
2068 CfaExpr.push_back(Elt: dwarf::DW_CFA_expression);
2069 uint8_t buffer[16];
2070 unsigned DwarfReg = TRI->getDwarfRegNum(Reg: MachineFramePtr, isEH: true);
2071 CfaExpr.append(in_start: buffer, in_end: buffer + encodeULEB128(Value: DwarfReg, p: buffer));
2072 CfaExpr.push_back(Elt: 2);
2073 CfaExpr.push_back(Elt: (uint8_t)(dwarf::DW_OP_breg0 + DwarfReg));
2074 CfaExpr.push_back(Elt: 0);
2075 // DW_CFA_expression: reg5 DW_OP_breg5 +0
2076 BuildCFI(MBB, MBBI, DL,
2077 CFIInst: MCCFIInstruction::createEscape(L: nullptr, Vals: CfaExpr.str()),
2078 Flag: MachineInstr::FrameSetup);
2079 } else if (!X86FI->hasSwiftAsyncContext()) {
2080 // Mark effective beginning of when frame pointer becomes valid.
2081 // Define the current CFA to use the EBP/RBP register.
2082 unsigned DwarfFramePtr = TRI->getDwarfRegNum(Reg: MachineFramePtr, isEH: true);
2083 BuildCFI(
2084 MBB, MBBI, DL,
2085 CFIInst: MCCFIInstruction::createDefCfaRegister(L: nullptr, Register: DwarfFramePtr),
2086 Flag: MachineInstr::FrameSetup);
2087 }
2088 }
2089
2090 if (NeedsWinFPO) {
2091 // .cv_fpo_setframe $FramePtr
2092 // NeedsWinFPO is Win32 only, so we're never using Unwind v3, hence it
2093 // is always inserted afterwards.
2094 assert(!IsWin64UnwindV3);
2095 HasWinCFI = true;
2096 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_SetFrame))
2097 .addImm(Val: FramePtr)
2098 .addImm(Val: 0)
2099 .setMIFlag(MachineInstr::FrameSetup);
2100 }
2101 }
2102 }
2103 } else {
2104 assert(!IsFunclet && "funclets without FPs not yet implemented");
2105 NumBytes =
2106 StackSize - (X86FI->getCalleeSavedFrameSize() + TailCallArgReserveSize);
2107 }
2108
2109 // Update the offset adjustment, which is mainly used by codeview to translate
2110 // from ESP to VFRAME relative local variable offsets.
2111 if (!IsFunclet) {
2112 if (HasFP && TRI->hasStackRealignment(MF))
2113 MFI.setOffsetAdjustment(-NumBytes);
2114 else
2115 MFI.setOffsetAdjustment(-StackSize);
2116 }
2117
2118 // For EH funclets, only allocate enough space for outgoing calls. Save the
2119 // NumBytes value that we would've used for the parent frame.
2120 unsigned ParentFrameNumBytes = NumBytes;
2121 if (IsFunclet)
2122 NumBytes = getWinEHFuncletFrameSize(MF);
2123
2124 // Skip the callee-saved push instructions.
2125 bool PushedRegs = false;
2126 int StackOffset = 2 * stackGrowth;
2127 MachineBasicBlock::const_iterator LastCSPush = MBBI;
2128 auto IsCSPush = [&](const MachineBasicBlock::iterator &MBBI) {
2129 if (MBBI == MBB.end() || !MBBI->getFlag(Flag: MachineInstr::FrameSetup))
2130 return false;
2131 unsigned Opc = MBBI->getOpcode();
2132 return Opc == X86::PUSH32r || Opc == X86::PUSH64r || Opc == X86::PUSHP64r ||
2133 Opc == X86::PUSH2 || Opc == X86::PUSH2P;
2134 };
2135
2136 while (IsCSPush(MBBI)) {
2137 PushedRegs = true;
2138 Register Reg = MBBI->getOperand(i: 0).getReg();
2139 LastCSPush = MBBI;
2140 unsigned Opc = LastCSPush->getOpcode();
2141 bool IsPush2 = Opc == X86::PUSH2 || Opc == X86::PUSH2P;
2142
2143 // V3: emit SEH pseudo before the real instruction.
2144 EmitSEHBefore([&]() {
2145 if (IsPush2) {
2146 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_Push2Regs))
2147 .addImm(Val: Reg)
2148 .addImm(Val: LastCSPush->getOperand(i: 1).getReg())
2149 .setMIFlag(MachineInstr::FrameSetup);
2150 } else {
2151 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_PushReg))
2152 .addImm(Val: Reg)
2153 .setMIFlag(MachineInstr::FrameSetup);
2154 }
2155 });
2156 ++MBBI;
2157
2158 if (!HasFP && NeedsDwarfCFI) {
2159 // Mark callee-saved push instruction.
2160 // Define the current CFA rule to use the provided offset.
2161 assert(StackSize);
2162 // Compared to push, push2 introduces more stack offset (one more
2163 // register).
2164 if (IsPush2)
2165 StackOffset += stackGrowth;
2166 BuildCFI(MBB, MBBI, DL,
2167 CFIInst: MCCFIInstruction::cfiDefCfaOffset(L: nullptr, Offset: -StackOffset),
2168 Flag: MachineInstr::FrameSetup);
2169 StackOffset += stackGrowth;
2170 }
2171
2172 // V1/V2: emit SEH pseudo after the real instruction.
2173 EmitSEHAfter([&]() {
2174 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_PushReg))
2175 .addImm(Val: Reg)
2176 .setMIFlag(MachineInstr::FrameSetup);
2177 if (IsPush2)
2178 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_PushReg))
2179 .addImm(Val: LastCSPush->getOperand(i: 1).getReg())
2180 .setMIFlag(MachineInstr::FrameSetup);
2181 });
2182 }
2183
2184 // Realign stack after we pushed callee-saved registers (so that we'll be
2185 // able to calculate their offsets from the frame pointer).
2186 // Don't do this for Win64, it needs to realign the stack after the prologue.
2187 if (!IsWin64Prologue && !IsFunclet && TRI->hasStackRealignment(MF) &&
2188 !ArgBaseReg.isValid()) {
2189 assert(HasFP && "There should be a frame pointer if stack is realigned.");
2190 auto EmitSEHStackAlign = [&]() {
2191 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_StackAlign))
2192 .addImm(Val: MaxAlign)
2193 .setMIFlag(MachineInstr::FrameSetup);
2194 };
2195 EmitSEHBefore(EmitSEHStackAlign);
2196 BuildStackAlignAND(MBB, MBBI, DL, Reg: StackPtr, MaxAlign);
2197 EmitSEHAfter(EmitSEHStackAlign);
2198 }
2199
2200 // If there is an SUB32ri of ESP immediately before this instruction, merge
2201 // the two. This can be the case when tail call elimination is enabled and
2202 // the callee has more arguments than the caller.
2203 NumBytes = mergeSPUpdates(
2204 MBB, MBBI, CalcNewOffset: [NumBytes](int64_t Offset) { return NumBytes - Offset; },
2205 doMergeWithPrevious: true);
2206
2207 // Adjust stack pointer: ESP -= numbytes.
2208
2209 // Windows and cygwin/mingw require a prologue helper routine when allocating
2210 // more than 4K bytes on the stack. Windows uses __chkstk and cygwin/mingw
2211 // uses __alloca. __alloca and the 32-bit version of __chkstk will probe the
2212 // stack and adjust the stack pointer in one go. The 64-bit version of
2213 // __chkstk is only responsible for probing the stack. The 64-bit prologue is
2214 // responsible for adjusting the stack pointer. Touching the stack at 4K
2215 // increments is necessary to ensure that the guard pages used by the OS
2216 // virtual memory manager are allocated in correct sequence.
2217 uint64_t AlignedNumBytes = NumBytes;
2218 if (IsWin64Prologue && !IsFunclet && TRI->hasStackRealignment(MF))
2219 AlignedNumBytes = alignTo(Value: AlignedNumBytes, Align: MaxAlign);
2220
2221 auto EmitSEHStackAlloc = [&]() {
2222 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_StackAlloc))
2223 .addImm(Val: NumBytes)
2224 .setMIFlag(MachineInstr::FrameSetup);
2225 };
2226 if (NumBytes)
2227 EmitSEHBefore(EmitSEHStackAlloc);
2228
2229 if (AlignedNumBytes >= StackProbeSize && EmitStackProbeCall) {
2230 assert(!X86FI->getUsesRedZone() &&
2231 "The Red Zone is not accounted for in stack probes");
2232
2233 // Check whether EAX is livein for this block.
2234 bool isEAXAlive = isEAXLiveIn(MBB);
2235
2236 if (isEAXAlive) {
2237 if (Is64Bit) {
2238 // Save RAX
2239 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH64r))
2240 .addReg(RegNo: X86::RAX, Flags: RegState::Kill)
2241 .setMIFlag(MachineInstr::FrameSetup);
2242 } else {
2243 // Save EAX
2244 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH32r))
2245 .addReg(RegNo: X86::EAX, Flags: RegState::Kill)
2246 .setMIFlag(MachineInstr::FrameSetup);
2247 }
2248 }
2249
2250 if (Is64Bit) {
2251 // Handle the 64-bit Windows ABI case where we need to call __chkstk.
2252 // Function prologue is responsible for adjusting the stack pointer.
2253 int64_t Alloc = isEAXAlive ? NumBytes - 8 : NumBytes;
2254 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::getMOVriOpcode(Use64BitReg: Is64Bit, Imm: Alloc)),
2255 DestReg: X86::RAX)
2256 .addImm(Val: Alloc)
2257 .setMIFlag(MachineInstr::FrameSetup);
2258 } else {
2259 // Allocate NumBytes-4 bytes on stack in case of isEAXAlive.
2260 // We'll also use 4 already allocated bytes for EAX.
2261 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV32ri), DestReg: X86::EAX)
2262 .addImm(Val: isEAXAlive ? NumBytes - 4 : NumBytes)
2263 .setMIFlag(MachineInstr::FrameSetup);
2264 }
2265
2266 // Call __chkstk, __chkstk_ms, or __alloca.
2267 emitStackProbe(MF, MBB, MBBI, DL, InProlog: true);
2268
2269 if (isEAXAlive) {
2270 // Restore RAX/EAX
2271 MachineInstr *MI;
2272 if (Is64Bit)
2273 MI = addRegOffset(MIB: BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64rm), DestReg: X86::RAX),
2274 Reg: StackPtr, isKill: false, Offset: NumBytes - 8);
2275 else
2276 MI = addRegOffset(MIB: BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: X86::MOV32rm), DestReg: X86::EAX),
2277 Reg: StackPtr, isKill: false, Offset: NumBytes - 4);
2278 MI->setFlag(MachineInstr::FrameSetup);
2279 MBB.insert(I: MBBI, MI);
2280 }
2281 } else if (NumBytes) {
2282 emitSPUpdate(MBB, MBBI, DL, NumBytes: -(int64_t)NumBytes, /*InEpilogue=*/false);
2283 }
2284
2285 if (NumBytes)
2286 EmitSEHAfter(EmitSEHStackAlloc);
2287
2288 int SEHFrameOffset = 0;
2289 Register SPOrEstablisher;
2290 if (IsFunclet) {
2291 if (IsClrFunclet) {
2292 // The establisher parameter passed to a CLR funclet is actually a pointer
2293 // to the (mostly empty) frame of its nearest enclosing funclet; we have
2294 // to find the root function establisher frame by loading the PSPSym from
2295 // the intermediate frame.
2296 unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF);
2297 MachinePointerInfo NoInfo;
2298 MBB.addLiveIn(PhysReg: Establisher);
2299 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64rm), DestReg: Establisher),
2300 Reg: Establisher, isKill: false, Offset: PSPSlotOffset)
2301 .addMemOperand(MMO: MF.getMachineMemOperand(
2302 PtrInfo: NoInfo, F: MachineMemOperand::MOLoad, Size: SlotSize, BaseAlignment: Align(SlotSize)));
2303 ;
2304 // Save the root establisher back into the current funclet's (mostly
2305 // empty) frame, in case a sub-funclet or the GC needs it.
2306 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64mr)), Reg: StackPtr,
2307 isKill: false, Offset: PSPSlotOffset)
2308 .addReg(RegNo: Establisher)
2309 .addMemOperand(MMO: MF.getMachineMemOperand(
2310 PtrInfo: NoInfo,
2311 F: MachineMemOperand::MOStore | MachineMemOperand::MOVolatile,
2312 Size: SlotSize, BaseAlignment: Align(SlotSize)));
2313 }
2314 SPOrEstablisher = Establisher;
2315 } else {
2316 SPOrEstablisher = StackPtr;
2317 }
2318
2319 if (IsWin64Prologue && HasFP) {
2320 // Set RBP to a small fixed offset from RSP. In the funclet case, we base
2321 // this calculation on the incoming establisher, which holds the value of
2322 // RSP from the parent frame at the end of the prologue.
2323 SEHFrameOffset = calculateSetFPREG(SPAdjust: ParentFrameNumBytes);
2324
2325 // If this is not a funclet, emit the CFI describing our frame pointer.
2326 if (NeedsWinCFI && !IsFunclet) {
2327 assert(!NeedsWinFPO && "this setframe incompatible with FPO data");
2328 HasWinCFI = true;
2329 if (isAsynchronousEHPersonality(Pers: Personality) || MF.hasEHFunclets()) {
2330 if (TRI->hasBasePointer(MF))
2331 MF.getWinEHFuncInfo()->SEHSetFrameOffset =
2332 getWinEHParentFrameOffset(MF);
2333 else
2334 MF.getWinEHFuncInfo()->SEHSetFrameOffset = SEHFrameOffset;
2335 }
2336 }
2337
2338 auto EmitSEHSetFrame = [&]() {
2339 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_SetFrame))
2340 .addImm(Val: FramePtr)
2341 .addImm(Val: SEHFrameOffset)
2342 .setMIFlag(MachineInstr::FrameSetup);
2343 };
2344
2345 if (!IsFunclet)
2346 EmitSEHBefore(EmitSEHSetFrame);
2347
2348 if (SEHFrameOffset)
2349 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::LEA64r), DestReg: FramePtr),
2350 Reg: SPOrEstablisher, isKill: false, Offset: SEHFrameOffset);
2351 else
2352 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64rr), DestReg: FramePtr)
2353 .addReg(RegNo: SPOrEstablisher);
2354
2355 if (!IsFunclet)
2356 EmitSEHAfter(EmitSEHSetFrame);
2357 } else if (IsFunclet && STI.is32Bit()) {
2358 // Reset EBP / ESI to something good for funclets.
2359 MBBI = restoreWin32EHStackPointers(MBB, MBBI, DL);
2360 // If we're a catch funclet, we can be returned to via catchret. Save ESP
2361 // into the registration node so that the runtime will restore it for us.
2362 if (!MBB.isCleanupFuncletEntry()) {
2363 assert(Personality == EHPersonality::MSVC_CXX);
2364 Register FrameReg;
2365 int FI = MF.getWinEHFuncInfo()->EHRegNodeFrameIndex;
2366 int64_t EHRegOffset = getFrameIndexReference(MF, FI, FrameReg).getFixed();
2367 // ESP is the first field, so no extra displacement is needed.
2368 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV32mr)), Reg: FrameReg,
2369 isKill: false, Offset: EHRegOffset)
2370 .addReg(RegNo: X86::ESP);
2371 }
2372 }
2373
2374 while (MBBI != MBB.end() && MBBI->getFlag(Flag: MachineInstr::FrameSetup)) {
2375 const MachineInstr &FrameInstr = *MBBI;
2376
2377 if (NeedsWinCFI) {
2378 int FI;
2379 if (Register Reg = TII.isStoreToStackSlot(MI: FrameInstr, FrameIndex&: FI)) {
2380 if (X86::FR64RegClass.contains(Reg)) {
2381 int Offset;
2382 Register IgnoredFrameReg;
2383 if (IsWin64Prologue && IsFunclet)
2384 Offset = getWin64EHFrameIndexRef(MF, FI, SPReg&: IgnoredFrameReg);
2385 else
2386 Offset =
2387 getFrameIndexReference(MF, FI, FrameReg&: IgnoredFrameReg).getFixed() +
2388 SEHFrameOffset;
2389
2390 assert(!NeedsWinFPO && "SEH_SaveXMM incompatible with FPO data");
2391 auto EmitSEHSaveXMM = [&]() {
2392 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_SaveXMM))
2393 .addImm(Val: Reg)
2394 .addImm(Val: Offset)
2395 .setMIFlag(MachineInstr::FrameSetup);
2396 };
2397 EmitSEHBefore(EmitSEHSaveXMM);
2398 ++MBBI;
2399 EmitSEHAfter(EmitSEHSaveXMM);
2400 continue;
2401 }
2402 }
2403 }
2404 ++MBBI;
2405 }
2406
2407 if (NeedsWinCFI && HasWinCFI) {
2408 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_EndPrologue))
2409 .setMIFlag(MachineInstr::FrameSetup);
2410 }
2411
2412 if (FnHasClrFunclet && !IsFunclet) {
2413 // Save the so-called Initial-SP (i.e. the value of the stack pointer
2414 // immediately after the prolog) into the PSPSlot so that funclets
2415 // and the GC can recover it.
2416 unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF);
2417 auto PSPInfo = MachinePointerInfo::getFixedStack(
2418 MF, FI: MF.getWinEHFuncInfo()->PSPSymFrameIdx);
2419 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64mr)), Reg: StackPtr, isKill: false,
2420 Offset: PSPSlotOffset)
2421 .addReg(RegNo: StackPtr)
2422 .addMemOperand(MMO: MF.getMachineMemOperand(
2423 PtrInfo: PSPInfo, F: MachineMemOperand::MOStore | MachineMemOperand::MOVolatile,
2424 Size: SlotSize, BaseAlignment: Align(SlotSize)));
2425 }
2426
2427 // Realign stack after we spilled callee-saved registers (so that we'll be
2428 // able to calculate their offsets from the frame pointer).
2429 // Win64 requires aligning the stack after the prologue.
2430 if (IsWin64Prologue && TRI->hasStackRealignment(MF)) {
2431 assert(HasFP && "There should be a frame pointer if stack is realigned.");
2432 BuildStackAlignAND(MBB, MBBI, DL, Reg: SPOrEstablisher, MaxAlign);
2433 }
2434
2435 // We already dealt with stack realignment and funclets above.
2436 if (IsFunclet && STI.is32Bit())
2437 return;
2438
2439 // If we need a base pointer, set it up here. It's whatever the value
2440 // of the stack pointer is at this point. Any variable size objects
2441 // will be allocated after this, so we can still use the base pointer
2442 // to reference locals.
2443 if (TRI->hasBasePointer(MF)) {
2444 // Update the base pointer with the current stack pointer.
2445 unsigned Opc = Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr;
2446 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: BasePtr)
2447 .addReg(RegNo: SPOrEstablisher)
2448 .setMIFlag(MachineInstr::FrameSetup);
2449 if (X86FI->getRestoreBasePointer()) {
2450 // Stash value of base pointer. Saving RSP instead of EBP shortens
2451 // dependence chain. Used by SjLj EH.
2452 unsigned Opm = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr;
2453 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opm)), Reg: FramePtr, isKill: true,
2454 Offset: X86FI->getRestoreBasePointerOffset())
2455 .addReg(RegNo: SPOrEstablisher)
2456 .setMIFlag(MachineInstr::FrameSetup);
2457 }
2458
2459 if (X86FI->getHasSEHFramePtrSave() && !IsFunclet) {
2460 // Stash the value of the frame pointer relative to the base pointer for
2461 // Win32 EH. This supports Win32 EH, which does the inverse of the above:
2462 // it recovers the frame pointer from the base pointer rather than the
2463 // other way around.
2464 unsigned Opm = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr;
2465 Register UsedReg;
2466 int Offset =
2467 getFrameIndexReference(MF, FI: X86FI->getSEHFramePtrSaveIndex(), FrameReg&: UsedReg)
2468 .getFixed();
2469 assert(UsedReg == BasePtr);
2470 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opm)), Reg: UsedReg, isKill: true, Offset)
2471 .addReg(RegNo: FramePtr)
2472 .setMIFlag(MachineInstr::FrameSetup);
2473 }
2474 }
2475 if (ArgBaseReg.isValid()) {
2476 // Save argument base pointer.
2477 auto *MI = X86FI->getStackPtrSaveMI();
2478 int FI = MI->getOperand(i: 1).getIndex();
2479 unsigned MOVmr = Is64Bit ? X86::MOV64mr : X86::MOV32mr;
2480 // movl %basereg, offset(%ebp)
2481 addFrameReference(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: MOVmr)), FI)
2482 .addReg(RegNo: ArgBaseReg)
2483 .setMIFlag(MachineInstr::FrameSetup);
2484 }
2485
2486 if (((!HasFP && NumBytes) || PushedRegs) && NeedsDwarfCFI) {
2487 // Mark end of stack pointer adjustment.
2488 if (!HasFP && NumBytes) {
2489 // Define the current CFA rule to use the provided offset.
2490 assert(StackSize);
2491 BuildCFI(
2492 MBB, MBBI, DL,
2493 CFIInst: MCCFIInstruction::cfiDefCfaOffset(L: nullptr, Offset: StackSize - stackGrowth),
2494 Flag: MachineInstr::FrameSetup);
2495 }
2496
2497 // Emit DWARF info specifying the offsets of the callee-saved registers.
2498 emitCalleeSavedFrameMoves(MBB, MBBI, DL, IsPrologue: true);
2499 }
2500
2501 // X86 Interrupt handling function cannot assume anything about the direction
2502 // flag (DF in EFLAGS register). Clear this flag by creating "cld" instruction
2503 // in each prologue of interrupt handler function.
2504 //
2505 // Create "cld" instruction only in these cases:
2506 // 1. The interrupt handling function uses any of the "rep" instructions.
2507 // 2. Interrupt handling function calls another function.
2508 // 3. If there are any inline asm blocks, as we do not know what they do
2509 //
2510 // TODO: We should also emit cld if we detect the use of std, but as of now,
2511 // the compiler does not even emit that instruction or even define it, so in
2512 // practice, this would only happen with inline asm, which we cover anyway.
2513 if (Fn.getCallingConv() == CallingConv::X86_INTR) {
2514 bool NeedsCLD = false;
2515
2516 for (const MachineBasicBlock &B : MF) {
2517 for (const MachineInstr &MI : B) {
2518 if (MI.isCall()) {
2519 NeedsCLD = true;
2520 break;
2521 }
2522
2523 if (isOpcodeRep(Opcode: MI.getOpcode())) {
2524 NeedsCLD = true;
2525 break;
2526 }
2527
2528 if (MI.isInlineAsm()) {
2529 // TODO: Parse asm for rep instructions or call sites?
2530 // For now, let's play it safe and emit a cld instruction
2531 // just in case.
2532 NeedsCLD = true;
2533 break;
2534 }
2535 }
2536 }
2537
2538 if (NeedsCLD) {
2539 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::CLD))
2540 .setMIFlag(MachineInstr::FrameSetup);
2541 }
2542 }
2543
2544 // At this point we know if the function has WinCFI or not.
2545 MF.setHasWinCFI(HasWinCFI);
2546}
2547
2548bool X86FrameLowering::canUseLEAForSPInEpilogue(
2549 const MachineFunction &MF) const {
2550 // We can't use LEA instructions for adjusting the stack pointer if we don't
2551 // have a frame pointer in the Win64 ABI. Only ADD instructions may be used
2552 // to deallocate the stack.
2553 // This means that we can use LEA for SP in two situations:
2554 // 1. We *aren't* using the Win64 ABI which means we are free to use LEA.
2555 // 2. We *have* a frame pointer which means we are permitted to use LEA.
2556 return !MF.usesWindowsUnwindInfo() || hasFP(MF);
2557}
2558
2559static bool isFuncletReturnInstr(MachineInstr &MI) {
2560 switch (MI.getOpcode()) {
2561 case X86::CATCHRET:
2562 case X86::CLEANUPRET:
2563 return true;
2564 default:
2565 return false;
2566 }
2567 llvm_unreachable("impossible");
2568}
2569
2570// CLR funclets use a special "Previous Stack Pointer Symbol" slot on the
2571// stack. It holds a pointer to the bottom of the root function frame. The
2572// establisher frame pointer passed to a nested funclet may point to the
2573// (mostly empty) frame of its parent funclet, but it will need to find
2574// the frame of the root function to access locals. To facilitate this,
2575// every funclet copies the pointer to the bottom of the root function
2576// frame into a PSPSym slot in its own (mostly empty) stack frame. Using the
2577// same offset for the PSPSym in the root function frame that's used in the
2578// funclets' frames allows each funclet to dynamically accept any ancestor
2579// frame as its establisher argument (the runtime doesn't guarantee the
2580// immediate parent for some reason lost to history), and also allows the GC,
2581// which uses the PSPSym for some bookkeeping, to find it in any funclet's
2582// frame with only a single offset reported for the entire method.
2583unsigned
2584X86FrameLowering::getPSPSlotOffsetFromSP(const MachineFunction &MF) const {
2585 const WinEHFuncInfo &Info = *MF.getWinEHFuncInfo();
2586 Register SPReg;
2587 int Offset = getFrameIndexReferencePreferSP(MF, FI: Info.PSPSymFrameIdx, FrameReg&: SPReg,
2588 /*IgnoreSPUpdates*/ true)
2589 .getFixed();
2590 assert(Offset >= 0 && SPReg == TRI->getStackRegister());
2591 return static_cast<unsigned>(Offset);
2592}
2593
2594unsigned
2595X86FrameLowering::getWinEHFuncletFrameSize(const MachineFunction &MF) const {
2596 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
2597 // This is the size of the pushed CSRs.
2598 unsigned CSSize = X86FI->getCalleeSavedFrameSize();
2599 // This is the size of callee saved XMMs.
2600 const auto &WinEHXMMSlotInfo = X86FI->getWinEHXMMSlotInfo();
2601 unsigned XMMSize =
2602 WinEHXMMSlotInfo.size() * TRI->getSpillSize(RC: X86::VR128RegClass);
2603 // This is the amount of stack a funclet needs to allocate.
2604 unsigned UsedSize;
2605 EHPersonality Personality =
2606 classifyEHPersonality(Pers: MF.getFunction().getPersonalityFn());
2607 if (Personality == EHPersonality::CoreCLR) {
2608 // CLR funclets need to hold enough space to include the PSPSym, at the
2609 // same offset from the stack pointer (immediately after the prolog) as it
2610 // resides at in the main function.
2611 UsedSize = getPSPSlotOffsetFromSP(MF) + SlotSize;
2612 } else {
2613 // Other funclets just need enough stack for outgoing call arguments.
2614 UsedSize = MF.getFrameInfo().getMaxCallFrameSize();
2615 }
2616 // RBP is not included in the callee saved register block. After pushing RBP,
2617 // everything is 16 byte aligned. Everything we allocate before an outgoing
2618 // call must also be 16 byte aligned.
2619 unsigned FrameSizeMinusRBP = alignTo(Size: CSSize + UsedSize, A: getStackAlign());
2620 // Subtract out the size of the callee saved registers. This is how much stack
2621 // each funclet will allocate.
2622 return FrameSizeMinusRBP + XMMSize - CSSize;
2623}
2624
2625static bool isTailCallOpcode(unsigned Opc) {
2626 return Opc == X86::TCRETURNri || Opc == X86::TCRETURN_WIN64ri ||
2627 Opc == X86::TCRETURN_HIPE32ri || Opc == X86::TCRETURNdi ||
2628 Opc == X86::TCRETURNmi || Opc == X86::TCRETURNri64 ||
2629 Opc == X86::TCRETURNri64_ImpCall || Opc == X86::TCRETURNdi64 ||
2630 Opc == X86::TCRETURNmi64 || Opc == X86::TCRETURN_WINmi64;
2631}
2632
2633void X86FrameLowering::emitEpilogue(MachineFunction &MF,
2634 MachineBasicBlock &MBB) const {
2635 const MachineFrameInfo &MFI = MF.getFrameInfo();
2636 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
2637 MachineBasicBlock::iterator Terminator = MBB.getFirstTerminator();
2638 MachineBasicBlock::iterator MBBI = Terminator;
2639 DebugLoc DL;
2640 if (MBBI != MBB.end())
2641 DL = MBBI->getDebugLoc();
2642 // standard x86_64 uses 64-bit frame/stack pointers, x32 - 32-bit.
2643 const bool Is64BitILP32 = STI.isTarget64BitILP32();
2644 Register FramePtr = TRI->getFrameRegister(MF);
2645 Register MachineFramePtr =
2646 Is64BitILP32 ? Register(getX86SubSuperRegister(Reg: FramePtr, Size: 64)) : FramePtr;
2647
2648 bool IsWin64Prologue = MF.usesWindowsUnwindInfo();
2649 bool NeedsWin64CFI =
2650 IsWin64Prologue && MF.getFunction().needsUnwindTableEntry();
2651 // For V3 unwind, epilog SEH pseudos are emitted inline before each
2652 // unwind-effecting instruction.
2653 bool IsWin64UnwindV3 =
2654 NeedsWin64CFI && MF.hasWinCFI() && requireWinX64UnwindV3(MF);
2655 bool IsFunclet = MBBI == MBB.end() ? false : isFuncletReturnInstr(MI&: *MBBI);
2656
2657 // Get the number of bytes to allocate from the FrameInfo.
2658 uint64_t StackSize = MFI.getStackSize();
2659 uint64_t MaxAlign = calculateMaxStackAlign(MF);
2660 unsigned CSSize = X86FI->getCalleeSavedFrameSize();
2661 unsigned TailCallArgReserveSize = -X86FI->getTCReturnAddrDelta();
2662 bool HasFP = hasFP(MF);
2663 uint64_t NumBytes = 0;
2664
2665 bool NeedsDwarfCFI = (!MF.getTarget().getTargetTriple().isOSDarwin() &&
2666 !MF.getTarget().getTargetTriple().isOSWindows() &&
2667 !MF.getTarget().getTargetTriple().isUEFI()) &&
2668 MF.needsFrameMoves();
2669
2670 Register ArgBaseReg;
2671 if (auto *MI = X86FI->getStackPtrSaveMI()) {
2672 unsigned Opc = X86::LEA32r;
2673 Register StackReg = X86::ESP;
2674 ArgBaseReg = MI->getOperand(i: 0).getReg();
2675 if (STI.is64Bit()) {
2676 Opc = X86::LEA64r;
2677 StackReg = X86::RSP;
2678 }
2679 // leal -4(%basereg), %esp
2680 // .cfi_def_cfa %esp, 4
2681 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: StackReg)
2682 .addUse(RegNo: ArgBaseReg)
2683 .addImm(Val: 1)
2684 .addUse(RegNo: X86::NoRegister)
2685 .addImm(Val: -(int64_t)SlotSize)
2686 .addUse(RegNo: X86::NoRegister)
2687 .setMIFlag(MachineInstr::FrameDestroy);
2688 if (NeedsDwarfCFI) {
2689 unsigned DwarfStackPtr = TRI->getDwarfRegNum(Reg: StackReg, isEH: true);
2690 BuildCFI(MBB, MBBI, DL,
2691 CFIInst: MCCFIInstruction::cfiDefCfa(L: nullptr, Register: DwarfStackPtr, Offset: SlotSize),
2692 Flag: MachineInstr::FrameDestroy);
2693 --MBBI;
2694 }
2695 --MBBI;
2696 }
2697
2698 if (IsFunclet) {
2699 assert(HasFP && "EH funclets without FP not yet implemented");
2700 NumBytes = getWinEHFuncletFrameSize(MF);
2701 } else if (HasFP) {
2702 // Calculate required stack adjustment.
2703 uint64_t FrameSize = StackSize - SlotSize;
2704 NumBytes = FrameSize - CSSize - TailCallArgReserveSize;
2705
2706 // Callee-saved registers were pushed on stack before the stack was
2707 // realigned.
2708 if (TRI->hasStackRealignment(MF) && !IsWin64Prologue)
2709 NumBytes = alignTo(Value: FrameSize, Align: MaxAlign);
2710 } else {
2711 NumBytes = StackSize - CSSize - TailCallArgReserveSize;
2712 }
2713 uint64_t SEHStackAllocAmt = NumBytes;
2714
2715 unsigned SEHFrameOffset = 0;
2716 if (IsWin64Prologue && HasFP)
2717 SEHFrameOffset = calculateSetFPREG(SPAdjust: SEHStackAllocAmt);
2718
2719 // AfterPop is the position to insert .cfi_restore.
2720 MachineBasicBlock::iterator AfterPop = MBBI;
2721 if (HasFP) {
2722 if (X86FI->hasSwiftAsyncContext()) {
2723 // Discard the context.
2724 int64_t Offset = mergeSPAdd(MBB, MBBI, AddOffset: 16, doMergeWithPrevious: true);
2725 emitSPUpdate(MBB, MBBI, DL, NumBytes: Offset, /*InEpilogue*/ true);
2726 }
2727 // Pop EBP.
2728 if (IsWin64UnwindV3)
2729 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_PushReg))
2730 .addImm(Val: FramePtr)
2731 .setMIFlag(MachineInstr::FrameDestroy);
2732 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
2733 MCID: TII.get(Opcode: getPOPOpcode(ST: MF.getSubtarget<X86Subtarget>())),
2734 DestReg: MachineFramePtr)
2735 .setMIFlag(MachineInstr::FrameDestroy);
2736
2737 // We need to reset FP to its untagged state on return. Bit 60 is currently
2738 // used to show the presence of an extended frame.
2739 if (X86FI->hasSwiftAsyncContext()) {
2740 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::BTR64ri8), DestReg: MachineFramePtr)
2741 .addUse(RegNo: MachineFramePtr)
2742 .addImm(Val: 60)
2743 .setMIFlag(MachineInstr::FrameDestroy);
2744 }
2745
2746 if (NeedsDwarfCFI) {
2747 if (!ArgBaseReg.isValid()) {
2748 unsigned DwarfStackPtr =
2749 TRI->getDwarfRegNum(Reg: Is64Bit ? X86::RSP : X86::ESP, isEH: true);
2750 BuildCFI(MBB, MBBI, DL,
2751 CFIInst: MCCFIInstruction::cfiDefCfa(L: nullptr, Register: DwarfStackPtr, Offset: SlotSize),
2752 Flag: MachineInstr::FrameDestroy);
2753 }
2754 if (!MBB.succ_empty() && !MBB.isReturnBlock()) {
2755 unsigned DwarfFramePtr = TRI->getDwarfRegNum(Reg: MachineFramePtr, isEH: true);
2756 BuildCFI(MBB, MBBI: AfterPop, DL,
2757 CFIInst: MCCFIInstruction::createRestore(L: nullptr, Register: DwarfFramePtr),
2758 Flag: MachineInstr::FrameDestroy);
2759 --MBBI;
2760 --AfterPop;
2761 }
2762 --MBBI;
2763 }
2764 }
2765
2766 MachineBasicBlock::iterator FirstCSPop = MBBI;
2767 // Skip the callee-saved pop instructions.
2768 while (MBBI != MBB.begin()) {
2769 MachineBasicBlock::iterator PI = std::prev(x: MBBI);
2770 unsigned Opc = PI->getOpcode();
2771
2772 if (Opc != X86::DBG_VALUE && !PI->isTerminator()) {
2773 if (!PI->getFlag(Flag: MachineInstr::FrameDestroy) ||
2774 (Opc != X86::POP32r && Opc != X86::POP64r && Opc != X86::BTR64ri8 &&
2775 Opc != X86::ADD64ri32 && Opc != X86::POPP64r && Opc != X86::POP2 &&
2776 Opc != X86::POP2P && Opc != X86::LEA64r && Opc != X86::SEH_PushReg &&
2777 Opc != X86::SEH_Push2Regs && Opc != X86::SEH_StackAlloc &&
2778 Opc != X86::ADD64ri32_NF))
2779 break;
2780 FirstCSPop = PI;
2781 }
2782
2783 --MBBI;
2784 }
2785 if (ArgBaseReg.isValid()) {
2786 // Restore argument base pointer.
2787 auto *MI = X86FI->getStackPtrSaveMI();
2788 int FI = MI->getOperand(i: 1).getIndex();
2789 unsigned MOVrm = Is64Bit ? X86::MOV64rm : X86::MOV32rm;
2790 // movl offset(%ebp), %basereg
2791 addFrameReference(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: MOVrm), DestReg: ArgBaseReg), FI)
2792 .setMIFlag(MachineInstr::FrameDestroy);
2793 }
2794 MBBI = FirstCSPop;
2795
2796 if (IsFunclet && Terminator->getOpcode() == X86::CATCHRET)
2797 emitCatchRetReturnValue(MBB, MBBI: FirstCSPop, CatchRet: &*Terminator);
2798
2799 if (MBBI != MBB.end())
2800 DL = MBBI->getDebugLoc();
2801 // If there is an ADD32ri or SUB32ri of ESP immediately before this
2802 // instruction, merge the two instructions.
2803 if (NumBytes || MFI.hasVarSizedObjects())
2804 NumBytes = mergeSPAdd(MBB, MBBI, AddOffset: NumBytes, doMergeWithPrevious: true);
2805
2806 if (IsWin64UnwindV3 && NeedsWin64CFI && MF.hasWinCFI()) {
2807 // Find the XMM restores that were tagged with FrameDestroy, now that we
2808 // know the offset we can emit the SEH pseudos for them.
2809 auto EpilogStart = MBBI;
2810 {
2811 auto ScanIt = MBBI;
2812 while (ScanIt != MBB.begin()) {
2813 auto PI = std::prev(x: ScanIt);
2814 int FI;
2815 if (PI->getFlag(Flag: MachineInstr::FrameDestroy) &&
2816 TII.isLoadFromStackSlot(MI: *PI, FrameIndex&: FI)) {
2817 Register Reg = PI->getOperand(i: 0).getReg();
2818 if (X86::FR64RegClass.contains(Reg)) {
2819 Register IgnoredFrameReg;
2820 int Offset =
2821 getFrameIndexReference(MF, FI, FrameReg&: IgnoredFrameReg).getFixed() +
2822 SEHFrameOffset;
2823 BuildMI(BB&: MBB, I: PI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_SaveXMM))
2824 .addImm(Val: Reg)
2825 .addImm(Val: Offset)
2826 .setMIFlag(MachineInstr::FrameDestroy);
2827 // std::prev(PI) is the SEH_SaveXMM we just inserted (before PI).
2828 // We start ScanIt from that point so that the next
2829 // std::prev(ScanIt) will examine the instruction before the pseudo,
2830 // i.e. the next potential XMM restore further up the block.
2831 EpilogStart = std::prev(x: PI);
2832 ScanIt = EpilogStart;
2833 continue;
2834 }
2835 }
2836 break;
2837 }
2838 }
2839
2840 // For V3, SEH_BeginEpilogue must be emitted before any epilog SEH pseudos.
2841 BuildMI(BB&: MBB, I: EpilogStart, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_BeginEpilogue))
2842 .setMIFlag(MachineInstr::FrameDestroy);
2843 }
2844
2845 // If dynamic alloca is used, then reset esp to point to the last callee-saved
2846 // slot before popping them off! Same applies for the case, when stack was
2847 // realigned. Don't do this if this was a funclet epilogue, since the funclets
2848 // will not do realignment or dynamic stack allocation.
2849 if (((TRI->hasStackRealignment(MF)) || MFI.hasVarSizedObjects()) &&
2850 !IsFunclet) {
2851 if (TRI->hasStackRealignment(MF))
2852 MBBI = FirstCSPop;
2853 uint64_t LEAAmount =
2854 IsWin64Prologue ? SEHStackAllocAmt - SEHFrameOffset : -CSSize;
2855
2856 if (X86FI->hasSwiftAsyncContext())
2857 LEAAmount -= 16;
2858
2859 // There are only two legal forms of epilogue:
2860 // - add SEHAllocationSize, %rsp
2861 // - lea SEHAllocationSize(%FramePtr), %rsp
2862 //
2863 // 'mov %FramePtr, %rsp' will not be recognized as an epilogue sequence.
2864 // However, we may use this sequence if we have a frame pointer because the
2865 // effects of the prologue can safely be undone.
2866 if (IsWin64UnwindV3) {
2867 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_SetFrame))
2868 .addImm(Val: FramePtr)
2869 .addImm(Val: SEHFrameOffset)
2870 .setMIFlag(MachineInstr::FrameDestroy);
2871 if (SEHStackAllocAmt)
2872 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_StackAlloc))
2873 .addImm(Val: SEHStackAllocAmt)
2874 .setMIFlag(MachineInstr::FrameDestroy);
2875 }
2876 if (LEAAmount != 0) {
2877 unsigned Opc = getLEArOpcode(IsLP64: Uses64BitFramePtr);
2878 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: StackPtr), Reg: FramePtr,
2879 isKill: false, Offset: LEAAmount);
2880 --MBBI;
2881 } else {
2882 unsigned Opc = (Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr);
2883 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: StackPtr).addReg(RegNo: FramePtr);
2884 --MBBI;
2885 }
2886 } else if (NumBytes) {
2887 // Adjust stack pointer back: ESP += numbytes.
2888 if (IsWin64UnwindV3)
2889 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_StackAlloc))
2890 .addImm(Val: NumBytes)
2891 .setMIFlag(MachineInstr::FrameDestroy);
2892 emitSPUpdate(MBB, MBBI, DL, NumBytes, /*InEpilogue=*/true);
2893 if (!HasFP && NeedsDwarfCFI) {
2894 // Define the current CFA rule to use the provided offset.
2895 BuildCFI(MBB, MBBI, DL,
2896 CFIInst: MCCFIInstruction::cfiDefCfaOffset(
2897 L: nullptr, Offset: CSSize + TailCallArgReserveSize + SlotSize),
2898 Flag: MachineInstr::FrameDestroy);
2899 }
2900 --MBBI;
2901 }
2902
2903 // For V1/V2, emit SEH_BeginEpilogue after stack restore code.
2904 if (!IsWin64UnwindV3 && NeedsWin64CFI && MF.hasWinCFI())
2905 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_BeginEpilogue))
2906 .setMIFlag(MachineInstr::FrameDestroy);
2907
2908 if (!HasFP && NeedsDwarfCFI) {
2909 MBBI = FirstCSPop;
2910 int64_t Offset = -(int64_t)CSSize - SlotSize;
2911 // Mark callee-saved pop instruction.
2912 // Define the current CFA rule to use the provided offset.
2913 while (MBBI != MBB.end()) {
2914 MachineBasicBlock::iterator PI = MBBI;
2915 unsigned Opc = PI->getOpcode();
2916 ++MBBI;
2917 if (Opc == X86::POP32r || Opc == X86::POP64r || Opc == X86::POPP64r ||
2918 Opc == X86::POP2 || Opc == X86::POP2P) {
2919 Offset += SlotSize;
2920 // Compared to pop, pop2 introduces more stack offset (one more
2921 // register).
2922 if (Opc == X86::POP2 || Opc == X86::POP2P)
2923 Offset += SlotSize;
2924 BuildCFI(MBB, MBBI, DL,
2925 CFIInst: MCCFIInstruction::cfiDefCfaOffset(L: nullptr, Offset: -Offset),
2926 Flag: MachineInstr::FrameDestroy);
2927 }
2928 }
2929 }
2930
2931 // Emit DWARF info specifying the restores of the callee-saved registers.
2932 // For epilogue with return inside or being other block without successor,
2933 // no need to generate .cfi_restore for callee-saved registers.
2934 if (NeedsDwarfCFI && !MBB.succ_empty())
2935 emitCalleeSavedFrameMoves(MBB, MBBI: AfterPop, DL, IsPrologue: false);
2936
2937 if (Terminator == MBB.end() || !isTailCallOpcode(Opc: Terminator->getOpcode())) {
2938 // Add the return addr area delta back since we are not tail calling.
2939 int64_t Delta = X86FI->getTCReturnAddrDelta();
2940 assert(Delta <= 0 && "TCDelta should never be positive");
2941 if (Delta) {
2942 // Check for possible merge with preceding ADD instruction.
2943 int64_t Offset = mergeSPAdd(MBB, MBBI&: Terminator, AddOffset: -Delta, doMergeWithPrevious: true);
2944 emitSPUpdate(MBB, MBBI&: Terminator, DL, NumBytes: Offset, /*InEpilogue=*/true);
2945 }
2946 }
2947
2948 // Emit tilerelease for AMX kernel.
2949 if (X86FI->getAMXProgModel() == AMXProgModelEnum::ManagedRA)
2950 BuildMI(BB&: MBB, I: Terminator, MIMD: DL, MCID: TII.get(Opcode: X86::TILERELEASE));
2951
2952 if (NeedsWin64CFI && MF.hasWinCFI())
2953 BuildMI(BB&: MBB, I: Terminator, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_EndEpilogue))
2954 .setMIFlag(MachineInstr::FrameDestroy);
2955}
2956
2957StackOffset X86FrameLowering::getFrameIndexReference(const MachineFunction &MF,
2958 int FI,
2959 Register &FrameReg) const {
2960 const MachineFrameInfo &MFI = MF.getFrameInfo();
2961
2962 bool IsFixed = MFI.isFixedObjectIndex(ObjectIdx: FI);
2963 // We can't calculate offset from frame pointer if the stack is realigned,
2964 // so enforce usage of stack/base pointer. The base pointer is used when we
2965 // have dynamic allocas in addition to dynamic realignment.
2966 if (TRI->hasBasePointer(MF))
2967 FrameReg = IsFixed ? TRI->getFramePtr() : TRI->getBaseRegister();
2968 else if (TRI->hasStackRealignment(MF))
2969 FrameReg = IsFixed ? TRI->getFramePtr() : TRI->getStackRegister();
2970 else
2971 FrameReg = TRI->getFrameRegister(MF);
2972
2973 // Offset will hold the offset from the stack pointer at function entry to the
2974 // object.
2975 // We need to factor in additional offsets applied during the prologue to the
2976 // frame, base, and stack pointer depending on which is used.
2977 int64_t Offset = MFI.getObjectOffset(ObjectIdx: FI) - getOffsetOfLocalArea();
2978 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
2979 unsigned CSSize = X86FI->getCalleeSavedFrameSize();
2980 uint64_t StackSize = MFI.getStackSize();
2981 bool IsWin64Prologue = MF.usesWindowsUnwindInfo();
2982 int64_t FPDelta = 0;
2983
2984 // In an x86 interrupt, remove the offset we added to account for the return
2985 // address from any stack object allocated in the caller's frame. Interrupts
2986 // do not have a standard return address. Fixed objects in the current frame,
2987 // such as SSE register spills, should not get this treatment.
2988 if (MF.getFunction().getCallingConv() == CallingConv::X86_INTR &&
2989 Offset >= 0) {
2990 Offset += getOffsetOfLocalArea();
2991 }
2992
2993 if (IsWin64Prologue) {
2994 assert(!MFI.hasCalls() || (StackSize % 16) == 8);
2995
2996 // Calculate required stack adjustment.
2997 uint64_t FrameSize = StackSize - SlotSize;
2998 // If required, include space for extra hidden slot for stashing base
2999 // pointer.
3000 if (X86FI->getRestoreBasePointer())
3001 FrameSize += SlotSize;
3002 uint64_t NumBytes = FrameSize - CSSize;
3003
3004 uint64_t SEHFrameOffset = calculateSetFPREG(SPAdjust: NumBytes);
3005 if (FI && FI == X86FI->getFAIndex())
3006 return StackOffset::getFixed(Fixed: -SEHFrameOffset);
3007
3008 // FPDelta is the offset from the "traditional" FP location of the old base
3009 // pointer followed by return address and the location required by the
3010 // restricted Win64 prologue.
3011 // Add FPDelta to all offsets below that go through the frame pointer.
3012 FPDelta = FrameSize - SEHFrameOffset;
3013 assert((!MFI.hasCalls() || (FPDelta % 16) == 0) &&
3014 "FPDelta isn't aligned per the Win64 ABI!");
3015 }
3016
3017 if (FrameReg == TRI->getFramePtr()) {
3018 // Skip saved EBP/RBP
3019 Offset += SlotSize;
3020
3021 // Account for restricted Windows prologue.
3022 Offset += FPDelta;
3023
3024 // Skip the RETADDR move area
3025 int TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta();
3026 if (TailCallReturnAddrDelta < 0)
3027 Offset -= TailCallReturnAddrDelta;
3028
3029 return StackOffset::getFixed(Fixed: Offset);
3030 }
3031
3032 // FrameReg is either the stack pointer or a base pointer. But the base is
3033 // located at the end of the statically known StackSize so the distinction
3034 // doesn't really matter.
3035 if (TRI->hasStackRealignment(MF) || TRI->hasBasePointer(MF))
3036 assert(isAligned(MFI.getObjectAlign(FI), -(Offset + StackSize)));
3037 return StackOffset::getFixed(Fixed: Offset + StackSize);
3038}
3039
3040int X86FrameLowering::getWin64EHFrameIndexRef(const MachineFunction &MF, int FI,
3041 Register &FrameReg) const {
3042 const MachineFrameInfo &MFI = MF.getFrameInfo();
3043 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
3044 const auto &WinEHXMMSlotInfo = X86FI->getWinEHXMMSlotInfo();
3045 const auto it = WinEHXMMSlotInfo.find(Val: FI);
3046
3047 if (it == WinEHXMMSlotInfo.end())
3048 return getFrameIndexReference(MF, FI, FrameReg).getFixed();
3049
3050 FrameReg = TRI->getStackRegister();
3051 return alignDown(Value: MFI.getMaxCallFrameSize(), Align: getStackAlign().value()) +
3052 it->second;
3053}
3054
3055StackOffset
3056X86FrameLowering::getFrameIndexReferenceSP(const MachineFunction &MF, int FI,
3057 Register &FrameReg,
3058 int Adjustment) const {
3059 const MachineFrameInfo &MFI = MF.getFrameInfo();
3060 FrameReg = TRI->getStackRegister();
3061 return StackOffset::getFixed(Fixed: MFI.getObjectOffset(ObjectIdx: FI) -
3062 getOffsetOfLocalArea() + Adjustment);
3063}
3064
3065StackOffset
3066X86FrameLowering::getFrameIndexReferencePreferSP(const MachineFunction &MF,
3067 int FI, Register &FrameReg,
3068 bool IgnoreSPUpdates) const {
3069
3070 const MachineFrameInfo &MFI = MF.getFrameInfo();
3071 // Does not include any dynamic realign.
3072 const uint64_t StackSize = MFI.getStackSize();
3073 // LLVM arranges the stack as follows:
3074 // ...
3075 // ARG2
3076 // ARG1
3077 // RETADDR
3078 // PUSH RBP <-- RBP points here
3079 // PUSH CSRs
3080 // ~~~~~~~ <-- possible stack realignment (non-win64)
3081 // ...
3082 // STACK OBJECTS
3083 // ... <-- RSP after prologue points here
3084 // ~~~~~~~ <-- possible stack realignment (win64)
3085 //
3086 // if (hasVarSizedObjects()):
3087 // ... <-- "base pointer" (ESI/RBX) points here
3088 // DYNAMIC ALLOCAS
3089 // ... <-- RSP points here
3090 //
3091 // Case 1: In the simple case of no stack realignment and no dynamic
3092 // allocas, both "fixed" stack objects (arguments and CSRs) are addressable
3093 // with fixed offsets from RSP.
3094 //
3095 // Case 2: In the case of stack realignment with no dynamic allocas, fixed
3096 // stack objects are addressed with RBP and regular stack objects with RSP.
3097 //
3098 // Case 3: In the case of dynamic allocas and stack realignment, RSP is used
3099 // to address stack arguments for outgoing calls and nothing else. The "base
3100 // pointer" points to local variables, and RBP points to fixed objects.
3101 //
3102 // In cases 2 and 3, we can only answer for non-fixed stack objects, and the
3103 // answer we give is relative to the SP after the prologue, and not the
3104 // SP in the middle of the function.
3105
3106 if (MFI.isFixedObjectIndex(ObjectIdx: FI) && TRI->hasStackRealignment(MF) &&
3107 !STI.isTargetWin64())
3108 return getFrameIndexReference(MF, FI, FrameReg);
3109
3110 // If !hasReservedCallFrame the function might have SP adjustement in the
3111 // body. So, even though the offset is statically known, it depends on where
3112 // we are in the function.
3113 if (!IgnoreSPUpdates && !hasReservedCallFrame(MF))
3114 return getFrameIndexReference(MF, FI, FrameReg);
3115
3116 // We don't handle tail calls, and shouldn't be seeing them either.
3117 assert(MF.getInfo<X86MachineFunctionInfo>()->getTCReturnAddrDelta() >= 0 &&
3118 "we don't handle this case!");
3119
3120 // This is how the math works out:
3121 //
3122 // %rsp grows (i.e. gets lower) left to right. Each box below is
3123 // one word (eight bytes). Obj0 is the stack slot we're trying to
3124 // get to.
3125 //
3126 // ----------------------------------
3127 // | BP | Obj0 | Obj1 | ... | ObjN |
3128 // ----------------------------------
3129 // ^ ^ ^ ^
3130 // A B C E
3131 //
3132 // A is the incoming stack pointer.
3133 // (B - A) is the local area offset (-8 for x86-64) [1]
3134 // (C - A) is the Offset returned by MFI.getObjectOffset for Obj0 [2]
3135 //
3136 // |(E - B)| is the StackSize (absolute value, positive). For a
3137 // stack that grown down, this works out to be (B - E). [3]
3138 //
3139 // E is also the value of %rsp after stack has been set up, and we
3140 // want (C - E) -- the value we can add to %rsp to get to Obj0. Now
3141 // (C - E) == (C - A) - (B - A) + (B - E)
3142 // { Using [1], [2] and [3] above }
3143 // == getObjectOffset - LocalAreaOffset + StackSize
3144
3145 return getFrameIndexReferenceSP(MF, FI, FrameReg, Adjustment: StackSize);
3146}
3147
3148bool X86FrameLowering::assignCalleeSavedSpillSlots(
3149 MachineFunction &MF, const TargetRegisterInfo *TRI,
3150 std::vector<CalleeSavedInfo> &CSI) const {
3151 MachineFrameInfo &MFI = MF.getFrameInfo();
3152 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
3153
3154 unsigned CalleeSavedFrameSize = 0;
3155 unsigned XMMCalleeSavedFrameSize = 0;
3156 auto &WinEHXMMSlotInfo = X86FI->getWinEHXMMSlotInfo();
3157 int SpillSlotOffset = getOffsetOfLocalArea() + X86FI->getTCReturnAddrDelta();
3158
3159 int64_t TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta();
3160
3161 if (TailCallReturnAddrDelta < 0) {
3162 // create RETURNADDR area
3163 // arg
3164 // arg
3165 // RETADDR
3166 // { ...
3167 // RETADDR area
3168 // ...
3169 // }
3170 // [EBP]
3171 MFI.CreateFixedObject(Size: -TailCallReturnAddrDelta,
3172 SPOffset: TailCallReturnAddrDelta - SlotSize, IsImmutable: true);
3173 }
3174
3175 // Spill the BasePtr if it's used.
3176 if (this->TRI->hasBasePointer(MF)) {
3177 // Allocate a spill slot for EBP if we have a base pointer and EH funclets.
3178 if (MF.hasEHFunclets()) {
3179 int FI = MFI.CreateSpillStackObject(Size: SlotSize, Alignment: Align(SlotSize));
3180 X86FI->setHasSEHFramePtrSave(true);
3181 X86FI->setSEHFramePtrSaveIndex(FI);
3182 }
3183 }
3184
3185 bool IsFPRemovedFromCSI = false;
3186 if (hasFP(MF)) {
3187 // emitPrologue always spills frame register the first thing.
3188 SpillSlotOffset -= SlotSize;
3189 MFI.CreateFixedSpillStackObject(Size: SlotSize, SPOffset: SpillSlotOffset);
3190
3191 // The async context lives directly before the frame pointer, and we
3192 // allocate a second slot to preserve stack alignment.
3193 if (X86FI->hasSwiftAsyncContext()) {
3194 SpillSlotOffset -= SlotSize;
3195 MFI.CreateFixedSpillStackObject(Size: SlotSize, SPOffset: SpillSlotOffset);
3196 SpillSlotOffset -= SlotSize;
3197 }
3198
3199 // Since emitPrologue and emitEpilogue will handle spilling and restoring of
3200 // the frame register, we can delete it from CSI list and not have to worry
3201 // about avoiding it later.
3202 Register FPReg = TRI->getFrameRegister(MF);
3203 for (unsigned i = 0; i < CSI.size(); ++i) {
3204 if (TRI->regsOverlap(RegA: CSI[i].getReg(), RegB: FPReg)) {
3205 CSI.erase(position: CSI.begin() + i);
3206 IsFPRemovedFromCSI = true;
3207 break;
3208 }
3209 }
3210 }
3211
3212 // Strategy:
3213 // 1. Use push2 when
3214 // a) number of CSR > 1 if no need padding
3215 // b) number of CSR > 2 if need padding
3216 // c) stack alignment >= 16 bytes
3217 // 2. When the number of CSR push is odd
3218 // a. Start to use push2 from the 1st push if stack is 16B aligned.
3219 // b. Start to use push2 from the 2nd push if stack is not 16B aligned.
3220 // 3. When the number of CSR push is even, start to use push2 from the 1st
3221 // push and make the stack 16B aligned before the push
3222 unsigned NumRegsForPush2 = 0;
3223 if (STI.hasPush2Pop2() && getStackAlignment() >= 16) {
3224 unsigned NumCSGPR = llvm::count_if(Range&: CSI, P: [](const CalleeSavedInfo &I) {
3225 return X86::GR64RegClass.contains(Reg: I.getReg());
3226 });
3227 bool UsePush2Pop2 = !IsFPRemovedFromCSI ? NumCSGPR > 2 : NumCSGPR > 1;
3228 NumRegsForPush2 =
3229 UsePush2Pop2
3230 ? alignDown(Value: IsFPRemovedFromCSI ? NumCSGPR : NumCSGPR - 1, Align: 2)
3231 : 0;
3232 }
3233
3234 // Assign slots for GPRs. It increases frame size.
3235 for (CalleeSavedInfo &I : llvm::reverse(C&: CSI)) {
3236 MCRegister Reg = I.getReg();
3237
3238 if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg))
3239 continue;
3240
3241 // A CSR is a candidate for push2/pop2 when it's slot offset is 16B aligned
3242 // or only an odd number of registers in the candidates.
3243 if (X86FI->getNumCandidatesForPush2Pop2() < NumRegsForPush2 &&
3244 (SpillSlotOffset % 16 == 0 ||
3245 X86FI->getNumCandidatesForPush2Pop2() % 2))
3246 X86FI->addCandidateForPush2Pop2(Reg);
3247
3248 SpillSlotOffset -= SlotSize;
3249 CalleeSavedFrameSize += SlotSize;
3250
3251 int SlotIndex = MFI.CreateFixedSpillStackObject(Size: SlotSize, SPOffset: SpillSlotOffset);
3252 I.setFrameIdx(SlotIndex);
3253 }
3254
3255 // Adjust the offset of spill slot as we know the accurate callee saved frame
3256 // size.
3257 if (X86FI->getRestoreBasePointer()) {
3258 SpillSlotOffset -= SlotSize;
3259 CalleeSavedFrameSize += SlotSize;
3260
3261 MFI.CreateFixedSpillStackObject(Size: SlotSize, SPOffset: SpillSlotOffset);
3262 // TODO: saving the slot index is better?
3263 X86FI->setRestoreBasePointer(CalleeSavedFrameSize);
3264 }
3265 assert(X86FI->getNumCandidatesForPush2Pop2() % 2 == 0 &&
3266 "Expect even candidates for push2/pop2");
3267 if (X86FI->getNumCandidatesForPush2Pop2())
3268 ++NumFunctionUsingPush2Pop2;
3269 X86FI->setCalleeSavedFrameSize(CalleeSavedFrameSize);
3270 MFI.setCVBytesOfCalleeSavedRegisters(CalleeSavedFrameSize);
3271
3272 // Assign slots for XMMs.
3273 for (CalleeSavedInfo &I : llvm::reverse(C&: CSI)) {
3274 MCRegister Reg = I.getReg();
3275 if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg))
3276 continue;
3277
3278 const TargetRegisterClass *RC = getCalleeSavedSpillRC(Reg, STI, TRI: *TRI);
3279 unsigned Size = TRI->getSpillSize(RC: *RC);
3280 Align Alignment = TRI->getSpillAlign(RC: *RC);
3281 // ensure alignment
3282 assert(SpillSlotOffset < 0 && "SpillSlotOffset should always < 0 on X86");
3283 SpillSlotOffset = -alignTo(Size: -SpillSlotOffset, A: Alignment);
3284
3285 // spill into slot
3286 SpillSlotOffset -= Size;
3287 int SlotIndex = MFI.CreateFixedSpillStackObject(Size, SPOffset: SpillSlotOffset);
3288 I.setFrameIdx(SlotIndex);
3289 MFI.ensureMaxAlignment(Alignment);
3290
3291 // Save the start offset and size of XMM in stack frame for funclets.
3292 if (X86::VR128RegClass.contains(Reg)) {
3293 WinEHXMMSlotInfo[SlotIndex] = XMMCalleeSavedFrameSize;
3294 XMMCalleeSavedFrameSize += Size;
3295 }
3296 }
3297
3298 return true;
3299}
3300
3301bool X86FrameLowering::spillCalleeSavedRegisters(
3302 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
3303 ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
3304 DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
3305
3306 // Don't save CSRs in 32-bit EH funclets. The caller saves EBX, EBP, ESI, EDI
3307 // for us, and there are no XMM CSRs on Win32.
3308 if (MBB.isEHFuncletEntry() && STI.is32Bit() && STI.isOSWindows())
3309 return true;
3310
3311 // Push GPRs. It increases frame size.
3312 const MachineFunction &MF = *MBB.getParent();
3313 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
3314
3315 // Update LiveIn of the basic block and decide whether we can add a kill flag
3316 // to the use.
3317 auto UpdateLiveInCheckCanKill = [&](Register Reg) {
3318 const MachineRegisterInfo &MRI = MF.getRegInfo();
3319 // Do not set a kill flag on values that are also marked as live-in. This
3320 // happens with the @llvm-returnaddress intrinsic and with arguments
3321 // passed in callee saved registers.
3322 // Omitting the kill flags is conservatively correct even if the live-in
3323 // is not used after all.
3324 if (MRI.isLiveIn(Reg))
3325 return false;
3326 MBB.addLiveIn(PhysReg: Reg);
3327 // Check if any subregister is live-in
3328 for (MCRegAliasIterator AReg(Reg, TRI, false); AReg.isValid(); ++AReg)
3329 if (MRI.isLiveIn(Reg: *AReg))
3330 return false;
3331 return true;
3332 };
3333 auto UpdateLiveInGetKillRegState = [&](Register Reg) {
3334 return getKillRegState(B: UpdateLiveInCheckCanKill(Reg));
3335 };
3336
3337 for (auto RI = CSI.rbegin(), RE = CSI.rend(); RI != RE; ++RI) {
3338 MCRegister Reg = RI->getReg();
3339 if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg))
3340 continue;
3341
3342 if (X86FI->isCandidateForPush2Pop2(Reg)) {
3343 MCRegister Reg2 = (++RI)->getReg();
3344 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: getPUSH2Opcode(ST: STI)))
3345 .addReg(RegNo: Reg, Flags: UpdateLiveInGetKillRegState(Reg))
3346 .addReg(RegNo: Reg2, Flags: UpdateLiveInGetKillRegState(Reg2))
3347 .setMIFlag(MachineInstr::FrameSetup);
3348 } else {
3349 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: getPUSHOpcode(ST: STI)))
3350 .addReg(RegNo: Reg, Flags: UpdateLiveInGetKillRegState(Reg))
3351 .setMIFlag(MachineInstr::FrameSetup);
3352 }
3353 }
3354
3355 if (X86FI->getRestoreBasePointer()) {
3356 unsigned Opc = STI.is64Bit() ? X86::PUSH64r : X86::PUSH32r;
3357 Register BaseReg = this->TRI->getBaseRegister();
3358 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: Opc))
3359 .addReg(RegNo: BaseReg, Flags: getKillRegState(B: true))
3360 .setMIFlag(MachineInstr::FrameSetup);
3361 }
3362
3363 // Make XMM regs spilled. X86 does not have ability of push/pop XMM.
3364 // It can be done by spilling XMMs to stack frame.
3365 for (const CalleeSavedInfo &I : llvm::reverse(C&: CSI)) {
3366 MCRegister Reg = I.getReg();
3367 if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg))
3368 continue;
3369
3370 // Add the callee-saved register as live-in. It's killed at the spill.
3371 MBB.addLiveIn(PhysReg: Reg);
3372 const TargetRegisterClass *RC = getCalleeSavedSpillRC(Reg, STI, TRI: *TRI);
3373
3374 TII.storeRegToStackSlot(MBB, MI, SrcReg: Reg, isKill: true, FrameIndex: I.getFrameIdx(), RC, VReg: Register(),
3375 Flags: MachineInstr::FrameSetup);
3376 }
3377
3378 return true;
3379}
3380
3381void X86FrameLowering::emitCatchRetReturnValue(MachineBasicBlock &MBB,
3382 MachineBasicBlock::iterator MBBI,
3383 MachineInstr *CatchRet) const {
3384 // SEH shouldn't use catchret.
3385 assert(!isAsynchronousEHPersonality(classifyEHPersonality(
3386 MBB.getParent()->getFunction().getPersonalityFn())) &&
3387 "SEH should not use CATCHRET");
3388 const DebugLoc &DL = CatchRet->getDebugLoc();
3389 MachineBasicBlock *CatchRetTarget = CatchRet->getOperand(i: 0).getMBB();
3390
3391 // Fill EAX/RAX with the address of the target block.
3392 if (STI.is64Bit()) {
3393 // LEA64r CatchRetTarget(%rip), %rax
3394 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::LEA64r), DestReg: X86::RAX)
3395 .addReg(RegNo: X86::RIP)
3396 .addImm(Val: 0)
3397 .addReg(RegNo: 0)
3398 .addMBB(MBB: CatchRetTarget)
3399 .addReg(RegNo: 0);
3400 } else {
3401 // MOV32ri $CatchRetTarget, %eax
3402 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV32ri), DestReg: X86::EAX)
3403 .addMBB(MBB: CatchRetTarget);
3404 }
3405
3406 // Record that we've taken the address of CatchRetTarget and no longer just
3407 // reference it in a terminator.
3408 CatchRetTarget->setMachineBlockAddressTaken();
3409}
3410
3411bool X86FrameLowering::restoreCalleeSavedRegisters(
3412 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
3413 MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
3414 if (CSI.empty())
3415 return false;
3416
3417 if (MI != MBB.end() && isFuncletReturnInstr(MI&: *MI) && STI.isOSWindows()) {
3418 // Don't restore CSRs in 32-bit EH funclets. Matches
3419 // spillCalleeSavedRegisters.
3420 if (STI.is32Bit())
3421 return true;
3422 // Don't restore CSRs before an SEH catchret. SEH except blocks do not form
3423 // funclets. emitEpilogue transforms these to normal jumps.
3424 if (MI->getOpcode() == X86::CATCHRET) {
3425 const Function &F = MBB.getParent()->getFunction();
3426 bool IsSEH = isAsynchronousEHPersonality(
3427 Pers: classifyEHPersonality(Pers: F.getPersonalityFn()));
3428 if (IsSEH)
3429 return true;
3430 }
3431 }
3432
3433 DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
3434 MachineFunction &MF = *MBB.getParent();
3435 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
3436
3437 bool NeedsWin64CFI = MF.needsWinCFI();
3438 bool IsWin64UnwindV3 = NeedsWin64CFI && requireWinX64UnwindV3(MF);
3439
3440 // Reload XMMs from stack frame.
3441 for (const CalleeSavedInfo &I : CSI) {
3442 MCRegister Reg = I.getReg();
3443 if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg))
3444 continue;
3445
3446 const TargetRegisterClass *RC = getCalleeSavedSpillRC(Reg, STI, TRI: *TRI);
3447 TII.loadRegFromStackSlot(MBB, MI, DestReg: Reg, FrameIndex: I.getFrameIdx(), RC, VReg: Register(), SubReg: 0,
3448 Flags: MachineInstr::FrameDestroy);
3449 }
3450
3451 // Clear the stack slot for spill base pointer register.
3452 if (X86FI->getRestoreBasePointer()) {
3453 if (IsWin64UnwindV3)
3454 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_PushReg))
3455 .addImm(Val: this->TRI->getBaseRegister())
3456 .setMIFlag(MachineInstr::FrameDestroy);
3457 unsigned Opc = STI.is64Bit() ? X86::POP64r : X86::POP32r;
3458 Register BaseReg = this->TRI->getBaseRegister();
3459 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: BaseReg)
3460 .setMIFlag(MachineInstr::FrameDestroy);
3461 }
3462
3463 // POP GPRs.
3464 for (auto I = CSI.begin(), E = CSI.end(); I != E; ++I) {
3465 MCRegister Reg = I->getReg();
3466 if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg))
3467 continue;
3468
3469 if (X86FI->isCandidateForPush2Pop2(Reg)) {
3470 MCRegister Reg2 = (++I)->getReg();
3471 if (IsWin64UnwindV3) {
3472 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_Push2Regs))
3473 .addImm(Val: Reg)
3474 .addImm(Val: Reg2)
3475 .setMIFlag(MachineInstr::FrameDestroy);
3476 }
3477 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: getPOP2Opcode(ST: STI)), DestReg: Reg)
3478 .addReg(RegNo: Reg2, Flags: RegState::Define)
3479 .setMIFlag(MachineInstr::FrameDestroy);
3480 } else {
3481 if (IsWin64UnwindV3)
3482 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_PushReg))
3483 .addImm(Val: Reg)
3484 .setMIFlag(MachineInstr::FrameDestroy);
3485 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: getPOPOpcode(ST: STI)), DestReg: Reg)
3486 .setMIFlag(MachineInstr::FrameDestroy);
3487 }
3488 }
3489
3490 return true;
3491}
3492
3493void X86FrameLowering::determineCalleeSaves(MachineFunction &MF,
3494 BitVector &SavedRegs,
3495 RegScavenger *RS) const {
3496 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
3497
3498 // Spill the BasePtr if it's used.
3499 if (TRI->hasBasePointer(MF)) {
3500 Register BasePtr = TRI->getBaseRegister();
3501 if (STI.isTarget64BitILP32())
3502 BasePtr = getX86SubSuperRegister(Reg: BasePtr, Size: 64);
3503 SavedRegs.set(BasePtr);
3504 }
3505 if (STI.hasUserReservedRegisters()) {
3506 for (int Reg = SavedRegs.find_first(); Reg != -1;
3507 Reg = SavedRegs.find_next(Prev: Reg)) {
3508 if (STI.isRegisterReservedByUser(i: Reg)) {
3509 SavedRegs.reset(Idx: Reg);
3510 }
3511 }
3512 }
3513}
3514
3515static bool HasNestArgument(const MachineFunction *MF) {
3516 const Function &F = MF->getFunction();
3517 for (Function::const_arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E;
3518 I++) {
3519 if (I->hasNestAttr() && !I->use_empty())
3520 return true;
3521 }
3522 return false;
3523}
3524
3525/// GetScratchRegister - Get a temp register for performing work in the
3526/// segmented stack and the Erlang/HiPE stack prologue. Depending on platform
3527/// and the properties of the function either one or two registers will be
3528/// needed. Set primary to true for the first register, false for the second.
3529static unsigned GetScratchRegister(bool Is64Bit, bool IsLP64,
3530 const MachineFunction &MF, bool Primary) {
3531 CallingConv::ID CallingConvention = MF.getFunction().getCallingConv();
3532
3533 // Erlang stuff.
3534 if (CallingConvention == CallingConv::HiPE) {
3535 if (Is64Bit)
3536 return Primary ? X86::R14 : X86::R13;
3537 else
3538 return Primary ? X86::EBX : X86::EDI;
3539 }
3540
3541 if (Is64Bit) {
3542 if (IsLP64)
3543 return Primary ? X86::R11 : X86::R12;
3544 else
3545 return Primary ? X86::R11D : X86::R12D;
3546 }
3547
3548 bool IsNested = HasNestArgument(MF: &MF);
3549
3550 if (CallingConvention == CallingConv::X86_FastCall ||
3551 CallingConvention == CallingConv::Fast ||
3552 CallingConvention == CallingConv::Tail) {
3553 if (IsNested)
3554 report_fatal_error(reason: "Segmented stacks does not support fastcall with "
3555 "nested function.");
3556 return Primary ? X86::EAX : X86::ECX;
3557 }
3558 if (IsNested)
3559 return Primary ? X86::EDX : X86::EAX;
3560 return Primary ? X86::ECX : X86::EAX;
3561}
3562
3563// The stack limit in the TCB is set to this many bytes above the actual stack
3564// limit.
3565static const uint64_t kSplitStackAvailable = 256;
3566
3567void X86FrameLowering::adjustForSegmentedStacks(
3568 MachineFunction &MF, MachineBasicBlock &PrologueMBB) const {
3569 MachineFrameInfo &MFI = MF.getFrameInfo();
3570 uint64_t StackSize;
3571 unsigned TlsReg, TlsOffset;
3572 DebugLoc DL;
3573
3574 // To support shrink-wrapping we would need to insert the new blocks
3575 // at the right place and update the branches to PrologueMBB.
3576 assert(&(*MF.begin()) == &PrologueMBB && "Shrink-wrapping not supported yet");
3577
3578 unsigned ScratchReg = GetScratchRegister(Is64Bit, IsLP64, MF, Primary: true);
3579 assert(!MF.getRegInfo().isLiveIn(ScratchReg) &&
3580 "Scratch register is live-in");
3581
3582 if (MF.getFunction().isVarArg())
3583 report_fatal_error(reason: "Segmented stacks do not support vararg functions.");
3584 if (!STI.isTargetLinux() && !STI.isTargetDarwin() && !STI.isTargetWin32() &&
3585 !STI.isTargetWin64() && !STI.isTargetFreeBSD() &&
3586 !STI.isTargetDragonFly())
3587 report_fatal_error(reason: "Segmented stacks not supported on this platform.");
3588
3589 // Eventually StackSize will be calculated by a link-time pass; which will
3590 // also decide whether checking code needs to be injected into this particular
3591 // prologue.
3592 StackSize = MFI.getStackSize();
3593
3594 if (!MFI.needsSplitStackProlog())
3595 return;
3596
3597 MachineBasicBlock *allocMBB = MF.CreateMachineBasicBlock();
3598 MachineBasicBlock *checkMBB = MF.CreateMachineBasicBlock();
3599 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
3600 bool IsNested = false;
3601
3602 // We need to know if the function has a nest argument only in 64 bit mode.
3603 if (Is64Bit)
3604 IsNested = HasNestArgument(MF: &MF);
3605
3606 // The MOV R10, RAX needs to be in a different block, since the RET we emit in
3607 // allocMBB needs to be last (terminating) instruction.
3608
3609 for (const auto &LI : PrologueMBB.liveins()) {
3610 allocMBB->addLiveIn(RegMaskPair: LI);
3611 checkMBB->addLiveIn(RegMaskPair: LI);
3612 }
3613
3614 if (IsNested)
3615 allocMBB->addLiveIn(PhysReg: IsLP64 ? X86::R10 : X86::R10D);
3616
3617 MF.push_front(MBB: allocMBB);
3618 MF.push_front(MBB: checkMBB);
3619
3620 // When the frame size is less than 256 we just compare the stack
3621 // boundary directly to the value of the stack pointer, per gcc.
3622 bool CompareStackPointer = StackSize < kSplitStackAvailable;
3623
3624 // Read the limit off the current stacklet off the stack_guard location.
3625 if (Is64Bit) {
3626 if (STI.isTargetLinux()) {
3627 TlsReg = X86::FS;
3628 TlsOffset = IsLP64 ? 0x70 : 0x40;
3629 } else if (STI.isTargetDarwin()) {
3630 TlsReg = X86::GS;
3631 TlsOffset = 0x60 + 90 * 8; // See pthread_machdep.h. Steal TLS slot 90.
3632 } else if (STI.isTargetWin64()) {
3633 TlsReg = X86::GS;
3634 TlsOffset = 0x28; // pvArbitrary, reserved for application use
3635 } else if (STI.isTargetFreeBSD()) {
3636 TlsReg = X86::FS;
3637 TlsOffset = 0x18;
3638 } else if (STI.isTargetDragonFly()) {
3639 TlsReg = X86::FS;
3640 TlsOffset = 0x20; // use tls_tcb.tcb_segstack
3641 } else {
3642 report_fatal_error(reason: "Segmented stacks not supported on this platform.");
3643 }
3644
3645 if (CompareStackPointer)
3646 ScratchReg = IsLP64 ? X86::RSP : X86::ESP;
3647 else
3648 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: IsLP64 ? X86::LEA64r : X86::LEA64_32r),
3649 DestReg: ScratchReg)
3650 .addReg(RegNo: X86::RSP)
3651 .addImm(Val: 1)
3652 .addReg(RegNo: 0)
3653 .addImm(Val: -StackSize)
3654 .addReg(RegNo: 0);
3655
3656 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: IsLP64 ? X86::CMP64rm : X86::CMP32rm))
3657 .addReg(RegNo: ScratchReg)
3658 .addReg(RegNo: 0)
3659 .addImm(Val: 1)
3660 .addReg(RegNo: 0)
3661 .addImm(Val: TlsOffset)
3662 .addReg(RegNo: TlsReg);
3663 } else {
3664 if (STI.isTargetLinux()) {
3665 TlsReg = X86::GS;
3666 TlsOffset = 0x30;
3667 } else if (STI.isTargetDarwin()) {
3668 TlsReg = X86::GS;
3669 TlsOffset = 0x48 + 90 * 4;
3670 } else if (STI.isTargetWin32()) {
3671 TlsReg = X86::FS;
3672 TlsOffset = 0x14; // pvArbitrary, reserved for application use
3673 } else if (STI.isTargetDragonFly()) {
3674 TlsReg = X86::FS;
3675 TlsOffset = 0x10; // use tls_tcb.tcb_segstack
3676 } else if (STI.isTargetFreeBSD()) {
3677 report_fatal_error(reason: "Segmented stacks not supported on FreeBSD i386.");
3678 } else {
3679 report_fatal_error(reason: "Segmented stacks not supported on this platform.");
3680 }
3681
3682 if (CompareStackPointer)
3683 ScratchReg = X86::ESP;
3684 else
3685 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: X86::LEA32r), DestReg: ScratchReg)
3686 .addReg(RegNo: X86::ESP)
3687 .addImm(Val: 1)
3688 .addReg(RegNo: 0)
3689 .addImm(Val: -StackSize)
3690 .addReg(RegNo: 0);
3691
3692 if (STI.isTargetLinux() || STI.isTargetWin32() || STI.isTargetWin64() ||
3693 STI.isTargetDragonFly()) {
3694 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: X86::CMP32rm))
3695 .addReg(RegNo: ScratchReg)
3696 .addReg(RegNo: 0)
3697 .addImm(Val: 0)
3698 .addReg(RegNo: 0)
3699 .addImm(Val: TlsOffset)
3700 .addReg(RegNo: TlsReg);
3701 } else if (STI.isTargetDarwin()) {
3702
3703 // TlsOffset doesn't fit into a mod r/m byte so we need an extra register.
3704 unsigned ScratchReg2;
3705 bool SaveScratch2;
3706 if (CompareStackPointer) {
3707 // The primary scratch register is available for holding the TLS offset.
3708 ScratchReg2 = GetScratchRegister(Is64Bit, IsLP64, MF, Primary: true);
3709 SaveScratch2 = false;
3710 } else {
3711 // Need to use a second register to hold the TLS offset
3712 ScratchReg2 = GetScratchRegister(Is64Bit, IsLP64, MF, Primary: false);
3713
3714 // Unfortunately, with fastcc the second scratch register may hold an
3715 // argument.
3716 SaveScratch2 = MF.getRegInfo().isLiveIn(Reg: ScratchReg2);
3717 }
3718
3719 // If Scratch2 is live-in then it needs to be saved.
3720 assert((!MF.getRegInfo().isLiveIn(ScratchReg2) || SaveScratch2) &&
3721 "Scratch register is live-in and not saved");
3722
3723 if (SaveScratch2)
3724 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH32r))
3725 .addReg(RegNo: ScratchReg2, Flags: RegState::Kill);
3726
3727 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: X86::MOV32ri), DestReg: ScratchReg2)
3728 .addImm(Val: TlsOffset);
3729 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: X86::CMP32rm))
3730 .addReg(RegNo: ScratchReg)
3731 .addReg(RegNo: ScratchReg2)
3732 .addImm(Val: 1)
3733 .addReg(RegNo: 0)
3734 .addImm(Val: 0)
3735 .addReg(RegNo: TlsReg);
3736
3737 if (SaveScratch2)
3738 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: X86::POP32r), DestReg: ScratchReg2);
3739 }
3740 }
3741
3742 // This jump is taken if SP >= (Stacklet Limit + Stack Space required).
3743 // It jumps to normal execution of the function body.
3744 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
3745 .addMBB(MBB: &PrologueMBB)
3746 .addImm(Val: X86::COND_A);
3747
3748 // On 32 bit we first push the arguments size and then the frame size. On 64
3749 // bit, we pass the stack frame size in r10 and the argument size in r11.
3750 if (Is64Bit) {
3751 // Functions with nested arguments use R10, so it needs to be saved across
3752 // the call to _morestack
3753
3754 const unsigned RegAX = IsLP64 ? X86::RAX : X86::EAX;
3755 const unsigned Reg10 = IsLP64 ? X86::R10 : X86::R10D;
3756 const unsigned Reg11 = IsLP64 ? X86::R11 : X86::R11D;
3757 const unsigned MOVrr = IsLP64 ? X86::MOV64rr : X86::MOV32rr;
3758
3759 if (IsNested)
3760 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: MOVrr), DestReg: RegAX).addReg(RegNo: Reg10);
3761
3762 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: X86::getMOVriOpcode(Use64BitReg: IsLP64, Imm: StackSize)),
3763 DestReg: Reg10)
3764 .addImm(Val: StackSize);
3765 BuildMI(BB: allocMBB, MIMD: DL,
3766 MCID: TII.get(Opcode: X86::getMOVriOpcode(Use64BitReg: IsLP64, Imm: X86FI->getArgumentStackSize())),
3767 DestReg: Reg11)
3768 .addImm(Val: X86FI->getArgumentStackSize());
3769 } else {
3770 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH32i))
3771 .addImm(Val: X86FI->getArgumentStackSize());
3772 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH32i)).addImm(Val: StackSize);
3773 }
3774
3775 // __morestack is in libgcc
3776 if (Is64Bit && MF.getTarget().getCodeModel() == CodeModel::Large) {
3777 // Under the large code model, we cannot assume that __morestack lives
3778 // within 2^31 bytes of the call site, so we cannot use pc-relative
3779 // addressing. We cannot perform the call via a temporary register,
3780 // as the rax register may be used to store the static chain, and all
3781 // other suitable registers may be either callee-save or used for
3782 // parameter passing. We cannot use the stack at this point either
3783 // because __morestack manipulates the stack directly.
3784 //
3785 // To avoid these issues, perform an indirect call via a read-only memory
3786 // location containing the address.
3787 //
3788 // This solution is not perfect, as it assumes that the .rodata section
3789 // is laid out within 2^31 bytes of each function body, but this seems
3790 // to be sufficient for JIT.
3791 // FIXME: Add retpoline support and remove the error here..
3792 if (STI.useIndirectThunkCalls())
3793 report_fatal_error(reason: "Emitting morestack calls on 64-bit with the large "
3794 "code model and thunks not yet implemented.");
3795 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: X86::CALL64m))
3796 .addReg(RegNo: X86::RIP)
3797 .addImm(Val: 0)
3798 .addReg(RegNo: 0)
3799 .addExternalSymbol(FnName: "__morestack_addr")
3800 .addReg(RegNo: 0);
3801 } else {
3802 if (Is64Bit)
3803 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: X86::CALL64pcrel32))
3804 .addExternalSymbol(FnName: "__morestack");
3805 else
3806 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: X86::CALLpcrel32))
3807 .addExternalSymbol(FnName: "__morestack");
3808 }
3809
3810 if (IsNested)
3811 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: X86::MORESTACK_RET_RESTORE_R10));
3812 else
3813 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: X86::MORESTACK_RET));
3814
3815 allocMBB->addSuccessor(Succ: &PrologueMBB);
3816
3817 checkMBB->addSuccessor(Succ: allocMBB, Prob: BranchProbability::getZero());
3818 checkMBB->addSuccessor(Succ: &PrologueMBB, Prob: BranchProbability::getOne());
3819
3820#ifdef EXPENSIVE_CHECKS
3821 MF.verify();
3822#endif
3823}
3824
3825/// Lookup an ERTS parameter in the !hipe.literals named metadata node.
3826/// HiPE provides Erlang Runtime System-internal parameters, such as PCB offsets
3827/// to fields it needs, through a named metadata node "hipe.literals" containing
3828/// name-value pairs.
3829static unsigned getHiPELiteral(NamedMDNode *HiPELiteralsMD,
3830 const StringRef LiteralName) {
3831 for (int i = 0, e = HiPELiteralsMD->getNumOperands(); i != e; ++i) {
3832 MDNode *Node = HiPELiteralsMD->getOperand(i);
3833 if (Node->getNumOperands() != 2)
3834 continue;
3835 MDString *NodeName = dyn_cast<MDString>(Val: Node->getOperand(I: 0));
3836 ValueAsMetadata *NodeVal = dyn_cast<ValueAsMetadata>(Val: Node->getOperand(I: 1));
3837 if (!NodeName || !NodeVal)
3838 continue;
3839 ConstantInt *ValConst = dyn_cast_or_null<ConstantInt>(Val: NodeVal->getValue());
3840 if (ValConst && NodeName->getString() == LiteralName) {
3841 return ValConst->getZExtValue();
3842 }
3843 }
3844
3845 report_fatal_error(reason: "HiPE literal " + LiteralName +
3846 " required but not provided");
3847}
3848
3849// Return true if there are no non-ehpad successors to MBB and there are no
3850// non-meta instructions between MBBI and MBB.end().
3851static bool blockEndIsUnreachable(const MachineBasicBlock &MBB,
3852 MachineBasicBlock::const_iterator MBBI) {
3853 return llvm::all_of(
3854 Range: MBB.successors(),
3855 P: [](const MachineBasicBlock *Succ) { return Succ->isEHPad(); }) &&
3856 std::all_of(first: MBBI, last: MBB.end(), pred: [](const MachineInstr &MI) {
3857 return MI.isMetaInstruction();
3858 });
3859}
3860
3861/// Erlang programs may need a special prologue to handle the stack size they
3862/// might need at runtime. That is because Erlang/OTP does not implement a C
3863/// stack but uses a custom implementation of hybrid stack/heap architecture.
3864/// (for more information see Eric Stenman's Ph.D. thesis:
3865/// http://publications.uu.se/uu/fulltext/nbn_se_uu_diva-2688.pdf)
3866///
3867/// CheckStack:
3868/// temp0 = sp - MaxStack
3869/// if( temp0 < SP_LIMIT(P) ) goto IncStack else goto OldStart
3870/// OldStart:
3871/// ...
3872/// IncStack:
3873/// call inc_stack # doubles the stack space
3874/// temp0 = sp - MaxStack
3875/// if( temp0 < SP_LIMIT(P) ) goto IncStack else goto OldStart
3876void X86FrameLowering::adjustForHiPEPrologue(
3877 MachineFunction &MF, MachineBasicBlock &PrologueMBB) const {
3878 MachineFrameInfo &MFI = MF.getFrameInfo();
3879 DebugLoc DL;
3880
3881 // To support shrink-wrapping we would need to insert the new blocks
3882 // at the right place and update the branches to PrologueMBB.
3883 assert(&(*MF.begin()) == &PrologueMBB && "Shrink-wrapping not supported yet");
3884
3885 // HiPE-specific values
3886 NamedMDNode *HiPELiteralsMD =
3887 MF.getFunction().getParent()->getNamedMetadata(Name: "hipe.literals");
3888 if (!HiPELiteralsMD)
3889 report_fatal_error(
3890 reason: "Can't generate HiPE prologue without runtime parameters");
3891 const unsigned HipeLeafWords = getHiPELiteral(
3892 HiPELiteralsMD, LiteralName: Is64Bit ? "AMD64_LEAF_WORDS" : "X86_LEAF_WORDS");
3893 const unsigned CCRegisteredArgs = Is64Bit ? 6 : 5;
3894 const unsigned Guaranteed = HipeLeafWords * SlotSize;
3895 unsigned CallerStkArity = MF.getFunction().arg_size() > CCRegisteredArgs
3896 ? MF.getFunction().arg_size() - CCRegisteredArgs
3897 : 0;
3898 unsigned MaxStack = MFI.getStackSize() + CallerStkArity * SlotSize + SlotSize;
3899
3900 assert(STI.isTargetLinux() &&
3901 "HiPE prologue is only supported on Linux operating systems.");
3902
3903 // Compute the largest caller's frame that is needed to fit the callees'
3904 // frames. This 'MaxStack' is computed from:
3905 //
3906 // a) the fixed frame size, which is the space needed for all spilled temps,
3907 // b) outgoing on-stack parameter areas, and
3908 // c) the minimum stack space this function needs to make available for the
3909 // functions it calls (a tunable ABI property).
3910 if (MFI.hasCalls()) {
3911 unsigned MoreStackForCalls = 0;
3912
3913 for (auto &MBB : MF) {
3914 for (auto &MI : MBB) {
3915 if (!MI.isCall())
3916 continue;
3917
3918 // Get callee operand.
3919 const MachineOperand &MO = MI.getOperand(i: 0);
3920
3921 // Only take account of global function calls (no closures etc.).
3922 if (!MO.isGlobal())
3923 continue;
3924
3925 const Function *F = dyn_cast<Function>(Val: MO.getGlobal());
3926 if (!F)
3927 continue;
3928
3929 // Do not update 'MaxStack' for primitive and built-in functions
3930 // (encoded with names either starting with "erlang."/"bif_" or not
3931 // having a ".", such as a simple <Module>.<Function>.<Arity>, or an
3932 // "_", such as the BIF "suspend_0") as they are executed on another
3933 // stack.
3934 if (F->getName().contains(Other: "erlang.") || F->getName().contains(Other: "bif_") ||
3935 F->getName().find_first_of(Chars: "._") == StringRef::npos)
3936 continue;
3937
3938 unsigned CalleeStkArity = F->arg_size() > CCRegisteredArgs
3939 ? F->arg_size() - CCRegisteredArgs
3940 : 0;
3941 if (HipeLeafWords - 1 > CalleeStkArity)
3942 MoreStackForCalls =
3943 std::max(a: MoreStackForCalls,
3944 b: (HipeLeafWords - 1 - CalleeStkArity) * SlotSize);
3945 }
3946 }
3947 MaxStack += MoreStackForCalls;
3948 }
3949
3950 // If the stack frame needed is larger than the guaranteed then runtime checks
3951 // and calls to "inc_stack_0" BIF should be inserted in the assembly prologue.
3952 if (MaxStack > Guaranteed) {
3953 MachineBasicBlock *stackCheckMBB = MF.CreateMachineBasicBlock();
3954 MachineBasicBlock *incStackMBB = MF.CreateMachineBasicBlock();
3955
3956 for (const auto &LI : PrologueMBB.liveins()) {
3957 stackCheckMBB->addLiveIn(RegMaskPair: LI);
3958 incStackMBB->addLiveIn(RegMaskPair: LI);
3959 }
3960
3961 MF.push_front(MBB: incStackMBB);
3962 MF.push_front(MBB: stackCheckMBB);
3963
3964 unsigned ScratchReg, SPReg, PReg, SPLimitOffset;
3965 unsigned LEAop, CMPop, CALLop;
3966 SPLimitOffset = getHiPELiteral(HiPELiteralsMD, LiteralName: "P_NSP_LIMIT");
3967 if (Is64Bit) {
3968 SPReg = X86::RSP;
3969 PReg = X86::RBP;
3970 LEAop = X86::LEA64r;
3971 CMPop = X86::CMP64rm;
3972 CALLop = X86::CALL64pcrel32;
3973 } else {
3974 SPReg = X86::ESP;
3975 PReg = X86::EBP;
3976 LEAop = X86::LEA32r;
3977 CMPop = X86::CMP32rm;
3978 CALLop = X86::CALLpcrel32;
3979 }
3980
3981 ScratchReg = GetScratchRegister(Is64Bit, IsLP64, MF, Primary: true);
3982 assert(!MF.getRegInfo().isLiveIn(ScratchReg) &&
3983 "HiPE prologue scratch register is live-in");
3984
3985 // Create new MBB for StackCheck:
3986 addRegOffset(MIB: BuildMI(BB: stackCheckMBB, MIMD: DL, MCID: TII.get(Opcode: LEAop), DestReg: ScratchReg), Reg: SPReg,
3987 isKill: false, Offset: -MaxStack);
3988 // SPLimitOffset is in a fixed heap location (pointed by BP).
3989 addRegOffset(MIB: BuildMI(BB: stackCheckMBB, MIMD: DL, MCID: TII.get(Opcode: CMPop)).addReg(RegNo: ScratchReg),
3990 Reg: PReg, isKill: false, Offset: SPLimitOffset);
3991 BuildMI(BB: stackCheckMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
3992 .addMBB(MBB: &PrologueMBB)
3993 .addImm(Val: X86::COND_AE);
3994
3995 // Create new MBB for IncStack:
3996 BuildMI(BB: incStackMBB, MIMD: DL, MCID: TII.get(Opcode: CALLop)).addExternalSymbol(FnName: "inc_stack_0");
3997 addRegOffset(MIB: BuildMI(BB: incStackMBB, MIMD: DL, MCID: TII.get(Opcode: LEAop), DestReg: ScratchReg), Reg: SPReg,
3998 isKill: false, Offset: -MaxStack);
3999 addRegOffset(MIB: BuildMI(BB: incStackMBB, MIMD: DL, MCID: TII.get(Opcode: CMPop)).addReg(RegNo: ScratchReg),
4000 Reg: PReg, isKill: false, Offset: SPLimitOffset);
4001 BuildMI(BB: incStackMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
4002 .addMBB(MBB: incStackMBB)
4003 .addImm(Val: X86::COND_LE);
4004
4005 stackCheckMBB->addSuccessor(Succ: &PrologueMBB, Prob: {99, 100});
4006 stackCheckMBB->addSuccessor(Succ: incStackMBB, Prob: {1, 100});
4007 incStackMBB->addSuccessor(Succ: &PrologueMBB, Prob: {99, 100});
4008 incStackMBB->addSuccessor(Succ: incStackMBB, Prob: {1, 100});
4009 }
4010#ifdef EXPENSIVE_CHECKS
4011 MF.verify();
4012#endif
4013}
4014
4015bool X86FrameLowering::adjustStackWithPops(MachineBasicBlock &MBB,
4016 MachineBasicBlock::iterator MBBI,
4017 const DebugLoc &DL,
4018 int Offset) const {
4019 if (Offset <= 0)
4020 return false;
4021
4022 if (Offset % SlotSize)
4023 return false;
4024
4025 int NumPops = Offset / SlotSize;
4026 // This is only worth it if we have at most 2 pops.
4027 if (NumPops != 1 && NumPops != 2)
4028 return false;
4029
4030 // Handle only the trivial case where the adjustment directly follows
4031 // a call. This is the most common one, anyway.
4032 if (MBBI == MBB.begin())
4033 return false;
4034 MachineBasicBlock::iterator Prev = std::prev(x: MBBI);
4035 if (!Prev->isCall() || !Prev->getOperand(i: 1).isRegMask())
4036 return false;
4037
4038 unsigned Regs[2];
4039 unsigned FoundRegs = 0;
4040
4041 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4042 const MachineOperand &RegMask = Prev->getOperand(i: 1);
4043
4044 auto &RegClass =
4045 Is64Bit ? X86::GR64_NOREX_NOSPRegClass : X86::GR32_NOREX_NOSPRegClass;
4046 // Try to find up to NumPops free registers.
4047 for (auto Candidate : RegClass) {
4048 // Poor man's liveness:
4049 // Since we're immediately after a call, any register that is clobbered
4050 // by the call and not defined by it can be considered dead.
4051 if (!RegMask.clobbersPhysReg(PhysReg: Candidate))
4052 continue;
4053
4054 // Don't clobber reserved registers
4055 if (MRI.isReserved(PhysReg: Candidate))
4056 continue;
4057
4058 bool IsDef = false;
4059 for (const MachineOperand &MO : Prev->implicit_operands()) {
4060 if (MO.isReg() && MO.isDef() &&
4061 TRI->isSuperOrSubRegisterEq(RegA: MO.getReg(), RegB: Candidate)) {
4062 IsDef = true;
4063 break;
4064 }
4065 }
4066
4067 if (IsDef)
4068 continue;
4069
4070 Regs[FoundRegs++] = Candidate;
4071 if (FoundRegs == (unsigned)NumPops)
4072 break;
4073 }
4074
4075 if (FoundRegs == 0)
4076 return false;
4077
4078 // If we found only one free register, but need two, reuse the same one twice.
4079 while (FoundRegs < (unsigned)NumPops)
4080 Regs[FoundRegs++] = Regs[0];
4081
4082 for (int i = 0; i < NumPops; ++i)
4083 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: STI.is64Bit() ? X86::POP64r : X86::POP32r),
4084 DestReg: Regs[i]);
4085
4086 return true;
4087}
4088
4089MachineBasicBlock::iterator X86FrameLowering::eliminateCallFramePseudoInstr(
4090 MachineFunction &MF, MachineBasicBlock &MBB,
4091 MachineBasicBlock::iterator I) const {
4092 bool reserveCallFrame = hasReservedCallFrame(MF);
4093 unsigned Opcode = I->getOpcode();
4094 bool isDestroy = Opcode == TII.getCallFrameDestroyOpcode();
4095 DebugLoc DL = I->getDebugLoc(); // copy DebugLoc as I will be erased.
4096 uint64_t Amount = TII.getFrameSize(I: *I);
4097 uint64_t InternalAmt = (isDestroy || Amount) ? TII.getFrameAdjustment(I: *I) : 0;
4098 I = MBB.erase(I);
4099 auto InsertPos = skipDebugInstructionsForward(It: I, End: MBB.end());
4100
4101 // Try to avoid emitting dead SP adjustments if the block end is unreachable,
4102 // typically because the function is marked noreturn (abort, throw,
4103 // assert_fail, etc).
4104 if (isDestroy && blockEndIsUnreachable(MBB, MBBI: I))
4105 return I;
4106
4107 if (!reserveCallFrame) {
4108 // If the stack pointer can be changed after prologue, turn the
4109 // adjcallstackup instruction into a 'sub ESP, <amt>' and the
4110 // adjcallstackdown instruction into 'add ESP, <amt>'
4111
4112 // We need to keep the stack aligned properly. To do this, we round the
4113 // amount of space needed for the outgoing arguments up to the next
4114 // alignment boundary.
4115 Amount = alignTo(Size: Amount, A: getStackAlign());
4116
4117 const Function &F = MF.getFunction();
4118 bool WindowsCFI = MF.usesWindowsUnwindInfo();
4119 bool DwarfCFI = !WindowsCFI && MF.needsFrameMoves();
4120
4121 // If we have any exception handlers in this function, and we adjust
4122 // the SP before calls, we may need to indicate this to the unwinder
4123 // using GNU_ARGS_SIZE. Note that this may be necessary even when
4124 // Amount == 0, because the preceding function may have set a non-0
4125 // GNU_ARGS_SIZE.
4126 // TODO: We don't need to reset this between subsequent functions,
4127 // if it didn't change.
4128 bool HasDwarfEHHandlers = !WindowsCFI && !MF.getLandingPads().empty();
4129
4130 if (HasDwarfEHHandlers && !isDestroy &&
4131 MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences())
4132 BuildCFI(MBB, MBBI: InsertPos, DL,
4133 CFIInst: MCCFIInstruction::createGnuArgsSize(L: nullptr, Size: Amount));
4134
4135 if (Amount == 0)
4136 return I;
4137
4138 // Factor out the amount that gets handled inside the sequence
4139 // (Pushes of argument for frame setup, callee pops for frame destroy)
4140 Amount -= InternalAmt;
4141
4142 // TODO: This is needed only if we require precise CFA.
4143 // If this is a callee-pop calling convention, emit a CFA adjust for
4144 // the amount the callee popped.
4145 if (isDestroy && InternalAmt && DwarfCFI && !hasFP(MF))
4146 BuildCFI(MBB, MBBI: InsertPos, DL,
4147 CFIInst: MCCFIInstruction::createAdjustCfaOffset(L: nullptr, Adjustment: -InternalAmt));
4148
4149 // Add Amount to SP to destroy a frame, or subtract to setup.
4150 int64_t StackAdjustment = isDestroy ? Amount : -Amount;
4151 int64_t CfaAdjustment = StackAdjustment;
4152
4153 if (StackAdjustment) {
4154 // Merge with any previous or following adjustment instruction. Note: the
4155 // instructions merged with here do not have CFI, so their stack
4156 // adjustments do not feed into CfaAdjustment
4157
4158 auto CalcCfaAdjust = [&CfaAdjustment](MachineBasicBlock::iterator PI,
4159 int64_t Offset) {
4160 CfaAdjustment += Offset;
4161 };
4162 auto CalcNewOffset = [&StackAdjustment](int64_t Offset) {
4163 return StackAdjustment + Offset;
4164 };
4165 StackAdjustment =
4166 mergeSPUpdates(MBB, MBBI&: InsertPos, FoundStackAdjust: CalcCfaAdjust, CalcNewOffset, doMergeWithPrevious: true);
4167 StackAdjustment =
4168 mergeSPUpdates(MBB, MBBI&: InsertPos, FoundStackAdjust: CalcCfaAdjust, CalcNewOffset, doMergeWithPrevious: false);
4169
4170 if (StackAdjustment) {
4171 if (!(F.hasMinSize() &&
4172 adjustStackWithPops(MBB, MBBI: InsertPos, DL, Offset: StackAdjustment)))
4173 BuildStackAdjustment(MBB, MBBI: InsertPos, DL, Offset: StackAdjustment,
4174 /*InEpilogue=*/false);
4175 }
4176 }
4177
4178 if (DwarfCFI && !hasFP(MF) && CfaAdjustment) {
4179 // If we don't have FP, but need to generate unwind information,
4180 // we need to set the correct CFA offset after the stack adjustment.
4181 // How much we adjust the CFA offset depends on whether we're emitting
4182 // CFI only for EH purposes or for debugging. EH only requires the CFA
4183 // offset to be correct at each call site, while for debugging we want
4184 // it to be more precise.
4185
4186 // TODO: When not using precise CFA, we also need to adjust for the
4187 // InternalAmt here.
4188 BuildCFI(
4189 MBB, MBBI: InsertPos, DL,
4190 CFIInst: MCCFIInstruction::createAdjustCfaOffset(L: nullptr, Adjustment: -CfaAdjustment));
4191 }
4192
4193 return I;
4194 }
4195
4196 if (InternalAmt) {
4197 MachineBasicBlock::iterator CI = I;
4198 MachineBasicBlock::iterator B = MBB.begin();
4199 while (CI != B && !std::prev(x: CI)->isCall())
4200 --CI;
4201 BuildStackAdjustment(MBB, MBBI: CI, DL, Offset: -InternalAmt, /*InEpilogue=*/false);
4202 }
4203
4204 return I;
4205}
4206
4207bool X86FrameLowering::canUseAsPrologue(const MachineBasicBlock &MBB) const {
4208 assert(MBB.getParent() && "Block is not attached to a function!");
4209 const MachineFunction &MF = *MBB.getParent();
4210 if (!MBB.isLiveIn(Reg: X86::EFLAGS))
4211 return true;
4212
4213 // If stack probes have to loop inline or call, that will clobber EFLAGS.
4214 // FIXME: we could allow cases that will use emitStackProbeInlineGenericBlock.
4215 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
4216 const X86TargetLowering &TLI = *STI.getTargetLowering();
4217 if (TLI.hasInlineStackProbe(MF) || TLI.hasStackProbeSymbol(MF))
4218 return false;
4219
4220 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
4221 return !TRI->hasStackRealignment(MF) && !X86FI->hasSwiftAsyncContext();
4222}
4223
4224bool X86FrameLowering::canUseAsEpilogue(const MachineBasicBlock &MBB) const {
4225 assert(MBB.getParent() && "Block is not attached to a function!");
4226
4227 // Win64 has strict requirements in terms of epilogue and we are
4228 // not taking a chance at messing with them.
4229 // I.e., unless this block is already an exit block, we can't use
4230 // it as an epilogue.
4231 if (STI.isTargetWin64() && !MBB.succ_empty() && !MBB.isReturnBlock())
4232 return false;
4233
4234 // Swift async context epilogue has a BTR instruction that clobbers parts of
4235 // EFLAGS.
4236 const MachineFunction &MF = *MBB.getParent();
4237 if (MF.getInfo<X86MachineFunctionInfo>()->hasSwiftAsyncContext())
4238 return !flagsNeedToBePreservedBeforeTheTerminators(MBB);
4239
4240 if (canUseLEAForSPInEpilogue(MF: *MBB.getParent()))
4241 return true;
4242
4243 // If we cannot use LEA to adjust SP, we may need to use ADD, which
4244 // clobbers the EFLAGS. Check that we do not need to preserve it,
4245 // otherwise, conservatively assume this is not
4246 // safe to insert the epilogue here.
4247 return !flagsNeedToBePreservedBeforeTheTerminators(MBB);
4248}
4249
4250bool X86FrameLowering::enableShrinkWrapping(const MachineFunction &MF) const {
4251 // If we may need to emit frameless compact unwind information, give
4252 // up as this is currently broken: PR25614.
4253 bool CompactUnwind =
4254 MF.getContext().getObjectFileInfo()->getCompactUnwindSection() != nullptr;
4255 return (MF.getFunction().hasFnAttribute(Kind: Attribute::NoUnwind) || hasFP(MF) ||
4256 !CompactUnwind) &&
4257 // The lowering of segmented stack and HiPE only support entry
4258 // blocks as prologue blocks: PR26107. This limitation may be
4259 // lifted if we fix:
4260 // - adjustForSegmentedStacks
4261 // - adjustForHiPEPrologue
4262 MF.getFunction().getCallingConv() != CallingConv::HiPE &&
4263 !MF.shouldSplitStack();
4264}
4265
4266MachineBasicBlock::iterator X86FrameLowering::restoreWin32EHStackPointers(
4267 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
4268 const DebugLoc &DL, bool RestoreSP) const {
4269 assert(STI.isTargetWindowsMSVC() && "funclets only supported in MSVC env");
4270 assert(STI.isTargetWin32() && "EBP/ESI restoration only required on win32");
4271 assert(STI.is32Bit() && !Uses64BitFramePtr &&
4272 "restoring EBP/ESI on non-32-bit target");
4273
4274 MachineFunction &MF = *MBB.getParent();
4275 Register FramePtr = TRI->getFrameRegister(MF);
4276 Register BasePtr = TRI->getBaseRegister();
4277 WinEHFuncInfo &FuncInfo = *MF.getWinEHFuncInfo();
4278 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
4279 MachineFrameInfo &MFI = MF.getFrameInfo();
4280
4281 // FIXME: Don't set FrameSetup flag in catchret case.
4282
4283 int FI = FuncInfo.EHRegNodeFrameIndex;
4284 int EHRegSize = MFI.getObjectSize(ObjectIdx: FI);
4285
4286 if (RestoreSP) {
4287 // MOV32rm -EHRegSize(%ebp), %esp
4288 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV32rm), DestReg: X86::ESP),
4289 Reg: X86::EBP, isKill: true, Offset: -EHRegSize)
4290 .setMIFlag(MachineInstr::FrameSetup);
4291 }
4292
4293 Register UsedReg;
4294 int EHRegOffset = getFrameIndexReference(MF, FI, FrameReg&: UsedReg).getFixed();
4295 int EndOffset = -EHRegOffset - EHRegSize;
4296 FuncInfo.EHRegNodeEndOffset = EndOffset;
4297
4298 if (UsedReg == FramePtr) {
4299 // ADD $offset, %ebp
4300 unsigned ADDri = getADDriOpcode(IsLP64: false);
4301 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: ADDri), DestReg: FramePtr)
4302 .addReg(RegNo: FramePtr)
4303 .addImm(Val: EndOffset)
4304 .setMIFlag(MachineInstr::FrameSetup)
4305 ->getOperand(i: 3)
4306 .setIsDead();
4307 assert(EndOffset >= 0 &&
4308 "end of registration object above normal EBP position!");
4309 } else if (UsedReg == BasePtr) {
4310 // LEA offset(%ebp), %esi
4311 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::LEA32r), DestReg: BasePtr),
4312 Reg: FramePtr, isKill: false, Offset: EndOffset)
4313 .setMIFlag(MachineInstr::FrameSetup);
4314 // MOV32rm SavedEBPOffset(%esi), %ebp
4315 assert(X86FI->getHasSEHFramePtrSave());
4316 int Offset =
4317 getFrameIndexReference(MF, FI: X86FI->getSEHFramePtrSaveIndex(), FrameReg&: UsedReg)
4318 .getFixed();
4319 assert(UsedReg == BasePtr);
4320 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV32rm), DestReg: FramePtr),
4321 Reg: UsedReg, isKill: true, Offset)
4322 .setMIFlag(MachineInstr::FrameSetup);
4323 } else {
4324 llvm_unreachable("32-bit frames with WinEH must use FramePtr or BasePtr");
4325 }
4326 return MBBI;
4327}
4328
4329int X86FrameLowering::getInitialCFAOffset(const MachineFunction &MF) const {
4330 return TRI->getSlotSize();
4331}
4332
4333Register
4334X86FrameLowering::getInitialCFARegister(const MachineFunction &MF) const {
4335 return StackPtr;
4336}
4337
4338TargetFrameLowering::DwarfFrameBase
4339X86FrameLowering::getDwarfFrameBase(const MachineFunction &MF) const {
4340 const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
4341 Register FrameRegister = RI->getFrameRegister(MF);
4342 if (getInitialCFARegister(MF) == FrameRegister &&
4343 MF.getInfo<X86MachineFunctionInfo>()->hasCFIAdjustCfa()) {
4344 DwarfFrameBase FrameBase;
4345 FrameBase.Kind = DwarfFrameBase::CFA;
4346 FrameBase.Location.Offset =
4347 -MF.getFrameInfo().getStackSize() - getInitialCFAOffset(MF);
4348 return FrameBase;
4349 }
4350
4351 return DwarfFrameBase{.Kind: DwarfFrameBase::Register, .Location: {.Reg: FrameRegister}};
4352}
4353
4354namespace {
4355// Struct used by orderFrameObjects to help sort the stack objects.
4356struct X86FrameSortingObject {
4357 bool IsValid = false; // true if we care about this Object.
4358 unsigned ObjectIndex = 0; // Index of Object into MFI list.
4359 unsigned ObjectSize = 0; // Size of Object in bytes.
4360 Align ObjectAlignment = Align(1); // Alignment of Object in bytes.
4361 unsigned ObjectNumUses = 0; // Object static number of uses.
4362};
4363
4364// The comparison function we use for std::sort to order our local
4365// stack symbols. The current algorithm is to use an estimated
4366// "density". This takes into consideration the size and number of
4367// uses each object has in order to roughly minimize code size.
4368// So, for example, an object of size 16B that is referenced 5 times
4369// will get higher priority than 4 4B objects referenced 1 time each.
4370// It's not perfect and we may be able to squeeze a few more bytes out of
4371// it (for example : 0(esp) requires fewer bytes, symbols allocated at the
4372// fringe end can have special consideration, given their size is less
4373// important, etc.), but the algorithmic complexity grows too much to be
4374// worth the extra gains we get. This gets us pretty close.
4375// The final order leaves us with objects with highest priority going
4376// at the end of our list.
4377struct X86FrameSortingComparator {
4378 inline bool operator()(const X86FrameSortingObject &A,
4379 const X86FrameSortingObject &B) const {
4380 uint64_t DensityAScaled, DensityBScaled;
4381
4382 // For consistency in our comparison, all invalid objects are placed
4383 // at the end. This also allows us to stop walking when we hit the
4384 // first invalid item after it's all sorted.
4385 if (!A.IsValid)
4386 return false;
4387 if (!B.IsValid)
4388 return true;
4389
4390 // The density is calculated by doing :
4391 // (double)DensityA = A.ObjectNumUses / A.ObjectSize
4392 // (double)DensityB = B.ObjectNumUses / B.ObjectSize
4393 // Since this approach may cause inconsistencies in
4394 // the floating point <, >, == comparisons, depending on the floating
4395 // point model with which the compiler was built, we're going
4396 // to scale both sides by multiplying with
4397 // A.ObjectSize * B.ObjectSize. This ends up factoring away
4398 // the division and, with it, the need for any floating point
4399 // arithmetic.
4400 DensityAScaled = static_cast<uint64_t>(A.ObjectNumUses) *
4401 static_cast<uint64_t>(B.ObjectSize);
4402 DensityBScaled = static_cast<uint64_t>(B.ObjectNumUses) *
4403 static_cast<uint64_t>(A.ObjectSize);
4404
4405 // If the two densities are equal, prioritize highest alignment
4406 // objects. This allows for similar alignment objects
4407 // to be packed together (given the same density).
4408 // There's room for improvement here, also, since we can pack
4409 // similar alignment (different density) objects next to each
4410 // other to save padding. This will also require further
4411 // complexity/iterations, and the overall gain isn't worth it,
4412 // in general. Something to keep in mind, though.
4413 if (DensityAScaled == DensityBScaled)
4414 return A.ObjectAlignment < B.ObjectAlignment;
4415
4416 return DensityAScaled < DensityBScaled;
4417 }
4418};
4419} // namespace
4420
4421// Order the symbols in the local stack.
4422// We want to place the local stack objects in some sort of sensible order.
4423// The heuristic we use is to try and pack them according to static number
4424// of uses and size of object in order to minimize code size.
4425void X86FrameLowering::orderFrameObjects(
4426 const MachineFunction &MF, SmallVectorImpl<int> &ObjectsToAllocate) const {
4427 const MachineFrameInfo &MFI = MF.getFrameInfo();
4428
4429 // Don't waste time if there's nothing to do.
4430 if (ObjectsToAllocate.empty())
4431 return;
4432
4433 // Create an array of all MFI objects. We won't need all of these
4434 // objects, but we're going to create a full array of them to make
4435 // it easier to index into when we're counting "uses" down below.
4436 // We want to be able to easily/cheaply access an object by simply
4437 // indexing into it, instead of having to search for it every time.
4438 std::vector<X86FrameSortingObject> SortingObjects(MFI.getObjectIndexEnd());
4439
4440 // Walk the objects we care about and mark them as such in our working
4441 // struct.
4442 for (auto &Obj : ObjectsToAllocate) {
4443 SortingObjects[Obj].IsValid = true;
4444 SortingObjects[Obj].ObjectIndex = Obj;
4445 SortingObjects[Obj].ObjectAlignment = MFI.getObjectAlign(ObjectIdx: Obj);
4446 // Set the size.
4447 int ObjectSize = MFI.getObjectSize(ObjectIdx: Obj);
4448 if (ObjectSize == 0)
4449 // Variable size. Just use 4.
4450 SortingObjects[Obj].ObjectSize = 4;
4451 else
4452 SortingObjects[Obj].ObjectSize = ObjectSize;
4453 }
4454
4455 // Count the number of uses for each object.
4456 for (auto &MBB : MF) {
4457 for (auto &MI : MBB) {
4458 if (MI.isDebugInstr())
4459 continue;
4460 for (const MachineOperand &MO : MI.operands()) {
4461 // Check to see if it's a local stack symbol.
4462 if (!MO.isFI())
4463 continue;
4464 int Index = MO.getIndex();
4465 // Check to see if it falls within our range, and is tagged
4466 // to require ordering.
4467 if (Index >= 0 && Index < MFI.getObjectIndexEnd() &&
4468 SortingObjects[Index].IsValid)
4469 SortingObjects[Index].ObjectNumUses++;
4470 }
4471 }
4472 }
4473
4474 // Sort the objects using X86FrameSortingAlgorithm (see its comment for
4475 // info).
4476 llvm::stable_sort(Range&: SortingObjects, C: X86FrameSortingComparator());
4477
4478 // Now modify the original list to represent the final order that
4479 // we want. The order will depend on whether we're going to access them
4480 // from the stack pointer or the frame pointer. For SP, the list should
4481 // end up with the END containing objects that we want with smaller offsets.
4482 // For FP, it should be flipped.
4483 int i = 0;
4484 for (auto &Obj : SortingObjects) {
4485 // All invalid items are sorted at the end, so it's safe to stop.
4486 if (!Obj.IsValid)
4487 break;
4488 ObjectsToAllocate[i++] = Obj.ObjectIndex;
4489 }
4490
4491 // Flip it if we're accessing off of the FP.
4492 if (!TRI->hasStackRealignment(MF) && hasFP(MF))
4493 std::reverse(first: ObjectsToAllocate.begin(), last: ObjectsToAllocate.end());
4494}
4495
4496unsigned
4497X86FrameLowering::getWinEHParentFrameOffset(const MachineFunction &MF) const {
4498 // RDX, the parent frame pointer, is homed into 16(%rsp) in the prologue.
4499 unsigned Offset = 16;
4500 // RBP is immediately pushed.
4501 Offset += SlotSize;
4502 // All callee-saved registers are then pushed.
4503 Offset += MF.getInfo<X86MachineFunctionInfo>()->getCalleeSavedFrameSize();
4504 // Every funclet allocates enough stack space for the largest outgoing call.
4505 Offset += getWinEHFuncletFrameSize(MF);
4506 return Offset;
4507}
4508
4509void X86FrameLowering::processFunctionBeforeFrameFinalized(
4510 MachineFunction &MF, RegScavenger *RS) const {
4511 // Mark the function as not having WinCFI. We will set it back to true in
4512 // emitPrologue if it gets called and emits CFI.
4513 MF.setHasWinCFI(false);
4514
4515 MachineFrameInfo &MFI = MF.getFrameInfo();
4516 // If the frame is big enough that we might need to scavenge a register to
4517 // handle huge offsets, reserve a stack slot for that now.
4518 if (!isInt<32>(x: MFI.estimateStackSize(MF))) {
4519 int FI = MFI.CreateStackObject(Size: SlotSize, Alignment: Align(SlotSize), isSpillSlot: false);
4520 RS->addScavengingFrameIndex(FI);
4521 }
4522
4523 // If we are using Windows x64 CFI, ensure that the stack is always 8 byte
4524 // aligned. The format doesn't support misaligned stack adjustments.
4525 if (MF.usesWindowsUnwindInfo())
4526 MF.getFrameInfo().ensureMaxAlignment(Alignment: Align(SlotSize));
4527
4528 // If this function isn't doing Win64-style C++ EH, we don't need to do
4529 // anything.
4530 if (STI.is64Bit() && MF.hasEHFunclets() &&
4531 classifyEHPersonality(Pers: MF.getFunction().getPersonalityFn()) ==
4532 EHPersonality::MSVC_CXX) {
4533 adjustFrameForMsvcCxxEh(MF);
4534 }
4535}
4536
4537void X86FrameLowering::adjustFrameForMsvcCxxEh(MachineFunction &MF) const {
4538 // Win64 C++ EH needs to allocate the UnwindHelp object at some fixed offset
4539 // relative to RSP after the prologue. Find the offset of the last fixed
4540 // object, so that we can allocate a slot immediately following it. If there
4541 // were no fixed objects, use offset -SlotSize, which is immediately after the
4542 // return address. Fixed objects have negative frame indices.
4543 MachineFrameInfo &MFI = MF.getFrameInfo();
4544 WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo();
4545 int64_t MinFixedObjOffset = -SlotSize;
4546 for (int I = MFI.getObjectIndexBegin(); I < 0; ++I)
4547 MinFixedObjOffset = std::min(a: MinFixedObjOffset, b: MFI.getObjectOffset(ObjectIdx: I));
4548
4549 for (WinEHTryBlockMapEntry &TBME : EHInfo.TryBlockMap) {
4550 for (WinEHHandlerType &H : TBME.HandlerArray) {
4551 int FrameIndex = H.CatchObj.FrameIndex;
4552 if ((FrameIndex != INT_MAX) && MFI.getObjectOffset(ObjectIdx: FrameIndex) == 0) {
4553 // Ensure alignment.
4554 unsigned Align = MFI.getObjectAlign(ObjectIdx: FrameIndex).value();
4555 MinFixedObjOffset -= std::abs(i: MinFixedObjOffset) % Align;
4556 MinFixedObjOffset -= MFI.getObjectSize(ObjectIdx: FrameIndex);
4557 MFI.setObjectOffset(ObjectIdx: FrameIndex, SPOffset: MinFixedObjOffset);
4558 }
4559 }
4560 }
4561
4562 // Ensure alignment.
4563 MinFixedObjOffset -= std::abs(i: MinFixedObjOffset) % 8;
4564 int64_t UnwindHelpOffset = MinFixedObjOffset - SlotSize;
4565 int UnwindHelpFI =
4566 MFI.CreateFixedObject(Size: SlotSize, SPOffset: UnwindHelpOffset, /*IsImmutable=*/false);
4567 EHInfo.UnwindHelpFrameIdx = UnwindHelpFI;
4568
4569 // Store -2 into UnwindHelp on function entry. We have to scan forwards past
4570 // other frame setup instructions.
4571 MachineBasicBlock &MBB = MF.front();
4572 auto MBBI = MBB.begin();
4573 while (MBBI != MBB.end() && MBBI->getFlag(Flag: MachineInstr::FrameSetup))
4574 ++MBBI;
4575
4576 DebugLoc DL = MBB.findDebugLoc(MBBI);
4577 addFrameReference(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64mi32)),
4578 FI: UnwindHelpFI)
4579 .addImm(Val: -2);
4580}
4581
4582void X86FrameLowering::processFunctionBeforeFrameIndicesReplaced(
4583 MachineFunction &MF, RegScavenger *RS) const {
4584 auto *X86FI = MF.getInfo<X86MachineFunctionInfo>();
4585
4586 if (STI.is32Bit() && MF.hasEHFunclets())
4587 restoreWinEHStackPointersInParent(MF);
4588 // We have emitted prolog and epilog. Don't need stack pointer saving
4589 // instruction any more.
4590 if (MachineInstr *MI = X86FI->getStackPtrSaveMI()) {
4591 MI->eraseFromParent();
4592 X86FI->setStackPtrSaveMI(nullptr);
4593 }
4594}
4595
4596void X86FrameLowering::restoreWinEHStackPointersInParent(
4597 MachineFunction &MF) const {
4598 // 32-bit functions have to restore stack pointers when control is transferred
4599 // back to the parent function. These blocks are identified as eh pads that
4600 // are not funclet entries.
4601 bool IsSEH = isAsynchronousEHPersonality(
4602 Pers: classifyEHPersonality(Pers: MF.getFunction().getPersonalityFn()));
4603 for (MachineBasicBlock &MBB : MF) {
4604 bool NeedsRestore = MBB.isEHPad() && !MBB.isEHFuncletEntry();
4605 if (NeedsRestore)
4606 restoreWin32EHStackPointers(MBB, MBBI: MBB.begin(), DL: DebugLoc(),
4607 /*RestoreSP=*/IsSEH);
4608 }
4609}
4610
4611// Compute the alignment gap between current SP after spilling FP/BP and the
4612// next properly aligned stack offset.
4613static int computeFPBPAlignmentGap(MachineFunction &MF,
4614 const TargetRegisterClass *RC,
4615 unsigned NumSpilledRegs) {
4616 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
4617 unsigned AllocSize = TRI->getSpillSize(RC: *RC) * NumSpilledRegs;
4618 Align StackAlign = MF.getSubtarget().getFrameLowering()->getStackAlign();
4619 unsigned AlignedSize = alignTo(Size: AllocSize, A: StackAlign);
4620 return AlignedSize - AllocSize;
4621}
4622
4623void X86FrameLowering::spillFPBPUsingSP(MachineFunction &MF,
4624 MachineBasicBlock::iterator BeforeMI,
4625 Register FP, Register BP,
4626 int SPAdjust) const {
4627 assert(FP.isValid() || BP.isValid());
4628
4629 MachineBasicBlock *MBB = BeforeMI->getParent();
4630 DebugLoc DL = BeforeMI->getDebugLoc();
4631
4632 // Spill FP.
4633 if (FP.isValid()) {
4634 BuildMI(BB&: *MBB, I: BeforeMI, MIMD: DL,
4635 MCID: TII.get(Opcode: getPUSHOpcode(ST: MF.getSubtarget<X86Subtarget>())))
4636 .addReg(RegNo: FP);
4637 }
4638
4639 // Spill BP.
4640 if (BP.isValid()) {
4641 BuildMI(BB&: *MBB, I: BeforeMI, MIMD: DL,
4642 MCID: TII.get(Opcode: getPUSHOpcode(ST: MF.getSubtarget<X86Subtarget>())))
4643 .addReg(RegNo: BP);
4644 }
4645
4646 // Make sure SP is aligned.
4647 if (SPAdjust)
4648 emitSPUpdate(MBB&: *MBB, MBBI&: BeforeMI, DL, NumBytes: -SPAdjust, InEpilogue: false);
4649
4650 // Emit unwinding information.
4651 if (FP.isValid() && needsDwarfCFI(MF)) {
4652 // Emit .cfi_remember_state to remember old frame.
4653 unsigned CFIIndex =
4654 MF.addFrameInst(Inst: MCCFIInstruction::createRememberState(L: nullptr));
4655 BuildMI(BB&: *MBB, I: BeforeMI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::CFI_INSTRUCTION))
4656 .addCFIIndex(CFIIndex);
4657
4658 // Setup new CFA value with DW_CFA_def_cfa_expression:
4659 // DW_OP_breg7+offset, DW_OP_deref, DW_OP_consts 16, DW_OP_plus
4660 SmallString<64> CfaExpr;
4661 uint8_t buffer[16];
4662 int Offset = SPAdjust;
4663 if (BP.isValid())
4664 Offset += TRI->getSpillSize(RC: *TRI->getMinimalPhysRegClass(Reg: BP));
4665 // If BeforeMI is a frame setup instruction, we need to adjust the position
4666 // and offset of the new cfi instruction.
4667 if (TII.isFrameSetup(I: *BeforeMI)) {
4668 Offset += alignTo(Size: TII.getFrameSize(I: *BeforeMI), A: getStackAlign());
4669 BeforeMI = std::next(x: BeforeMI);
4670 }
4671 Register StackPtr = TRI->getStackRegister();
4672 if (STI.isTarget64BitILP32())
4673 StackPtr = Register(getX86SubSuperRegister(Reg: StackPtr, Size: 64));
4674 unsigned DwarfStackPtr = TRI->getDwarfRegNum(Reg: StackPtr, isEH: true);
4675 CfaExpr.push_back(Elt: (uint8_t)(dwarf::DW_OP_breg0 + DwarfStackPtr));
4676 CfaExpr.append(in_start: buffer, in_end: buffer + encodeSLEB128(Value: Offset, p: buffer));
4677 CfaExpr.push_back(Elt: dwarf::DW_OP_deref);
4678 CfaExpr.push_back(Elt: dwarf::DW_OP_consts);
4679 CfaExpr.append(in_start: buffer, in_end: buffer + encodeSLEB128(Value: SlotSize * 2, p: buffer));
4680 CfaExpr.push_back(Elt: (uint8_t)dwarf::DW_OP_plus);
4681
4682 SmallString<64> DefCfaExpr;
4683 DefCfaExpr.push_back(Elt: dwarf::DW_CFA_def_cfa_expression);
4684 DefCfaExpr.append(in_start: buffer, in_end: buffer + encodeSLEB128(Value: CfaExpr.size(), p: buffer));
4685 DefCfaExpr.append(RHS: CfaExpr.str());
4686 BuildCFI(MBB&: *MBB, MBBI: BeforeMI, DL,
4687 CFIInst: MCCFIInstruction::createEscape(L: nullptr, Vals: DefCfaExpr.str()),
4688 Flag: MachineInstr::FrameSetup);
4689 }
4690}
4691
4692void X86FrameLowering::restoreFPBPUsingSP(MachineFunction &MF,
4693 MachineBasicBlock::iterator AfterMI,
4694 Register FP, Register BP,
4695 int SPAdjust) const {
4696 assert(FP.isValid() || BP.isValid());
4697
4698 // Adjust SP so it points to spilled FP or BP.
4699 MachineBasicBlock *MBB = AfterMI->getParent();
4700 MachineBasicBlock::iterator Pos = std::next(x: AfterMI);
4701 DebugLoc DL = AfterMI->getDebugLoc();
4702 if (SPAdjust)
4703 emitSPUpdate(MBB&: *MBB, MBBI&: Pos, DL, NumBytes: SPAdjust, InEpilogue: false);
4704
4705 // Restore BP.
4706 if (BP.isValid()) {
4707 BuildMI(BB&: *MBB, I: Pos, MIMD: DL,
4708 MCID: TII.get(Opcode: getPOPOpcode(ST: MF.getSubtarget<X86Subtarget>())), DestReg: BP);
4709 }
4710
4711 // Restore FP.
4712 if (FP.isValid()) {
4713 BuildMI(BB&: *MBB, I: Pos, MIMD: DL,
4714 MCID: TII.get(Opcode: getPOPOpcode(ST: MF.getSubtarget<X86Subtarget>())), DestReg: FP);
4715
4716 // Emit unwinding information.
4717 if (needsDwarfCFI(MF)) {
4718 // Restore original frame with .cfi_restore_state.
4719 unsigned CFIIndex =
4720 MF.addFrameInst(Inst: MCCFIInstruction::createRestoreState(L: nullptr));
4721 BuildMI(BB&: *MBB, I: Pos, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::CFI_INSTRUCTION))
4722 .addCFIIndex(CFIIndex);
4723 }
4724 }
4725}
4726
4727void X86FrameLowering::saveAndRestoreFPBPUsingSP(
4728 MachineFunction &MF, MachineBasicBlock::iterator BeforeMI,
4729 MachineBasicBlock::iterator AfterMI, bool SpillFP, bool SpillBP) const {
4730 assert(SpillFP || SpillBP);
4731
4732 Register FP, BP;
4733 const TargetRegisterClass *RC;
4734 unsigned NumRegs = 0;
4735
4736 if (SpillFP) {
4737 FP = TRI->getFrameRegister(MF);
4738 if (STI.isTarget64BitILP32())
4739 FP = Register(getX86SubSuperRegister(Reg: FP, Size: 64));
4740 RC = TRI->getMinimalPhysRegClass(Reg: FP);
4741 ++NumRegs;
4742 }
4743 if (SpillBP) {
4744 BP = TRI->getBaseRegister();
4745 if (STI.isTarget64BitILP32())
4746 BP = Register(getX86SubSuperRegister(Reg: BP, Size: 64));
4747 RC = TRI->getMinimalPhysRegClass(Reg: BP);
4748 ++NumRegs;
4749 }
4750 int SPAdjust = computeFPBPAlignmentGap(MF, RC, NumSpilledRegs: NumRegs);
4751
4752 spillFPBPUsingSP(MF, BeforeMI, FP, BP, SPAdjust);
4753 restoreFPBPUsingSP(MF, AfterMI, FP, BP, SPAdjust);
4754}
4755
4756bool X86FrameLowering::skipSpillFPBP(
4757 MachineFunction &MF, MachineBasicBlock::reverse_iterator &MI) const {
4758 if (MI->getOpcode() == X86::LCMPXCHG16B_SAVE_RBX) {
4759 // The pseudo instruction LCMPXCHG16B_SAVE_RBX is generated in the form
4760 // SaveRbx = COPY RBX
4761 // SaveRbx = LCMPXCHG16B_SAVE_RBX ..., SaveRbx, implicit-def rbx
4762 // And later LCMPXCHG16B_SAVE_RBX is expanded to restore RBX from SaveRbx.
4763 // We should skip this instruction sequence.
4764 int FI;
4765 Register Reg;
4766 while (!(MI->getOpcode() == TargetOpcode::COPY &&
4767 MI->getOperand(i: 1).getReg() == X86::RBX) &&
4768 !((Reg = TII.isStoreToStackSlot(MI: *MI, FrameIndex&: FI)) && Reg == X86::RBX))
4769 ++MI;
4770 return true;
4771 }
4772 return false;
4773}
4774
4775static bool isFPBPAccess(const MachineInstr &MI, Register FP, Register BP,
4776 const TargetRegisterInfo *TRI, bool &AccessFP,
4777 bool &AccessBP) {
4778 AccessFP = AccessBP = false;
4779 if (FP) {
4780 if (MI.findRegisterUseOperandIdx(Reg: FP, TRI, isKill: false) != -1 ||
4781 MI.findRegisterDefOperandIdx(Reg: FP, TRI, isDead: false, Overlap: true) != -1)
4782 AccessFP = true;
4783 }
4784 if (BP) {
4785 if (MI.findRegisterUseOperandIdx(Reg: BP, TRI, isKill: false) != -1 ||
4786 MI.findRegisterDefOperandIdx(Reg: BP, TRI, isDead: false, Overlap: true) != -1)
4787 AccessBP = true;
4788 }
4789 return AccessFP || AccessBP;
4790}
4791
4792// Invoke instruction has been lowered to normal function call. We try to figure
4793// out if MI comes from Invoke.
4794// Do we have any better method?
4795static bool isInvoke(const MachineInstr &MI, bool InsideEHLabels) {
4796 if (!MI.isCall())
4797 return false;
4798 if (InsideEHLabels)
4799 return true;
4800
4801 const MachineBasicBlock *MBB = MI.getParent();
4802 if (!MBB->hasEHPadSuccessor())
4803 return false;
4804
4805 // Check if there is another call instruction from MI to the end of MBB.
4806 MachineBasicBlock::const_iterator MBBI = MI, ME = MBB->end();
4807 for (++MBBI; MBBI != ME; ++MBBI)
4808 if (MBBI->isCall())
4809 return false;
4810 return true;
4811}
4812
4813/// Given the live range of FP or BP (DefMI, KillMI), check if there is any
4814/// interfered stack access in the range, usually generated by register spill.
4815void X86FrameLowering::checkInterferedAccess(
4816 MachineFunction &MF, MachineBasicBlock::reverse_iterator DefMI,
4817 MachineBasicBlock::reverse_iterator KillMI, bool SpillFP,
4818 bool SpillBP) const {
4819 if (DefMI == KillMI)
4820 return;
4821 if (TRI->hasBasePointer(MF)) {
4822 if (!SpillBP)
4823 return;
4824 } else {
4825 if (!SpillFP)
4826 return;
4827 }
4828
4829 auto MI = KillMI;
4830 while (MI != DefMI) {
4831 if (any_of(Range: MI->operands(),
4832 P: [](const MachineOperand &MO) { return MO.isFI(); }))
4833 MF.getContext().reportError(L: SMLoc(),
4834 Msg: "Interference usage of base pointer/frame "
4835 "pointer.");
4836 MI++;
4837 }
4838}
4839
4840/// If a function uses base pointer and the base pointer is clobbered by inline
4841/// asm, RA doesn't detect this case, and after the inline asm, the base pointer
4842/// contains garbage value.
4843/// For example if a 32b x86 function uses base pointer esi, and esi is
4844/// clobbered by following inline asm
4845/// asm("rep movsb" : "+D"(ptr), "+S"(x), "+c"(c)::"memory");
4846/// We need to save esi before the asm and restore it after the asm.
4847///
4848/// The problem can also occur to frame pointer if there is a function call, and
4849/// the callee uses a different calling convention and clobbers the fp.
4850///
4851/// Because normal frame objects (spill slots) are accessed through fp/bp
4852/// register, so we can't spill fp/bp to normal spill slots.
4853///
4854/// FIXME: There are 2 possible enhancements:
4855/// 1. In many cases there are different physical registers not clobbered by
4856/// inline asm, we can use one of them as base pointer. Or use a virtual
4857/// register as base pointer and let RA allocate a physical register to it.
4858/// 2. If there is no other instructions access stack with fp/bp from the
4859/// inline asm to the epilog, and no cfi requirement for a correct fp, we can
4860/// skip the save and restore operations.
4861void X86FrameLowering::spillFPBP(MachineFunction &MF) const {
4862 Register FP, BP;
4863 const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering();
4864 if (TFI.hasFP(MF))
4865 FP = TRI->getFrameRegister(MF);
4866 if (TRI->hasBasePointer(MF))
4867 BP = TRI->getBaseRegister();
4868
4869 // Currently only inline asm and function call can clobbers fp/bp. So we can
4870 // do some quick test and return early.
4871 if (!MF.hasInlineAsm()) {
4872 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
4873 if (!X86FI->getFPClobberedByCall())
4874 FP = 0;
4875 if (!X86FI->getBPClobberedByCall())
4876 BP = 0;
4877 }
4878 if (!FP && !BP)
4879 return;
4880
4881 for (MachineBasicBlock &MBB : MF) {
4882 bool InsideEHLabels = false;
4883 auto MI = MBB.rbegin(), ME = MBB.rend();
4884 auto TermMI = MBB.getFirstTerminator();
4885 if (TermMI == MBB.begin())
4886 continue;
4887 MI = *(std::prev(x: TermMI));
4888
4889 while (MI != ME) {
4890 // Skip frame setup/destroy instructions.
4891 // Skip Invoke (call inside try block) instructions.
4892 // Skip instructions handled by target.
4893 if (MI->getFlag(Flag: MachineInstr::MIFlag::FrameSetup) ||
4894 MI->getFlag(Flag: MachineInstr::MIFlag::FrameDestroy) ||
4895 isInvoke(MI: *MI, InsideEHLabels) || skipSpillFPBP(MF, MI)) {
4896 ++MI;
4897 continue;
4898 }
4899
4900 if (MI->getOpcode() == TargetOpcode::EH_LABEL) {
4901 InsideEHLabels = !InsideEHLabels;
4902 ++MI;
4903 continue;
4904 }
4905
4906 bool AccessFP, AccessBP;
4907 // Check if fp or bp is used in MI.
4908 if (!isFPBPAccess(MI: *MI, FP, BP, TRI, AccessFP, AccessBP)) {
4909 ++MI;
4910 continue;
4911 }
4912
4913 // Look for the range [DefMI, KillMI] in which fp or bp is defined and
4914 // used.
4915 bool FPLive = false, BPLive = false;
4916 bool SpillFP = false, SpillBP = false;
4917 auto DefMI = MI, KillMI = MI;
4918 do {
4919 SpillFP |= AccessFP;
4920 SpillBP |= AccessBP;
4921
4922 // Maintain FPLive and BPLive.
4923 if (FPLive && MI->findRegisterDefOperandIdx(Reg: FP, TRI, isDead: false, Overlap: true) != -1)
4924 FPLive = false;
4925 if (FP && MI->findRegisterUseOperandIdx(Reg: FP, TRI, isKill: false) != -1)
4926 FPLive = true;
4927 if (BPLive && MI->findRegisterDefOperandIdx(Reg: BP, TRI, isDead: false, Overlap: true) != -1)
4928 BPLive = false;
4929 if (BP && MI->findRegisterUseOperandIdx(Reg: BP, TRI, isKill: false) != -1)
4930 BPLive = true;
4931
4932 DefMI = MI++;
4933 } while ((MI != ME) &&
4934 (FPLive || BPLive ||
4935 isFPBPAccess(MI: *MI, FP, BP, TRI, AccessFP, AccessBP)));
4936
4937 // Don't need to save/restore if FP is accessed through llvm.frameaddress.
4938 if (FPLive && !SpillBP)
4939 continue;
4940
4941 // If the bp is clobbered by a call, we should save and restore outside of
4942 // the frame setup instructions.
4943 if (KillMI->isCall() && DefMI != ME) {
4944 auto FrameSetup = std::next(x: DefMI);
4945 // Look for frame setup instruction toward the start of the BB.
4946 // If we reach another call instruction, it means no frame setup
4947 // instruction for the current call instruction.
4948 while (FrameSetup != ME && !TII.isFrameSetup(I: *FrameSetup) &&
4949 !FrameSetup->isCall())
4950 ++FrameSetup;
4951 // If a frame setup instruction is found, we need to find out the
4952 // corresponding frame destroy instruction.
4953 if (FrameSetup != ME && TII.isFrameSetup(I: *FrameSetup) &&
4954 (TII.getFrameSize(I: *FrameSetup) ||
4955 TII.getFrameAdjustment(I: *FrameSetup))) {
4956 while (!TII.isFrameInstr(I: *KillMI))
4957 --KillMI;
4958 DefMI = FrameSetup;
4959 MI = DefMI;
4960 ++MI;
4961 }
4962 }
4963
4964 checkInterferedAccess(MF, DefMI, KillMI, SpillFP, SpillBP);
4965
4966 // Call target function to spill and restore FP and BP registers.
4967 saveAndRestoreFPBPUsingSP(MF, BeforeMI: &(*DefMI), AfterMI: &(*KillMI), SpillFP, SpillBP);
4968 }
4969 }
4970}
4971