1//===-- X86FrameLowering.cpp - X86 Frame Information ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the X86 implementation of TargetFrameLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "X86FrameLowering.h"
14#include "MCTargetDesc/X86MCTargetDesc.h"
15#include "X86.h"
16#include "X86InstrBuilder.h"
17#include "X86InstrInfo.h"
18#include "X86MachineFunctionInfo.h"
19#include "X86Subtarget.h"
20#include "X86TargetMachine.h"
21#include "llvm/ADT/Statistic.h"
22#include "llvm/CodeGen/LivePhysRegs.h"
23#include "llvm/CodeGen/MachineFrameInfo.h"
24#include "llvm/CodeGen/MachineFunction.h"
25#include "llvm/CodeGen/MachineInstrBuilder.h"
26#include "llvm/CodeGen/MachineModuleInfo.h"
27#include "llvm/CodeGen/MachineRegisterInfo.h"
28#include "llvm/CodeGen/RegisterScavenging.h"
29#include "llvm/CodeGen/WinEHFuncInfo.h"
30#include "llvm/IR/DataLayout.h"
31#include "llvm/IR/EHPersonalities.h"
32#include "llvm/IR/Function.h"
33#include "llvm/IR/Module.h"
34#include "llvm/MC/MCAsmInfo.h"
35#include "llvm/MC/MCObjectFileInfo.h"
36#include "llvm/MC/MCSymbol.h"
37#include "llvm/Support/LEB128.h"
38#include "llvm/Target/TargetOptions.h"
39#include <cstdlib>
40
41#define DEBUG_TYPE "x86-fl"
42
43STATISTIC(NumFrameLoopProbe, "Number of loop stack probes used in prologue");
44STATISTIC(NumFrameExtraProbe,
45 "Number of extra stack probes generated in prologue");
46STATISTIC(NumFunctionUsingPush2Pop2, "Number of functions using push2/pop2");
47
48using namespace llvm;
49
50bool llvm::requireWinX64UnwindV3(const MachineFunction &MF) {
51 const Function &Fn = MF.getFunction();
52
53 // Whole module is in V3 mode.
54 if (Fn.getParent()->getWinX64EHUnwindMode() == WinX64EHUnwindMode::V3)
55 return true;
56
57 // Otherwise promote a function that may use EGPR (R16-R31), which V1/V2
58 // unwind codes cannot encode. The per-function "+egpr" feature is the signal,
59 // so an auto-dispatch APX clone gets V3 while the baseline clone stays on the
60 // module default. We conservatively promote any egpr function rather than
61 // checking for an actual EGPR save, keeping this a cheap query. (PUSH2/POP2
62 // does not need V3: V1/V2 describe a PUSH2 as two SEH_PushReg codes.)
63 return Fn.needsUnwindTableEntry() &&
64 MF.getSubtarget<X86Subtarget>().hasEGPR();
65}
66
67static const TargetRegisterClass *
68getCalleeSavedSpillRC(MCRegister Reg, const X86Subtarget &STI,
69 const TargetRegisterInfo &TRI) {
70 if (X86::VK16RegClass.contains(Reg))
71 return STI.hasBWI() ? &X86::VK64RegClass : &X86::VK16RegClass;
72 return TRI.getMinimalPhysRegClass(Reg);
73}
74
75X86FrameLowering::X86FrameLowering(const X86Subtarget &STI,
76 MaybeAlign StackAlignOverride)
77 : TargetFrameLowering(StackGrowsDown, StackAlignOverride.valueOrOne(),
78 STI.is64Bit() ? -8 : -4),
79 STI(STI), TII(*STI.getInstrInfo()), TRI(STI.getRegisterInfo()) {
80 // Cache a bunch of frame-related predicates for this subtarget.
81 SlotSize = TRI->getSlotSize();
82 assert(SlotSize == 4 || SlotSize == 8);
83 Is64Bit = STI.is64Bit();
84 IsLP64 = STI.isTarget64BitLP64();
85 // standard x86_64 uses 64-bit frame/stack pointers, x32 - 32-bit.
86 Uses64BitFramePtr = STI.isTarget64BitLP64();
87 StackPtr = TRI->getStackRegister();
88}
89
90bool X86FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
91 return !MF.getFrameInfo().hasVarSizedObjects() &&
92 !MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences() &&
93 !MF.getInfo<X86MachineFunctionInfo>()->hasPreallocatedCall();
94}
95
96/// canSimplifyCallFramePseudos - If there is a reserved call frame, the
97/// call frame pseudos can be simplified. Having a FP, as in the default
98/// implementation, is not sufficient here since we can't always use it.
99/// Use a more nuanced condition.
100bool X86FrameLowering::canSimplifyCallFramePseudos(
101 const MachineFunction &MF) const {
102 return hasReservedCallFrame(MF) ||
103 MF.getInfo<X86MachineFunctionInfo>()->hasPreallocatedCall() ||
104 (hasFP(MF) && !TRI->hasStackRealignment(MF)) ||
105 TRI->hasBasePointer(MF);
106}
107
108// needsFrameIndexResolution - Do we need to perform FI resolution for
109// this function. Normally, this is required only when the function
110// has any stack objects. However, FI resolution actually has another job,
111// not apparent from the title - it resolves callframesetup/destroy
112// that were not simplified earlier.
113// So, this is required for x86 functions that have push sequences even
114// when there are no stack objects.
115bool X86FrameLowering::needsFrameIndexResolution(
116 const MachineFunction &MF) const {
117 return MF.getFrameInfo().hasStackObjects() ||
118 MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences();
119}
120
121/// hasFPImpl - Return true if the specified function should have a dedicated
122/// frame pointer register. This is true if the function has variable sized
123/// allocas or if frame pointer elimination is disabled.
124bool X86FrameLowering::hasFPImpl(const MachineFunction &MF) const {
125 const MachineFrameInfo &MFI = MF.getFrameInfo();
126 return (MF.disableFramePointerElim() || TRI->hasStackRealignment(MF) ||
127 MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken() ||
128 MFI.hasOpaqueSPAdjustment() ||
129 MF.getInfo<X86MachineFunctionInfo>()->getForceFramePointer() ||
130 MF.getInfo<X86MachineFunctionInfo>()->hasPreallocatedCall() ||
131 MF.callsUnwindInit() || MF.hasEHFunclets() || MF.callsEHReturn() ||
132 MFI.hasStackMap() || MFI.hasPatchPoint() ||
133 (MFI.hasCopyImplyingStackAdjustment() && MF.usesWindowsUnwindInfo()));
134}
135
136static unsigned getSUBriOpcode(bool IsLP64) {
137 return IsLP64 ? X86::SUB64ri32 : X86::SUB32ri;
138}
139
140static unsigned getADDriOpcode(bool IsLP64) {
141 return IsLP64 ? X86::ADD64ri32 : X86::ADD32ri;
142}
143
144static unsigned getSUBrrOpcode(bool IsLP64) {
145 return IsLP64 ? X86::SUB64rr : X86::SUB32rr;
146}
147
148static unsigned getADDrrOpcode(bool IsLP64) {
149 return IsLP64 ? X86::ADD64rr : X86::ADD32rr;
150}
151
152static unsigned getANDriOpcode(bool IsLP64, int64_t Imm) {
153 return IsLP64 ? X86::AND64ri32 : X86::AND32ri;
154}
155
156static unsigned getLEArOpcode(bool IsLP64) {
157 return IsLP64 ? X86::LEA64r : X86::LEA32r;
158}
159
160// Push-Pop Acceleration (PPX) hint is used to indicate that the POP reads the
161// value written by the PUSH from the stack. The processor tracks these marked
162// instructions internally and fast-forwards register data between matching PUSH
163// and POP instructions, without going through memory or through the training
164// loop of the Fast Store Forwarding Predictor (FSFP). Instead, a more efficient
165// memory-renaming optimization can be used.
166//
167// The PPX hint is purely a performance hint. Instructions with this hint have
168// the same functional semantics as those without. PPX hints set by the
169// compiler that violate the balancing rule may turn off the PPX optimization,
170// but they will not affect program semantics.
171//
172// Hence, PPX is used for balanced spill/reloads (Exceptions and setjmp/longjmp
173// are not considered).
174//
175// PUSH2 and POP2 are instructions for (respectively) pushing/popping 2
176// GPRs at a time to/from the stack.
177static unsigned getPUSHOpcode(const X86Subtarget &ST) {
178 return ST.is64Bit() ? (ST.hasPPX() ? X86::PUSHP64r : X86::PUSH64r)
179 : X86::PUSH32r;
180}
181static unsigned getPOPOpcode(const X86Subtarget &ST) {
182 return ST.is64Bit() ? (ST.hasPPX() ? X86::POPP64r : X86::POP64r)
183 : X86::POP32r;
184}
185static unsigned getPUSH2Opcode(const X86Subtarget &ST) {
186 return ST.hasPPX() ? X86::PUSH2P : X86::PUSH2;
187}
188static unsigned getPOP2Opcode(const X86Subtarget &ST) {
189 return ST.hasPPX() ? X86::POP2P : X86::POP2;
190}
191
192static bool isEAXLiveIn(MachineBasicBlock &MBB) {
193 for (MachineBasicBlock::RegisterMaskPair RegMask : MBB.liveins()) {
194 MCRegister Reg = RegMask.PhysReg;
195
196 if (Reg == X86::RAX || Reg == X86::EAX || Reg == X86::AX ||
197 Reg == X86::AH || Reg == X86::AL)
198 return true;
199 }
200
201 return false;
202}
203
204/// Check if the flags need to be preserved before the terminators.
205/// This would be the case, if the eflags is live-in of the region
206/// composed by the terminators or live-out of that region, without
207/// being defined by a terminator.
208static bool
209flagsNeedToBePreservedBeforeTheTerminators(const MachineBasicBlock &MBB) {
210 for (const MachineInstr &MI : MBB.terminators()) {
211 bool BreakNext = false;
212 for (const MachineOperand &MO : MI.operands()) {
213 if (!MO.isReg())
214 continue;
215 Register Reg = MO.getReg();
216 if (Reg != X86::EFLAGS)
217 continue;
218
219 // This terminator needs an eflags that is not defined
220 // by a previous another terminator:
221 // EFLAGS is live-in of the region composed by the terminators.
222 if (!MO.isDef())
223 return true;
224 // This terminator defines the eflags, i.e., we don't need to preserve it.
225 // However, we still need to check this specific terminator does not
226 // read a live-in value.
227 BreakNext = true;
228 }
229 // We found a definition of the eflags, no need to preserve them.
230 if (BreakNext)
231 return false;
232 }
233
234 // None of the terminators use or define the eflags.
235 // Check if they are live-out, that would imply we need to preserve them.
236 for (const MachineBasicBlock *Succ : MBB.successors())
237 if (Succ->isLiveIn(Reg: X86::EFLAGS))
238 return true;
239
240 return false;
241}
242
243constexpr uint64_t MaxSPChunk = (1ULL << 31) - 1;
244
245/// emitSPUpdate - Emit a series of instructions to increment / decrement the
246/// stack pointer by a constant value.
247void X86FrameLowering::emitSPUpdate(MachineBasicBlock &MBB,
248 MachineBasicBlock::iterator &MBBI,
249 const DebugLoc &DL, int64_t NumBytes,
250 bool InEpilogue) const {
251 bool isSub = NumBytes < 0;
252 uint64_t Offset = isSub ? -NumBytes : NumBytes;
253 MachineInstr::MIFlag Flag =
254 isSub ? MachineInstr::FrameSetup : MachineInstr::FrameDestroy;
255
256 if (!Uses64BitFramePtr && !isUInt<32>(x: Offset)) {
257 // We're being asked to adjust a 32-bit stack pointer by 4 GiB or more.
258 // This might be unreachable code, so don't complain now; just trap if
259 // it's reached at runtime.
260 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::TRAP));
261 return;
262 }
263
264 MachineFunction &MF = *MBB.getParent();
265 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
266 const X86TargetLowering &TLI = *STI.getTargetLowering();
267 const bool EmitInlineStackProbe = TLI.hasInlineStackProbe(MF);
268
269 // It's ok to not take into account large chunks when probing, as the
270 // allocation is split in smaller chunks anyway.
271 if (EmitInlineStackProbe && !InEpilogue) {
272
273 // This pseudo-instruction is going to be expanded, potentially using a
274 // loop, by inlineStackProbe().
275 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::STACKALLOC_W_PROBING)).addImm(Val: Offset);
276 return;
277 } else if (Offset > MaxSPChunk) {
278 // Rather than emit a long series of instructions for large offsets,
279 // load the offset into a register and do one sub/add
280 unsigned Reg = 0;
281 unsigned Rax = (unsigned)(Uses64BitFramePtr ? X86::RAX : X86::EAX);
282
283 if (isSub && !isEAXLiveIn(MBB))
284 Reg = Rax;
285 else
286 Reg = getX86SubSuperRegister(Reg: TRI->findDeadCallerSavedReg(MBB, MBBI),
287 Size: Uses64BitFramePtr ? 64 : 32);
288
289 unsigned AddSubRROpc = isSub ? getSUBrrOpcode(IsLP64: Uses64BitFramePtr)
290 : getADDrrOpcode(IsLP64: Uses64BitFramePtr);
291 if (Reg) {
292 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
293 MCID: TII.get(Opcode: X86::getMOVriOpcode(Use64BitReg: Uses64BitFramePtr, Imm: Offset)), DestReg: Reg)
294 .addImm(Val: Offset)
295 .setMIFlag(Flag);
296 MachineInstr *MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AddSubRROpc), DestReg: StackPtr)
297 .addReg(RegNo: StackPtr)
298 .addReg(RegNo: Reg);
299 MI->getOperand(i: 3).setIsDead(); // The EFLAGS implicit def is dead.
300 return;
301 } else if (Offset > 8 * MaxSPChunk) {
302 // If we would need more than 8 add or sub instructions (a >16GB stack
303 // frame), it's worth spilling RAX to materialize this immediate.
304 // pushq %rax
305 // movabsq +-$Offset+-SlotSize, %rax
306 // addq %rsp, %rax
307 // xchg %rax, (%rsp)
308 // movq (%rsp), %rsp
309 assert(Uses64BitFramePtr && "can't have 32-bit 16GB stack frame");
310 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH64r))
311 .addReg(RegNo: Rax, Flags: RegState::Kill)
312 .setMIFlag(Flag);
313 // Subtract is not commutative, so negate the offset and always use add.
314 // Subtract 8 less and add 8 more to account for the PUSH we just did.
315 if (isSub)
316 Offset = -(Offset - SlotSize);
317 else
318 Offset = Offset + SlotSize;
319 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
320 MCID: TII.get(Opcode: X86::getMOVriOpcode(Use64BitReg: Uses64BitFramePtr, Imm: Offset)), DestReg: Rax)
321 .addImm(Val: Offset)
322 .setMIFlag(Flag);
323 MachineInstr *MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::ADD64rr), DestReg: Rax)
324 .addReg(RegNo: Rax)
325 .addReg(RegNo: StackPtr);
326 MI->getOperand(i: 3).setIsDead(); // The EFLAGS implicit def is dead.
327 // Exchange the new SP in RAX with the top of the stack.
328 addRegOffset(
329 MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::XCHG64rm), DestReg: Rax).addReg(RegNo: Rax),
330 Reg: StackPtr, Offset: 0);
331 // Load new SP from the top of the stack into RSP.
332 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64rm), DestReg: StackPtr),
333 Reg: StackPtr, Offset: 0);
334 return;
335 }
336 }
337
338 while (Offset) {
339 if (Offset == SlotSize) {
340 // Use push / pop for slot sized adjustments as a size optimization. We
341 // need to find a dead register when using pop.
342 unsigned Reg = isSub ? (unsigned)(Is64Bit ? X86::RAX : X86::EAX)
343 : TRI->findDeadCallerSavedReg(MBB, MBBI);
344 if (Reg) {
345 unsigned Opc = isSub ? (Is64Bit ? X86::PUSH64r : X86::PUSH32r)
346 : (Is64Bit ? X86::POP64r : X86::POP32r);
347 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opc))
348 .addReg(RegNo: Reg, Flags: getDefRegState(B: !isSub) | getUndefRegState(B: isSub))
349 .setMIFlag(Flag);
350 return;
351 }
352 }
353
354 uint64_t ThisVal = std::min(a: Offset, b: MaxSPChunk);
355
356 BuildStackAdjustment(MBB, MBBI, DL, Offset: isSub ? -ThisVal : ThisVal, InEpilogue)
357 .setMIFlag(Flag);
358
359 Offset -= ThisVal;
360 }
361}
362
363MachineInstrBuilder X86FrameLowering::BuildStackAdjustment(
364 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
365 const DebugLoc &DL, int64_t Offset, bool InEpilogue) const {
366 assert(Offset != 0 && "zero offset stack adjustment requested");
367
368 // On Atom, using LEA to adjust SP is preferred, but using it in the epilogue
369 // is tricky.
370 bool UseLEA;
371 if (!InEpilogue) {
372 // Check if inserting the prologue at the beginning
373 // of MBB would require to use LEA operations.
374 // We need to use LEA operations if EFLAGS is live in, because
375 // it means an instruction will read it before it gets defined.
376 UseLEA = STI.useLeaForSP() || MBB.isLiveIn(Reg: X86::EFLAGS);
377 } else {
378 // If we can use LEA for SP but we shouldn't, check that none
379 // of the terminators uses the eflags. Otherwise we will insert
380 // a ADD that will redefine the eflags and break the condition.
381 // Alternatively, we could move the ADD, but this may not be possible
382 // and is an optimization anyway.
383 UseLEA = canUseLEAForSPInEpilogue(MF: *MBB.getParent());
384 if (UseLEA && !STI.useLeaForSP())
385 UseLEA = flagsNeedToBePreservedBeforeTheTerminators(MBB);
386 // If that assert breaks, that means we do not do the right thing
387 // in canUseAsEpilogue.
388 assert((UseLEA || !flagsNeedToBePreservedBeforeTheTerminators(MBB)) &&
389 "We shouldn't have allowed this insertion point");
390 }
391
392 MachineInstrBuilder MI;
393 // Use an NF (no-flags) variant as a smaller replacement for LEA when EFLAGS
394 // must be preserved (i.e. only when we would otherwise emit LEA). If EFLAGS
395 // is dead we prefer the plain SUB/ADD, which is shorter than the EVEX-encoded
396 // NF form. The NF stack-adjust opcodes below are 64-bit (SUB64ri32_NF/
397 // ADD64ri32_NF), so don't use them for the x32 ABI where the stack pointer is
398 // 32-bit. NF cannot reach a Win64 epilogue (which never uses LEA for the SP
399 // adjustment unless it has a frame pointer, and that path doesn't go through
400 // here), so the Windows epilogue unwinder never sees an undisassemblable NF
401 // add/sub.
402 bool UseNF = UseLEA && STI.hasNF() && Uses64BitFramePtr;
403 bool IsSub = Offset < 0;
404 uint64_t AbsOffset = IsSub ? -Offset : Offset;
405 if (UseNF) {
406 const unsigned Opc = IsSub ? X86::SUB64ri32_NF : X86::ADD64ri32_NF;
407 MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: StackPtr)
408 .addReg(RegNo: StackPtr)
409 .addImm(Val: AbsOffset);
410 // NF instructions define no EFLAGS, so there is nothing to mark dead.
411 } else if (UseLEA) {
412 MI = addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
413 MCID: TII.get(Opcode: getLEArOpcode(IsLP64: Uses64BitFramePtr)),
414 DestReg: StackPtr),
415 Reg: StackPtr, Offset);
416 } else {
417 unsigned Opc = IsSub ? getSUBriOpcode(IsLP64: Uses64BitFramePtr)
418 : getADDriOpcode(IsLP64: Uses64BitFramePtr);
419 int64_t Imm = AbsOffset;
420 // Prefer `add rsp, -128` over `sub rsp, 128` (and vice versa in the
421 // epilogue): 128 is the one magnitude whose negation fits the
422 // sign-extended 8-bit immediate while the value itself does not, so the
423 // flipped operation is three bytes shorter. EFLAGS is dead here (this
424 // branch clobbers it anyway). Windows CFI epilogues keep the canonical
425 // ADD: v1 unwind info describes no epilogues, so the unwinder detects
426 // one by disassembling forward for `add rsp, imm` (prologues are
427 // delimited by SizeOfProlog and never disassembled). Unwind v2/v3 do
428 // describe epilogues, but X86WinEHUnwindV2 expects the ADD spelling
429 // too.
430 if (AbsOffset == 128 &&
431 !(InEpilogue && MBB.getParent()->usesWindowsUnwindInfo())) {
432 Opc = IsSub ? getADDriOpcode(IsLP64: Uses64BitFramePtr)
433 : getSUBriOpcode(IsLP64: Uses64BitFramePtr);
434 Imm = -128;
435 }
436 MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: StackPtr)
437 .addReg(RegNo: StackPtr)
438 .addImm(Val: Imm);
439 MI->getOperand(i: 3).setIsDead(); // The EFLAGS implicit def is dead.
440 }
441 return MI;
442}
443
444template <typename FoundT, typename CalcT>
445int64_t X86FrameLowering::mergeSPUpdates(MachineBasicBlock &MBB,
446 MachineBasicBlock::iterator &MBBI,
447 FoundT FoundStackAdjust,
448 CalcT CalcNewOffset,
449 bool doMergeWithPrevious) const {
450 if ((doMergeWithPrevious && MBBI == MBB.begin()) ||
451 (!doMergeWithPrevious && MBBI == MBB.end()))
452 return CalcNewOffset(0);
453
454 MachineBasicBlock::iterator PI = doMergeWithPrevious ? std::prev(x: MBBI) : MBBI;
455
456 PI = skipDebugInstructionsBackward(It: PI, Begin: MBB.begin());
457 // It is assumed that ADD/SUB/LEA instruction is succeded by one CFI
458 // instruction, and that there are no DBG_VALUE or other instructions between
459 // ADD/SUB/LEA and its corresponding CFI instruction.
460 /* TODO: Add support for the case where there are multiple CFI instructions
461 below the ADD/SUB/LEA, e.g.:
462 ...
463 add
464 cfi_def_cfa_offset
465 cfi_offset
466 ...
467 */
468 if (doMergeWithPrevious && PI != MBB.begin() && PI->isCFIInstruction())
469 PI = std::prev(x: PI);
470
471 int64_t Offset = 0;
472 for (;;) {
473 unsigned Opc = PI->getOpcode();
474
475 if ((Opc == X86::ADD64ri32 || Opc == X86::ADD32ri ||
476 Opc == X86::ADD64ri32_NF) &&
477 PI->getOperand(i: 0).getReg() == StackPtr) {
478 assert(PI->getOperand(1).getReg() == StackPtr);
479 Offset = PI->getOperand(i: 2).getImm();
480 } else if ((Opc == X86::LEA32r || Opc == X86::LEA64_32r) &&
481 PI->getOperand(i: 0).getReg() == StackPtr &&
482 PI->getOperand(i: 1).getReg() == StackPtr &&
483 PI->getOperand(i: 2).getImm() == 1 &&
484 PI->getOperand(i: 3).getReg() == X86::NoRegister &&
485 PI->getOperand(i: 5).getReg() == X86::NoRegister) {
486 // For LEAs we have: def = lea SP, FI, noreg, Offset, noreg.
487 Offset = PI->getOperand(i: 4).getImm();
488 } else if ((Opc == X86::SUB64ri32 || Opc == X86::SUB32ri ||
489 Opc == X86::SUB64ri32_NF) &&
490 PI->getOperand(i: 0).getReg() == StackPtr) {
491 assert(PI->getOperand(1).getReg() == StackPtr);
492 Offset = -PI->getOperand(i: 2).getImm();
493 } else
494 return CalcNewOffset(0);
495
496 FoundStackAdjust(PI, Offset);
497 if ((uint64_t)std::abs(i: (int64_t)CalcNewOffset(Offset)) < MaxSPChunk)
498 break;
499
500 if (doMergeWithPrevious ? (PI == MBB.begin()) : (PI == MBB.end()))
501 return CalcNewOffset(0);
502
503 PI = doMergeWithPrevious ? std::prev(x: PI) : std::next(x: PI);
504 }
505
506 PI = MBB.erase(I: PI);
507 if (PI != MBB.end() && PI->isCFIInstruction()) {
508 auto CIs = MBB.getParent()->getFrameInstructions();
509 MCCFIInstruction CI = CIs[PI->getOperand(i: 0).getCFIIndex()];
510 if (CI.getOperation() == MCCFIInstruction::OpDefCfaOffset ||
511 CI.getOperation() == MCCFIInstruction::OpAdjustCfaOffset)
512 PI = MBB.erase(I: PI);
513 }
514 if (!doMergeWithPrevious)
515 MBBI = skipDebugInstructionsForward(It: PI, End: MBB.end());
516
517 return CalcNewOffset(Offset);
518}
519
520int64_t X86FrameLowering::mergeSPAdd(MachineBasicBlock &MBB,
521 MachineBasicBlock::iterator &MBBI,
522 int64_t AddOffset,
523 bool doMergeWithPrevious) const {
524 return mergeSPUpdates(
525 MBB, MBBI, CalcNewOffset: [AddOffset](int64_t Offset) { return AddOffset + Offset; },
526 doMergeWithPrevious);
527}
528
529void X86FrameLowering::BuildCFI(MachineBasicBlock &MBB,
530 MachineBasicBlock::iterator MBBI,
531 const DebugLoc &DL,
532 const MCCFIInstruction &CFIInst,
533 MachineInstr::MIFlag Flag) const {
534 MachineFunction &MF = *MBB.getParent();
535 unsigned CFIIndex = MF.addFrameInst(Inst: CFIInst);
536
537 if (CFIInst.getOperation() == MCCFIInstruction::OpAdjustCfaOffset)
538 MF.getInfo<X86MachineFunctionInfo>()->setHasCFIAdjustCfa(true);
539
540 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::CFI_INSTRUCTION))
541 .addCFIIndex(CFIIndex)
542 .setMIFlag(Flag);
543}
544
545/// Emits Dwarf Info specifying offsets of callee saved registers and
546/// frame pointer. This is called only when basic block sections are enabled.
547void X86FrameLowering::emitCalleeSavedFrameMovesFullCFA(
548 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
549 MachineFunction &MF = *MBB.getParent();
550 if (!hasFP(MF)) {
551 emitCalleeSavedFrameMoves(MBB, MBBI, DL: DebugLoc{}, IsPrologue: true);
552 return;
553 }
554 const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
555 const Register FramePtr = TRI->getFrameRegister(MF);
556 const Register MachineFramePtr =
557 STI.isTarget64BitILP32() ? Register(getX86SubSuperRegister(Reg: FramePtr, Size: 64))
558 : FramePtr;
559 unsigned DwarfReg = MRI->getDwarfRegNum(Reg: MachineFramePtr, isEH: true);
560 // Offset = space for return address + size of the frame pointer itself.
561 int64_t Offset = (Is64Bit ? 8 : 4) + (Uses64BitFramePtr ? 8 : 4);
562 BuildCFI(MBB, MBBI, DL: DebugLoc{},
563 CFIInst: MCCFIInstruction::createOffset(L: nullptr, Register: DwarfReg, Offset: -Offset));
564 emitCalleeSavedFrameMoves(MBB, MBBI, DL: DebugLoc{}, IsPrologue: true);
565}
566
567void X86FrameLowering::emitCalleeSavedFrameMoves(
568 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
569 const DebugLoc &DL, bool IsPrologue) const {
570 MachineFunction &MF = *MBB.getParent();
571 MachineFrameInfo &MFI = MF.getFrameInfo();
572 const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
573 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
574
575 // Add callee saved registers to move list.
576 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
577
578 // Calculate offsets.
579 for (const CalleeSavedInfo &I : CSI) {
580 int64_t Offset = MFI.getObjectOffset(ObjectIdx: I.getFrameIdx());
581 MCRegister Reg = I.getReg();
582 unsigned DwarfReg = MRI->getDwarfRegNum(Reg, isEH: true);
583
584 if (IsPrologue) {
585 if (X86FI->getStackPtrSaveMI()) {
586 // +2*SlotSize because there is return address and ebp at the bottom
587 // of the stack.
588 // | retaddr |
589 // | ebp |
590 // | |<--ebp
591 Offset += 2 * SlotSize;
592 SmallString<64> CfaExpr;
593 CfaExpr.push_back(Elt: dwarf::DW_CFA_expression);
594 uint8_t buffer[16];
595 CfaExpr.append(in_start: buffer, in_end: buffer + encodeULEB128(Value: DwarfReg, p: buffer));
596 CfaExpr.push_back(Elt: 2);
597 Register FramePtr = TRI->getFrameRegister(MF);
598 const Register MachineFramePtr =
599 STI.isTarget64BitILP32()
600 ? Register(getX86SubSuperRegister(Reg: FramePtr, Size: 64))
601 : FramePtr;
602 unsigned DwarfFramePtr = MRI->getDwarfRegNum(Reg: MachineFramePtr, isEH: true);
603 CfaExpr.push_back(Elt: (uint8_t)(dwarf::DW_OP_breg0 + DwarfFramePtr));
604 CfaExpr.append(in_start: buffer, in_end: buffer + encodeSLEB128(Value: Offset, p: buffer));
605 BuildCFI(MBB, MBBI, DL,
606 CFIInst: MCCFIInstruction::createEscape(L: nullptr, Vals: CfaExpr.str()),
607 Flag: MachineInstr::FrameSetup);
608 } else {
609 BuildCFI(MBB, MBBI, DL,
610 CFIInst: MCCFIInstruction::createOffset(L: nullptr, Register: DwarfReg, Offset));
611 }
612 } else {
613 BuildCFI(MBB, MBBI, DL,
614 CFIInst: MCCFIInstruction::createRestore(L: nullptr, Register: DwarfReg));
615 }
616 }
617 if (auto *MI = X86FI->getStackPtrSaveMI()) {
618 int FI = MI->getOperand(i: 1).getIndex();
619 int64_t Offset = MFI.getObjectOffset(ObjectIdx: FI) + 2 * SlotSize;
620 SmallString<64> CfaExpr;
621 Register FramePtr = TRI->getFrameRegister(MF);
622 const Register MachineFramePtr =
623 STI.isTarget64BitILP32()
624 ? Register(getX86SubSuperRegister(Reg: FramePtr, Size: 64))
625 : FramePtr;
626 unsigned DwarfFramePtr = MRI->getDwarfRegNum(Reg: MachineFramePtr, isEH: true);
627 CfaExpr.push_back(Elt: (uint8_t)(dwarf::DW_OP_breg0 + DwarfFramePtr));
628 uint8_t buffer[16];
629 CfaExpr.append(in_start: buffer, in_end: buffer + encodeSLEB128(Value: Offset, p: buffer));
630 CfaExpr.push_back(Elt: dwarf::DW_OP_deref);
631
632 SmallString<64> DefCfaExpr;
633 DefCfaExpr.push_back(Elt: dwarf::DW_CFA_def_cfa_expression);
634 DefCfaExpr.append(in_start: buffer, in_end: buffer + encodeSLEB128(Value: CfaExpr.size(), p: buffer));
635 DefCfaExpr.append(RHS: CfaExpr.str());
636 // DW_CFA_def_cfa_expression: DW_OP_breg5 offset, DW_OP_deref
637 BuildCFI(MBB, MBBI, DL,
638 CFIInst: MCCFIInstruction::createEscape(L: nullptr, Vals: DefCfaExpr.str()),
639 Flag: MachineInstr::FrameSetup);
640 }
641}
642
643void X86FrameLowering::emitZeroCallUsedRegs(BitVector RegsToZero,
644 MachineBasicBlock &MBB,
645 RegScavenger *) const {
646 const MachineFunction &MF = *MBB.getParent();
647
648 // Insertion point.
649 MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
650
651 // Fake a debug loc.
652 DebugLoc DL;
653 if (MBBI != MBB.end())
654 DL = MBBI->getDebugLoc();
655
656 // Zero out FP stack if referenced. Do this outside of the loop below so that
657 // it's done only once.
658 for (MCRegister Reg : RegsToZero.set_bits()) {
659 if (!X86::RFP80RegClass.contains(Reg))
660 continue;
661
662 // Do not push zeros over x87 return values. X86FloatingPoint records
663 // returned values as implicit ST0/ST1 uses on the return instruction.
664 unsigned NumFPRegs = 8;
665 if (MBBI->hasRegisterImplicitUseOperand(Reg: X86::ST0))
666 --NumFPRegs;
667 if (MBBI->hasRegisterImplicitUseOperand(Reg: X86::ST1))
668 --NumFPRegs;
669
670 for (unsigned i = 0; i != NumFPRegs; ++i)
671 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::LD_F0));
672
673 for (unsigned i = 0; i != NumFPRegs; ++i)
674 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::ST_FPrr)).addReg(RegNo: X86::ST0);
675 break;
676 }
677
678 // For GPRs, we only care to clear out the 32-bit register.
679 BitVector GPRsToZero(TRI->getNumRegs());
680 for (MCRegister Reg : RegsToZero.set_bits())
681 if (TRI->isGeneralPurposeRegister(MF, Reg)) {
682 GPRsToZero.set(getX86SubSuperRegister(Reg, Size: 32));
683 RegsToZero.reset(Idx: Reg);
684 }
685
686 // Zero out the GPRs first.
687 for (MCRegister Reg : GPRsToZero.set_bits())
688 TII.buildClearRegister(Reg, MBB, Iter: MBBI, DL);
689
690 // Coalesce the aliasing XMM/YMM/ZMM views of each vector register so a lane
691 // is cleared only once, mirroring the GPR handling above.
692 auto getVectorClearReg = [&](MCRegister Reg) -> MCRegister {
693 if (!X86::VR128RegClass.contains(Reg) &&
694 !X86::VR128XRegClass.contains(Reg) &&
695 !X86::VR256RegClass.contains(Reg) &&
696 !X86::VR256XRegClass.contains(Reg) && !X86::VR512RegClass.contains(Reg))
697 return MCRegister();
698
699 // Clearing the XMM zeroes the whole lane. XMM0-15 use the compact VEX form;
700 // XMM16-31 are EVEX-only, reachable only via the ZMM form.
701 MCRegister Xmm = TRI->getSubReg(Reg, Idx: X86::sub_xmm);
702 if (!Xmm)
703 Xmm = Reg;
704 if (X86::VR128RegClass.contains(Reg: Xmm))
705 return Xmm;
706 MCRegister Zmm =
707 TRI->getMatchingSuperReg(Reg: Xmm, SubIdx: X86::sub_xmm, RC: &X86::VR512RegClass);
708 assert(Zmm && "XMM16-31 must have an enclosing ZMM to clear through");
709 return Zmm;
710 };
711
712 BitVector VecRegsToZero(TRI->getNumRegs());
713 for (MCRegister Reg : RegsToZero.set_bits())
714 if (MCRegister Clear = getVectorClearReg(Reg)) {
715 VecRegsToZero.set(Clear.id());
716 RegsToZero.reset(Idx: Reg);
717 }
718
719 for (MCRegister Reg : VecRegsToZero.set_bits())
720 TII.buildClearRegister(Reg, MBB, Iter: MBBI, DL);
721
722 // Zero out the remaining registers (e.g. mask registers).
723 for (MCRegister Reg : RegsToZero.set_bits())
724 TII.buildClearRegister(Reg, MBB, Iter: MBBI, DL);
725}
726
727void X86FrameLowering::emitStackProbe(
728 MachineFunction &MF, MachineBasicBlock &MBB,
729 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog,
730 std::optional<MachineFunction::DebugInstrOperandPair> InstrNum) const {
731 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
732 if (STI.isTargetWindowsCoreCLR()) {
733 if (InProlog) {
734 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::STACKALLOC_W_PROBING))
735 .addImm(Val: 0 /* no explicit stack size */);
736 } else {
737 emitStackProbeInline(MF, MBB, MBBI, DL, InProlog: false);
738 }
739 } else {
740 emitStackProbeCall(MF, MBB, MBBI, DL, InProlog, InstrNum);
741 }
742}
743
744bool X86FrameLowering::stackProbeFunctionModifiesSP() const {
745 return STI.isOSWindows() && !STI.isTargetWin64();
746}
747
748void X86FrameLowering::inlineStackProbe(MachineFunction &MF,
749 MachineBasicBlock &PrologMBB) const {
750 auto Where = llvm::find_if(Range&: PrologMBB, P: [](MachineInstr &MI) {
751 return MI.getOpcode() == X86::STACKALLOC_W_PROBING;
752 });
753 if (Where != PrologMBB.end()) {
754 DebugLoc DL = PrologMBB.findDebugLoc(MBBI: Where);
755 emitStackProbeInline(MF, MBB&: PrologMBB, MBBI: Where, DL, InProlog: true);
756 Where->eraseFromParent();
757 }
758}
759
760void X86FrameLowering::emitStackProbeInline(MachineFunction &MF,
761 MachineBasicBlock &MBB,
762 MachineBasicBlock::iterator MBBI,
763 const DebugLoc &DL,
764 bool InProlog) const {
765 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
766 if (STI.isTargetWindowsCoreCLR() && STI.is64Bit())
767 emitStackProbeInlineWindowsCoreCLR64(MF, MBB, MBBI, DL, InProlog);
768 else
769 emitStackProbeInlineGeneric(MF, MBB, MBBI, DL, InProlog);
770}
771
772void X86FrameLowering::emitStackProbeInlineGeneric(
773 MachineFunction &MF, MachineBasicBlock &MBB,
774 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog) const {
775 MachineInstr &AllocWithProbe = *MBBI;
776 uint64_t Offset = AllocWithProbe.getOperand(i: 0).getImm();
777
778 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
779 const X86TargetLowering &TLI = *STI.getTargetLowering();
780 assert(!(STI.is64Bit() && STI.isTargetWindowsCoreCLR()) &&
781 "different expansion expected for CoreCLR 64 bit");
782
783 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
784 uint64_t ProbeChunk = StackProbeSize * 8;
785
786 uint64_t MaxAlign =
787 TRI->hasStackRealignment(MF) ? calculateMaxStackAlign(MF) : 0;
788
789 // Synthesize a loop or unroll it, depending on the number of iterations.
790 // BuildStackAlignAND ensures that only MaxAlign % StackProbeSize bits left
791 // between the unaligned rsp and current rsp.
792 if (Offset > ProbeChunk) {
793 emitStackProbeInlineGenericLoop(MF, MBB, MBBI, DL, Offset,
794 Align: MaxAlign % StackProbeSize);
795 } else {
796 emitStackProbeInlineGenericBlock(MF, MBB, MBBI, DL, Offset,
797 Align: MaxAlign % StackProbeSize);
798 }
799}
800
801void X86FrameLowering::emitStackProbeInlineGenericBlock(
802 MachineFunction &MF, MachineBasicBlock &MBB,
803 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, uint64_t Offset,
804 uint64_t AlignOffset) const {
805
806 // The code below is allocating one StackProbeSize at a time,
807 // then inserting a "probe" (write of 0), in order to hit any guard pages.
808 // A probe is only needed every StackProbeSize (no skipping entire pages).
809 //
810 // StackProbeSize (PAGE_SIZE)
811 // / \
812 // |----------|----------|------> rsp decreasing
813 // ^ ^ ^
814 // | | |
815 // CurrentOffset Offset (Offset + AlignOffset)
816 //
817 // Offset: size we are being asked to allocate on the stack, in bytes.
818 // AlignOffset: extra amount to allocate (< StackProbeSize),
819 // needed to satisfy any alignment for calling conventions, etc.
820
821 const bool NeedsDwarfCFI = needsDwarfCFI(MF);
822 const bool HasFP = hasFP(MF);
823 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
824 const X86TargetLowering &TLI = *STI.getTargetLowering();
825 const unsigned MovMIOpc = Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
826 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
827
828 uint64_t CurrentOffset = 0;
829
830 assert(AlignOffset < StackProbeSize);
831
832 // If the offset is so small it fits within a page, there's nothing to do.
833 if (StackProbeSize < Offset + AlignOffset) {
834
835 uint64_t StackAdjustment = StackProbeSize - AlignOffset;
836 BuildStackAdjustment(MBB, MBBI, DL, Offset: -StackAdjustment, /*InEpilogue=*/false)
837 .setMIFlag(MachineInstr::FrameSetup);
838 if (!HasFP && NeedsDwarfCFI) {
839 BuildCFI(
840 MBB, MBBI, DL,
841 CFIInst: MCCFIInstruction::createAdjustCfaOffset(L: nullptr, Adjustment: StackAdjustment));
842 }
843
844 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: MovMIOpc))
845 .setMIFlag(MachineInstr::FrameSetup),
846 Reg: StackPtr, Offset: 0)
847 .addImm(Val: 0)
848 .setMIFlag(MachineInstr::FrameSetup);
849 NumFrameExtraProbe++;
850 CurrentOffset = StackProbeSize - AlignOffset;
851 }
852
853 // For the next N - 1 pages, just probe.
854 while (CurrentOffset + StackProbeSize <= Offset) {
855 BuildStackAdjustment(MBB, MBBI, DL, Offset: -StackProbeSize, /*InEpilogue=*/false)
856 .setMIFlag(MachineInstr::FrameSetup);
857
858 if (!HasFP && NeedsDwarfCFI) {
859 BuildCFI(
860 MBB, MBBI, DL,
861 CFIInst: MCCFIInstruction::createAdjustCfaOffset(L: nullptr, Adjustment: StackProbeSize));
862 }
863 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: MovMIOpc))
864 .setMIFlag(MachineInstr::FrameSetup),
865 Reg: StackPtr, Offset: 0)
866 .addImm(Val: 0)
867 .setMIFlag(MachineInstr::FrameSetup);
868 NumFrameExtraProbe++;
869 CurrentOffset += StackProbeSize;
870 }
871
872 const uint64_t ChunkSize = Offset - CurrentOffset;
873 // We may not require a tail allocation. For example, this can happen
874 // if Offset is a multiple of StackProbeSize, and AlignOffset = 0.
875 if (ChunkSize == 0)
876 return;
877 else if (ChunkSize == SlotSize) {
878 // Use push for slot sized adjustments as a size optimization,
879 // like emitSPUpdate does when not probing.
880 unsigned Reg = Is64Bit ? X86::RAX : X86::EAX;
881 unsigned Opc = Is64Bit ? X86::PUSH64r : X86::PUSH32r;
882 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opc))
883 .addReg(RegNo: Reg, Flags: RegState::Undef)
884 .setMIFlag(MachineInstr::FrameSetup);
885 } else {
886 assert(!(ChunkSize > StackProbeSize) &&
887 "Tail allocation greater than page size");
888 BuildStackAdjustment(MBB, MBBI, DL, Offset: -ChunkSize, /*InEpilogue=*/false)
889 .setMIFlag(MachineInstr::FrameSetup);
890 }
891 // No need to adjust Dwarf CFA offset here, the last position of the stack has
892 // been defined
893}
894
895void X86FrameLowering::emitStackProbeInlineGenericLoop(
896 MachineFunction &MF, MachineBasicBlock &MBB,
897 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, uint64_t Offset,
898 uint64_t AlignOffset) const {
899 assert(Offset && "null offset");
900
901 assert(MBB.computeRegisterLiveness(TRI, X86::EFLAGS, MBBI) !=
902 MachineBasicBlock::LQR_Live &&
903 "Inline stack probe loop will clobber live EFLAGS.");
904
905 const bool NeedsDwarfCFI = needsDwarfCFI(MF);
906 const bool HasFP = hasFP(MF);
907 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
908 const X86TargetLowering &TLI = *STI.getTargetLowering();
909 const unsigned MovMIOpc = Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
910 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
911
912 if (AlignOffset) {
913 if (AlignOffset < StackProbeSize) {
914 // Perform a first smaller allocation followed by a probe.
915 BuildStackAdjustment(MBB, MBBI, DL, Offset: -AlignOffset, /*InEpilogue=*/false)
916 .setMIFlag(MachineInstr::FrameSetup);
917
918 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: MovMIOpc))
919 .setMIFlag(MachineInstr::FrameSetup),
920 Reg: StackPtr, Offset: 0)
921 .addImm(Val: 0)
922 .setMIFlag(MachineInstr::FrameSetup);
923 NumFrameExtraProbe++;
924 Offset -= AlignOffset;
925 }
926 }
927
928 // Synthesize a loop
929 NumFrameLoopProbe++;
930 const BasicBlock *LLVM_BB = MBB.getBasicBlock();
931
932 MachineBasicBlock *testMBB = MF.CreateMachineBasicBlock(BB: LLVM_BB);
933 MachineBasicBlock *tailMBB = MF.CreateMachineBasicBlock(BB: LLVM_BB);
934
935 MachineFunction::iterator MBBIter = ++MBB.getIterator();
936 MF.insert(MBBI: MBBIter, MBB: testMBB);
937 MF.insert(MBBI: MBBIter, MBB: tailMBB);
938
939 Register FinalStackProbed = Uses64BitFramePtr ? X86::R11
940 : Is64Bit ? X86::R11D
941 : X86::EAX;
942
943 // save loop bound
944 {
945 const uint64_t BoundOffset = alignDown(Value: Offset, Align: StackProbeSize);
946
947 // Can we calculate the loop bound using SUB with a 32-bit immediate?
948 // Note that the immediate gets sign-extended when used with a 64-bit
949 // register, so in that case we only have 31 bits to work with.
950 bool canUseSub =
951 Uses64BitFramePtr ? isUInt<31>(x: BoundOffset) : isUInt<32>(x: BoundOffset);
952
953 if (canUseSub) {
954 const unsigned SUBOpc = getSUBriOpcode(IsLP64: Uses64BitFramePtr);
955
956 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: FinalStackProbed)
957 .addReg(RegNo: StackPtr)
958 .setMIFlag(MachineInstr::FrameSetup);
959 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: SUBOpc), DestReg: FinalStackProbed)
960 .addReg(RegNo: FinalStackProbed)
961 .addImm(Val: BoundOffset)
962 .setMIFlag(MachineInstr::FrameSetup)
963 .setOperandDead(3); // implicit-def $eflags
964 } else if (Uses64BitFramePtr) {
965 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64ri), DestReg: FinalStackProbed)
966 .addImm(Val: -BoundOffset)
967 .setMIFlag(MachineInstr::FrameSetup);
968 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::ADD64rr), DestReg: FinalStackProbed)
969 .addReg(RegNo: FinalStackProbed)
970 .addReg(RegNo: StackPtr)
971 .setMIFlag(MachineInstr::FrameSetup)
972 .setOperandDead(3); // implicit-def $eflags
973 } else {
974 llvm_unreachable("Offset too large for 32-bit stack pointer");
975 }
976
977 // while in the loop, use loop-invariant reg for CFI,
978 // instead of the stack pointer, which changes during the loop
979 if (!HasFP && NeedsDwarfCFI) {
980 // x32 uses the same DWARF register numbers as x86-64,
981 // so there isn't a register number for r11d, we must use r11 instead
982 const Register DwarfFinalStackProbed =
983 STI.isTarget64BitILP32()
984 ? Register(getX86SubSuperRegister(Reg: FinalStackProbed, Size: 64))
985 : FinalStackProbed;
986
987 BuildCFI(MBB, MBBI, DL,
988 CFIInst: MCCFIInstruction::createDefCfaRegister(
989 L: nullptr, Register: TRI->getDwarfRegNum(Reg: DwarfFinalStackProbed, isEH: true)));
990 BuildCFI(MBB, MBBI, DL,
991 CFIInst: MCCFIInstruction::createAdjustCfaOffset(L: nullptr, Adjustment: BoundOffset));
992 }
993 }
994
995 // allocate a page
996 BuildStackAdjustment(MBB&: *testMBB, MBBI: testMBB->end(), DL, Offset: -StackProbeSize,
997 /*InEpilogue=*/false)
998 .setMIFlag(MachineInstr::FrameSetup);
999
1000 // touch the page
1001 addRegOffset(MIB: BuildMI(BB: testMBB, MIMD: DL, MCID: TII.get(Opcode: MovMIOpc))
1002 .setMIFlag(MachineInstr::FrameSetup),
1003 Reg: StackPtr, Offset: 0)
1004 .addImm(Val: 0)
1005 .setMIFlag(MachineInstr::FrameSetup);
1006
1007 // cmp with stack pointer bound
1008 BuildMI(BB: testMBB, MIMD: DL, MCID: TII.get(Opcode: Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
1009 .addReg(RegNo: StackPtr)
1010 .addReg(RegNo: FinalStackProbed)
1011 .setMIFlag(MachineInstr::FrameSetup);
1012
1013 // jump
1014 BuildMI(BB: testMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
1015 .addMBB(MBB: testMBB)
1016 .addImm(Val: X86::COND_NE)
1017 .setMIFlag(MachineInstr::FrameSetup);
1018 testMBB->addSuccessor(Succ: testMBB);
1019 testMBB->addSuccessor(Succ: tailMBB);
1020
1021 // BB management
1022 tailMBB->splice(Where: tailMBB->end(), Other: &MBB, From: MBBI, To: MBB.end());
1023 tailMBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
1024 MBB.addSuccessor(Succ: testMBB);
1025
1026 // handle tail
1027 const uint64_t TailOffset = Offset % StackProbeSize;
1028 MachineBasicBlock::iterator TailMBBIter = tailMBB->begin();
1029 if (TailOffset) {
1030 BuildStackAdjustment(MBB&: *tailMBB, MBBI: TailMBBIter, DL, Offset: -TailOffset,
1031 /*InEpilogue=*/false)
1032 .setMIFlag(MachineInstr::FrameSetup);
1033 }
1034
1035 // after the loop, switch back to stack pointer for CFI
1036 if (!HasFP && NeedsDwarfCFI) {
1037 // x32 uses the same DWARF register numbers as x86-64,
1038 // so there isn't a register number for esp, we must use rsp instead
1039 const Register DwarfStackPtr =
1040 STI.isTarget64BitILP32()
1041 ? Register(getX86SubSuperRegister(Reg: StackPtr, Size: 64))
1042 : Register(StackPtr);
1043
1044 BuildCFI(MBB&: *tailMBB, MBBI: TailMBBIter, DL,
1045 CFIInst: MCCFIInstruction::createDefCfaRegister(
1046 L: nullptr, Register: TRI->getDwarfRegNum(Reg: DwarfStackPtr, isEH: true)));
1047 }
1048
1049 // Update Live In information
1050 fullyRecomputeLiveIns(MBBs: {tailMBB, testMBB});
1051}
1052
1053void X86FrameLowering::emitStackProbeInlineWindowsCoreCLR64(
1054 MachineFunction &MF, MachineBasicBlock &MBB,
1055 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog) const {
1056 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
1057 assert(STI.is64Bit() && "different expansion needed for 32 bit");
1058 assert(STI.isTargetWindowsCoreCLR() && "custom expansion expects CoreCLR");
1059 const TargetInstrInfo &TII = *STI.getInstrInfo();
1060 const BasicBlock *LLVM_BB = MBB.getBasicBlock();
1061
1062 assert(MBB.computeRegisterLiveness(TRI, X86::EFLAGS, MBBI) !=
1063 MachineBasicBlock::LQR_Live &&
1064 "Inline stack probe loop will clobber live EFLAGS.");
1065
1066 // RAX contains the number of bytes of desired stack adjustment.
1067 // The handling here assumes this value has already been updated so as to
1068 // maintain stack alignment.
1069 //
1070 // We need to exit with RSP modified by this amount and execute suitable
1071 // page touches to notify the OS that we're growing the stack responsibly.
1072 // All stack probing must be done without modifying RSP.
1073 //
1074 // MBB:
1075 // SizeReg = RAX;
1076 // ZeroReg = 0
1077 // CopyReg = RSP
1078 // Flags, TestReg = CopyReg - SizeReg
1079 // FinalReg = !Flags.Ovf ? TestReg : ZeroReg
1080 // LimitReg = gs magic thread env access
1081 // if FinalReg >= LimitReg goto ContinueMBB
1082 // RoundBB:
1083 // RoundReg = page address of FinalReg
1084 // LoopMBB:
1085 // LoopReg = PHI(LimitReg,ProbeReg)
1086 // ProbeReg = LoopReg - PageSize
1087 // [ProbeReg] = 0
1088 // if (ProbeReg > RoundReg) goto LoopMBB
1089 // ContinueMBB:
1090 // RSP = RSP - RAX
1091 // [rest of original MBB]
1092
1093 // Set up the new basic blocks
1094 MachineBasicBlock *RoundMBB = MF.CreateMachineBasicBlock(BB: LLVM_BB);
1095 MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(BB: LLVM_BB);
1096 MachineBasicBlock *ContinueMBB = MF.CreateMachineBasicBlock(BB: LLVM_BB);
1097
1098 MachineFunction::iterator MBBIter = std::next(x: MBB.getIterator());
1099 MF.insert(MBBI: MBBIter, MBB: RoundMBB);
1100 MF.insert(MBBI: MBBIter, MBB: LoopMBB);
1101 MF.insert(MBBI: MBBIter, MBB: ContinueMBB);
1102
1103 // Split MBB and move the tail portion down to ContinueMBB.
1104 MachineBasicBlock::iterator BeforeMBBI = std::prev(x: MBBI);
1105 ContinueMBB->splice(Where: ContinueMBB->begin(), Other: &MBB, From: MBBI, To: MBB.end());
1106 ContinueMBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
1107
1108 // Some useful constants
1109 const int64_t ThreadEnvironmentStackLimit = 0x10;
1110 const int64_t PageSize = 0x1000;
1111 const int64_t PageMask = ~(PageSize - 1);
1112
1113 // Registers we need. For the normal case we use virtual
1114 // registers. For the prolog expansion we use RAX, RCX and RDX.
1115 MachineRegisterInfo &MRI = MF.getRegInfo();
1116 const TargetRegisterClass *RegClass = &X86::GR64RegClass;
1117 const Register
1118 SizeReg = InProlog ? X86::RAX : MRI.createVirtualRegister(RegClass),
1119 ZeroReg = InProlog ? X86::RCX : MRI.createVirtualRegister(RegClass),
1120 CopyReg = InProlog ? X86::RDX : MRI.createVirtualRegister(RegClass),
1121 TestReg = InProlog ? X86::RDX : MRI.createVirtualRegister(RegClass),
1122 FinalReg = InProlog ? X86::RDX : MRI.createVirtualRegister(RegClass),
1123 RoundedReg = InProlog ? X86::RDX : MRI.createVirtualRegister(RegClass),
1124 LimitReg = InProlog ? X86::RCX : MRI.createVirtualRegister(RegClass),
1125 JoinReg = InProlog ? X86::RCX : MRI.createVirtualRegister(RegClass),
1126 ProbeReg = InProlog ? X86::RCX : MRI.createVirtualRegister(RegClass);
1127
1128 // SP-relative offsets where we can save RCX and RDX.
1129 int64_t RCXShadowSlot = 0;
1130 int64_t RDXShadowSlot = 0;
1131
1132 // If inlining in the prolog, save RCX and RDX.
1133 if (InProlog) {
1134 // Compute the offsets. We need to account for things already
1135 // pushed onto the stack at this point: return address, frame
1136 // pointer (if used), and callee saves.
1137 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
1138 const int64_t CalleeSaveSize = X86FI->getCalleeSavedFrameSize();
1139 const bool HasFP = hasFP(MF);
1140
1141 // Check if we need to spill RCX and/or RDX.
1142 // Here we assume that no earlier prologue instruction changes RCX and/or
1143 // RDX, so checking the block live-ins is enough.
1144 const bool IsRCXLiveIn = MBB.isLiveIn(Reg: X86::RCX);
1145 const bool IsRDXLiveIn = MBB.isLiveIn(Reg: X86::RDX);
1146 int64_t InitSlot = 8 + CalleeSaveSize + (HasFP ? 8 : 0);
1147 // Assign the initial slot to both registers, then change RDX's slot if both
1148 // need to be spilled.
1149 if (IsRCXLiveIn)
1150 RCXShadowSlot = InitSlot;
1151 if (IsRDXLiveIn)
1152 RDXShadowSlot = InitSlot;
1153 if (IsRDXLiveIn && IsRCXLiveIn)
1154 RDXShadowSlot += 8;
1155 // Emit the saves if needed.
1156 if (IsRCXLiveIn)
1157 addRegOffset(MIB: BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64mr)), Reg: X86::RSP,
1158 Offset: RCXShadowSlot)
1159 .addReg(RegNo: X86::RCX);
1160 if (IsRDXLiveIn)
1161 addRegOffset(MIB: BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64mr)), Reg: X86::RSP,
1162 Offset: RDXShadowSlot)
1163 .addReg(RegNo: X86::RDX);
1164 } else {
1165 // Not in the prolog. Copy RAX to a virtual reg.
1166 BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64rr), DestReg: SizeReg).addReg(RegNo: X86::RAX);
1167 }
1168
1169 // Add code to MBB to check for overflow and set the new target stack pointer
1170 // to zero if so.
1171 BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::XOR64rr), DestReg: ZeroReg)
1172 .addReg(RegNo: ZeroReg, Flags: RegState::Undef)
1173 .addReg(RegNo: ZeroReg, Flags: RegState::Undef)
1174 .setOperandDead(3); // implicit-def $eflags
1175 BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64rr), DestReg: CopyReg).addReg(RegNo: X86::RSP);
1176 BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::SUB64rr), DestReg: TestReg)
1177 .addReg(RegNo: CopyReg)
1178 .addReg(RegNo: SizeReg);
1179 BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::CMOV64rr), DestReg: FinalReg)
1180 .addReg(RegNo: TestReg)
1181 .addReg(RegNo: ZeroReg)
1182 .addImm(Val: X86::COND_B);
1183
1184 // FinalReg now holds final stack pointer value, or zero if
1185 // allocation would overflow. Compare against the current stack
1186 // limit from the thread environment block. Note this limit is the
1187 // lowest touched page on the stack, not the point at which the OS
1188 // will cause an overflow exception, so this is just an optimization
1189 // to avoid unnecessarily touching pages that are below the current
1190 // SP but already committed to the stack by the OS.
1191 BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64rm), DestReg: LimitReg)
1192 .addReg(RegNo: 0)
1193 .addImm(Val: 1)
1194 .addReg(RegNo: 0)
1195 .addImm(Val: ThreadEnvironmentStackLimit)
1196 .addReg(RegNo: X86::GS);
1197 BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::CMP64rr)).addReg(RegNo: FinalReg).addReg(RegNo: LimitReg);
1198 // Jump if the desired stack pointer is at or above the stack limit.
1199 BuildMI(BB: &MBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
1200 .addMBB(MBB: ContinueMBB)
1201 .addImm(Val: X86::COND_AE);
1202
1203 // Add code to roundMBB to round the final stack pointer to a page boundary.
1204 if (InProlog)
1205 RoundMBB->addLiveIn(PhysReg: FinalReg);
1206 BuildMI(BB: RoundMBB, MIMD: DL, MCID: TII.get(Opcode: X86::AND64ri32), DestReg: RoundedReg)
1207 .addReg(RegNo: FinalReg)
1208 .addImm(Val: PageMask)
1209 .setOperandDead(3); // implicit-def $eflags
1210 BuildMI(BB: RoundMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JMP_1)).addMBB(MBB: LoopMBB);
1211
1212 // LimitReg now holds the current stack limit, RoundedReg page-rounded
1213 // final RSP value. Add code to loopMBB to decrement LimitReg page-by-page
1214 // and probe until we reach RoundedReg.
1215 if (!InProlog) {
1216 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII.get(Opcode: X86::PHI), DestReg: JoinReg)
1217 .addReg(RegNo: LimitReg)
1218 .addMBB(MBB: RoundMBB)
1219 .addReg(RegNo: ProbeReg)
1220 .addMBB(MBB: LoopMBB);
1221 }
1222
1223 if (InProlog)
1224 LoopMBB->addLiveIn(PhysReg: JoinReg);
1225 addRegOffset(MIB: BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII.get(Opcode: X86::LEA64r), DestReg: ProbeReg), Reg: JoinReg,
1226 Offset: -PageSize);
1227
1228 // Probe by storing a byte onto the stack.
1229 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII.get(Opcode: X86::MOV8mi))
1230 .addReg(RegNo: ProbeReg)
1231 .addImm(Val: 1)
1232 .addReg(RegNo: 0)
1233 .addImm(Val: 0)
1234 .addReg(RegNo: 0)
1235 .addImm(Val: 0);
1236
1237 if (InProlog)
1238 LoopMBB->addLiveIn(PhysReg: RoundedReg);
1239 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII.get(Opcode: X86::CMP64rr))
1240 .addReg(RegNo: RoundedReg)
1241 .addReg(RegNo: ProbeReg);
1242 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
1243 .addMBB(MBB: LoopMBB)
1244 .addImm(Val: X86::COND_NE);
1245
1246 MachineBasicBlock::iterator ContinueMBBI = ContinueMBB->getFirstNonPHI();
1247
1248 // If in prolog, restore RDX and RCX.
1249 if (InProlog) {
1250 if (RCXShadowSlot) // It means we spilled RCX in the prologue.
1251 addRegOffset(MIB: BuildMI(BB&: *ContinueMBB, I: ContinueMBBI, MIMD: DL,
1252 MCID: TII.get(Opcode: X86::MOV64rm), DestReg: X86::RCX),
1253 Reg: X86::RSP, Offset: RCXShadowSlot);
1254 if (RDXShadowSlot) // It means we spilled RDX in the prologue.
1255 addRegOffset(MIB: BuildMI(BB&: *ContinueMBB, I: ContinueMBBI, MIMD: DL,
1256 MCID: TII.get(Opcode: X86::MOV64rm), DestReg: X86::RDX),
1257 Reg: X86::RSP, Offset: RDXShadowSlot);
1258 }
1259
1260 // Now that the probing is done, add code to continueMBB to update
1261 // the stack pointer for real.
1262 BuildMI(BB&: *ContinueMBB, I: ContinueMBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SUB64rr), DestReg: X86::RSP)
1263 .addReg(RegNo: X86::RSP)
1264 .addReg(RegNo: SizeReg)
1265 .setOperandDead(3); // implicit-def $eflags
1266
1267 // Add the control flow edges we need.
1268 MBB.addSuccessor(Succ: ContinueMBB);
1269 MBB.addSuccessor(Succ: RoundMBB);
1270 RoundMBB->addSuccessor(Succ: LoopMBB);
1271 LoopMBB->addSuccessor(Succ: ContinueMBB);
1272 LoopMBB->addSuccessor(Succ: LoopMBB);
1273
1274 if (InProlog) {
1275 LivePhysRegs LiveRegs;
1276 computeAndAddLiveIns(LiveRegs, MBB&: *ContinueMBB);
1277 }
1278
1279 // Mark all the instructions added to the prolog as frame setup.
1280 if (InProlog) {
1281 for (++BeforeMBBI; BeforeMBBI != MBB.end(); ++BeforeMBBI) {
1282 BeforeMBBI->setFlag(MachineInstr::FrameSetup);
1283 }
1284 for (MachineInstr &MI : *RoundMBB) {
1285 MI.setFlag(MachineInstr::FrameSetup);
1286 }
1287 for (MachineInstr &MI : *LoopMBB) {
1288 MI.setFlag(MachineInstr::FrameSetup);
1289 }
1290 for (MachineInstr &MI :
1291 llvm::make_range(x: ContinueMBB->begin(), y: ContinueMBBI)) {
1292 MI.setFlag(MachineInstr::FrameSetup);
1293 }
1294 }
1295}
1296
1297void X86FrameLowering::emitStackProbeCall(
1298 MachineFunction &MF, MachineBasicBlock &MBB,
1299 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog,
1300 std::optional<MachineFunction::DebugInstrOperandPair> InstrNum) const {
1301 bool IsLargeCodeModel = MF.getTarget().getCodeModel() == CodeModel::Large;
1302
1303 // FIXME: Add indirect thunk support and remove this.
1304 if (Is64Bit && IsLargeCodeModel && STI.useIndirectThunkCalls())
1305 report_fatal_error(reason: "Emitting stack probe calls on 64-bit with the large "
1306 "code model and indirect thunks not yet implemented.");
1307
1308 assert(MBB.computeRegisterLiveness(TRI, X86::EFLAGS, MBBI) !=
1309 MachineBasicBlock::LQR_Live &&
1310 "Stack probe calls will clobber live EFLAGS.");
1311
1312 unsigned CallOp;
1313 if (Is64Bit)
1314 CallOp = IsLargeCodeModel ? X86::CALL64r : X86::CALL64pcrel32;
1315 else
1316 CallOp = X86::CALLpcrel32;
1317
1318 StringRef Symbol = STI.getTargetLowering()->getStackProbeSymbolName(MF);
1319
1320 MachineInstrBuilder CI;
1321 MachineBasicBlock::iterator ExpansionMBBI = std::prev(x: MBBI);
1322
1323 // All current stack probes take AX and SP as input, clobber flags, and
1324 // preserve all registers. x86_64 probes leave RSP unmodified.
1325 if (Is64Bit && MF.getTarget().getCodeModel() == CodeModel::Large) {
1326 // For the large code model, we have to call through a register. Use R11,
1327 // as it is scratch in all supported calling conventions.
1328 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64ri), DestReg: X86::R11)
1329 .addExternalSymbol(FnName: MF.createExternalSymbolName(Name: Symbol));
1330 CI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: CallOp)).addReg(RegNo: X86::R11);
1331 } else {
1332 CI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: CallOp))
1333 .addExternalSymbol(FnName: MF.createExternalSymbolName(Name: Symbol));
1334 }
1335
1336 unsigned AX = Uses64BitFramePtr ? X86::RAX : X86::EAX;
1337 unsigned SP = Uses64BitFramePtr ? X86::RSP : X86::ESP;
1338 bool EmitSPSub = STI.isTargetWin64() || !STI.isOSWindows();
1339
1340 CI.addReg(RegNo: AX, Flags: RegState::Implicit)
1341 .addReg(RegNo: SP, Flags: RegState::Implicit)
1342 .addReg(RegNo: AX, Flags: RegState::Define | RegState::Implicit |
1343 getDeadRegState(B: !EmitSPSub))
1344 .addReg(RegNo: SP, Flags: RegState::Define | RegState::Implicit)
1345 .addReg(RegNo: X86::EFLAGS,
1346 Flags: RegState::Define | RegState::Implicit | RegState::Dead);
1347
1348 MachineInstr *ModInst = CI;
1349 if (EmitSPSub) {
1350 // MSVC x32's _chkstk and cygwin/mingw's _alloca adjust %esp themselves.
1351 // MSVC x64's __chkstk and cygwin/mingw's ___chkstk_ms do not adjust %rsp
1352 // themselves. They also does not clobber %rax so we can reuse it when
1353 // adjusting %rsp.
1354 // All other platforms do not specify a particular ABI for the stack probe
1355 // function, so we arbitrarily define it to not adjust %esp/%rsp itself.
1356 ModInst =
1357 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: getSUBrrOpcode(IsLP64: Uses64BitFramePtr)), DestReg: SP)
1358 .addReg(RegNo: SP)
1359 .addReg(RegNo: AX)
1360 .setOperandDead(3); // implicit-def $eflags
1361 }
1362
1363 // DebugInfo variable locations -- if there's an instruction number for the
1364 // allocation (i.e., DYN_ALLOC_*), substitute it for the instruction that
1365 // modifies SP.
1366 if (InstrNum) {
1367 if (EmitSPSub) {
1368 // Label destination operand of the subtract.
1369 MF.makeDebugValueSubstitution(*InstrNum,
1370 {ModInst->getDebugInstrNum(), 0});
1371 } else {
1372 // Label the call. The operand number is the penultimate operand, zero
1373 // based.
1374 unsigned SPDefOperand = ModInst->getNumOperands() - 2;
1375 MF.makeDebugValueSubstitution(
1376 *InstrNum, {ModInst->getDebugInstrNum(), SPDefOperand});
1377 }
1378 }
1379
1380 if (InProlog) {
1381 // Apply the frame setup flag to all inserted instrs.
1382 for (++ExpansionMBBI; ExpansionMBBI != MBBI; ++ExpansionMBBI)
1383 ExpansionMBBI->setFlag(MachineInstr::FrameSetup);
1384 }
1385}
1386
1387static unsigned calculateSetFPREG(uint64_t SPAdjust) {
1388 // Win64 ABI has a less restrictive limitation of 240; 128 works equally well
1389 // and might require smaller successive adjustments.
1390 const uint64_t Win64MaxSEHOffset = 128;
1391 uint64_t SEHFrameOffset = std::min(a: SPAdjust, b: Win64MaxSEHOffset);
1392 // Win64 ABI requires 16-byte alignment for the UWOP_SET_FPREG opcode.
1393 return SEHFrameOffset & -16;
1394}
1395
1396// If we're forcing a stack realignment we can't rely on just the frame
1397// info, we need to know the ABI stack alignment as well in case we
1398// have a call out. Otherwise just make sure we have some alignment - we'll
1399// go with the minimum SlotSize.
1400uint64_t
1401X86FrameLowering::calculateMaxStackAlign(const MachineFunction &MF) const {
1402 const MachineFrameInfo &MFI = MF.getFrameInfo();
1403 Align MaxAlign = MFI.getMaxAlign(); // Desired stack alignment.
1404 Align StackAlign = getStackAlign();
1405 bool HasRealign = MF.getFunction().hasFnAttribute(Kind: "stackrealign");
1406 if (HasRealign) {
1407 if (MFI.hasCalls())
1408 MaxAlign = (StackAlign > MaxAlign) ? StackAlign : MaxAlign;
1409 else if (MaxAlign < SlotSize)
1410 MaxAlign = Align(SlotSize);
1411 }
1412
1413 if (!Is64Bit && MF.getFunction().getCallingConv() == CallingConv::X86_INTR)
1414 MaxAlign = std::max(a: MaxAlign, b: Align(16));
1415
1416 return MaxAlign.value();
1417}
1418
1419void X86FrameLowering::BuildStackAlignAND(MachineBasicBlock &MBB,
1420 MachineBasicBlock::iterator MBBI,
1421 const DebugLoc &DL, Register Reg,
1422 uint64_t MaxAlign) const {
1423 uint64_t Val = -MaxAlign;
1424 unsigned AndOp = getANDriOpcode(IsLP64: Uses64BitFramePtr, Imm: Val);
1425
1426 MachineFunction &MF = *MBB.getParent();
1427 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
1428 const X86TargetLowering &TLI = *STI.getTargetLowering();
1429 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
1430 const bool EmitInlineStackProbe = TLI.hasInlineStackProbe(MF);
1431
1432 // We want to make sure that (in worst case) less than StackProbeSize bytes
1433 // are not probed after the AND. This assumption is used in
1434 // emitStackProbeInlineGeneric.
1435 if (Reg == StackPtr && EmitInlineStackProbe && MaxAlign >= StackProbeSize) {
1436 {
1437 NumFrameLoopProbe++;
1438 MachineBasicBlock *entryMBB =
1439 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
1440 MachineBasicBlock *headMBB =
1441 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
1442 MachineBasicBlock *bodyMBB =
1443 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
1444 MachineBasicBlock *footMBB =
1445 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
1446
1447 MachineFunction::iterator MBBIter = MBB.getIterator();
1448 MF.insert(MBBI: MBBIter, MBB: entryMBB);
1449 MF.insert(MBBI: MBBIter, MBB: headMBB);
1450 MF.insert(MBBI: MBBIter, MBB: bodyMBB);
1451 MF.insert(MBBI: MBBIter, MBB: footMBB);
1452 const unsigned MovMIOpc = Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
1453 Register FinalStackProbed = Uses64BitFramePtr ? X86::R11
1454 : Is64Bit ? X86::R11D
1455 : X86::EAX;
1456
1457 // Setup entry block
1458 {
1459
1460 entryMBB->splice(Where: entryMBB->end(), Other: &MBB, From: MBB.begin(), To: MBBI);
1461 BuildMI(BB: entryMBB, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: FinalStackProbed)
1462 .addReg(RegNo: StackPtr)
1463 .setMIFlag(MachineInstr::FrameSetup);
1464 MachineInstr *MI =
1465 BuildMI(BB: entryMBB, MIMD: DL, MCID: TII.get(Opcode: AndOp), DestReg: FinalStackProbed)
1466 .addReg(RegNo: FinalStackProbed)
1467 .addImm(Val)
1468 .setMIFlag(MachineInstr::FrameSetup);
1469
1470 // The EFLAGS implicit def is dead.
1471 MI->getOperand(i: 3).setIsDead();
1472
1473 BuildMI(BB: entryMBB, MIMD: DL,
1474 MCID: TII.get(Opcode: Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
1475 .addReg(RegNo: FinalStackProbed)
1476 .addReg(RegNo: StackPtr)
1477 .setMIFlag(MachineInstr::FrameSetup);
1478 BuildMI(BB: entryMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
1479 .addMBB(MBB: &MBB)
1480 .addImm(Val: X86::COND_E)
1481 .setMIFlag(MachineInstr::FrameSetup);
1482 entryMBB->addSuccessor(Succ: headMBB);
1483 entryMBB->addSuccessor(Succ: &MBB);
1484 }
1485
1486 // Loop entry block
1487
1488 {
1489 const unsigned SUBOpc = getSUBriOpcode(IsLP64: Uses64BitFramePtr);
1490 BuildMI(BB: headMBB, MIMD: DL, MCID: TII.get(Opcode: SUBOpc), DestReg: StackPtr)
1491 .addReg(RegNo: StackPtr)
1492 .addImm(Val: StackProbeSize)
1493 .setMIFlag(MachineInstr::FrameSetup)
1494 .setOperandDead(3); // implicit-def $eflags
1495
1496 BuildMI(BB: headMBB, MIMD: DL,
1497 MCID: TII.get(Opcode: Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
1498 .addReg(RegNo: StackPtr)
1499 .addReg(RegNo: FinalStackProbed)
1500 .setMIFlag(MachineInstr::FrameSetup);
1501
1502 // jump to the footer if StackPtr < FinalStackProbed
1503 BuildMI(BB: headMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
1504 .addMBB(MBB: footMBB)
1505 .addImm(Val: X86::COND_B)
1506 .setMIFlag(MachineInstr::FrameSetup);
1507
1508 headMBB->addSuccessor(Succ: bodyMBB);
1509 headMBB->addSuccessor(Succ: footMBB);
1510 }
1511
1512 // setup loop body
1513 {
1514 addRegOffset(MIB: BuildMI(BB: bodyMBB, MIMD: DL, MCID: TII.get(Opcode: MovMIOpc))
1515 .setMIFlag(MachineInstr::FrameSetup),
1516 Reg: StackPtr, Offset: 0)
1517 .addImm(Val: 0)
1518 .setMIFlag(MachineInstr::FrameSetup);
1519
1520 const unsigned SUBOpc = getSUBriOpcode(IsLP64: Uses64BitFramePtr);
1521 BuildMI(BB: bodyMBB, MIMD: DL, MCID: TII.get(Opcode: SUBOpc), DestReg: StackPtr)
1522 .addReg(RegNo: StackPtr)
1523 .addImm(Val: StackProbeSize)
1524 .setMIFlag(MachineInstr::FrameSetup)
1525 .setOperandDead(3); // implicit-def $eflags
1526
1527 // cmp with stack pointer bound
1528 BuildMI(BB: bodyMBB, MIMD: DL,
1529 MCID: TII.get(Opcode: Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
1530 .addReg(RegNo: FinalStackProbed)
1531 .addReg(RegNo: StackPtr)
1532 .setMIFlag(MachineInstr::FrameSetup);
1533
1534 // jump back while FinalStackProbed < StackPtr
1535 BuildMI(BB: bodyMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
1536 .addMBB(MBB: bodyMBB)
1537 .addImm(Val: X86::COND_B)
1538 .setMIFlag(MachineInstr::FrameSetup);
1539 bodyMBB->addSuccessor(Succ: bodyMBB);
1540 bodyMBB->addSuccessor(Succ: footMBB);
1541 }
1542
1543 // setup loop footer
1544 {
1545 BuildMI(BB: footMBB, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: StackPtr)
1546 .addReg(RegNo: FinalStackProbed)
1547 .setMIFlag(MachineInstr::FrameSetup);
1548 addRegOffset(MIB: BuildMI(BB: footMBB, MIMD: DL, MCID: TII.get(Opcode: MovMIOpc))
1549 .setMIFlag(MachineInstr::FrameSetup),
1550 Reg: StackPtr, Offset: 0)
1551 .addImm(Val: 0)
1552 .setMIFlag(MachineInstr::FrameSetup);
1553 footMBB->addSuccessor(Succ: &MBB);
1554 }
1555
1556 fullyRecomputeLiveIns(MBBs: {footMBB, bodyMBB, headMBB, &MBB});
1557 }
1558 } else {
1559 MachineInstr *MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: AndOp), DestReg: Reg)
1560 .addReg(RegNo: Reg)
1561 .addImm(Val)
1562 .setMIFlag(MachineInstr::FrameSetup);
1563
1564 // The EFLAGS implicit def is dead.
1565 MI->getOperand(i: 3).setIsDead();
1566 }
1567}
1568
1569bool X86FrameLowering::has128ByteRedZone(const MachineFunction &MF) const {
1570 // x86-64 (non Win64) has a 128 byte red zone which is guaranteed not to be
1571 // clobbered by any interrupt handler.
1572 assert(&STI == &MF.getSubtarget<X86Subtarget>() &&
1573 "MF used frame lowering for wrong subtarget");
1574 const Function &Fn = MF.getFunction();
1575 const bool IsWin64CC = STI.isCallingConvWin64(CC: Fn.getCallingConv());
1576 return Is64Bit && !IsWin64CC && !Fn.hasFnAttribute(Kind: Attribute::NoRedZone);
1577}
1578
1579bool X86FrameLowering::needsDwarfCFI(const MachineFunction &MF) const {
1580 return !MF.usesWindowsUnwindInfo() && MF.needsFrameMoves();
1581}
1582
1583/// Return true if an opcode is part of the REP group of instructions
1584static bool isOpcodeRep(unsigned Opcode) {
1585 switch (Opcode) {
1586 case X86::REPNE_PREFIX:
1587 case X86::REP_MOVSB_32:
1588 case X86::REP_MOVSB_64:
1589 case X86::REP_MOVSD_32:
1590 case X86::REP_MOVSD_64:
1591 case X86::REP_MOVSQ_32:
1592 case X86::REP_MOVSQ_64:
1593 case X86::REP_MOVSW_32:
1594 case X86::REP_MOVSW_64:
1595 case X86::REP_PREFIX:
1596 case X86::REP_STOSB_32:
1597 case X86::REP_STOSB_64:
1598 case X86::REP_STOSD_32:
1599 case X86::REP_STOSD_64:
1600 case X86::REP_STOSQ_32:
1601 case X86::REP_STOSQ_64:
1602 case X86::REP_STOSW_32:
1603 case X86::REP_STOSW_64:
1604 return true;
1605 default:
1606 break;
1607 }
1608 return false;
1609}
1610
1611/// Returns the number of bytes between the end of the fixed and callee-save
1612/// area and the first local object, which PEI leaves as padding when it aligns
1613/// the local objects relative to the incoming stack pointer.
1614static uint64_t getUnusedLocalAreaPadding(const MachineFrameInfo &MFI) {
1615 int64_t FixedEnd = 0;
1616 int64_t LocalsTop = std::numeric_limits<int64_t>::max();
1617 for (int I : seq(Begin: MFI.getObjectIndexBegin(), End: MFI.getObjectIndexEnd())) {
1618 if (MFI.isDeadObjectIndex(ObjectIdx: I) || MFI.isVariableSizedObjectIndex(ObjectIdx: I) ||
1619 MFI.getStackID(ObjectIdx: I) != TargetStackID::Default)
1620 continue;
1621
1622 int64_t ObjOffset = MFI.getObjectOffset(ObjectIdx: I);
1623 int64_t ObjSize = MFI.getObjectSize(ObjectIdx: I);
1624 // Offsets are negative, measured from the incoming stack pointer.
1625 if (MFI.isFixedObjectIndex(ObjectIdx: I))
1626 FixedEnd = std::max(a: FixedEnd, b: -ObjOffset);
1627 else
1628 LocalsTop = std::min<int64_t>(a: LocalsTop, b: -ObjOffset - ObjSize);
1629 }
1630 if (LocalsTop == std::numeric_limits<int64_t>::max())
1631 return 0;
1632
1633 assert(LocalsTop >= FixedEnd && "Local object overlaps the fixed area");
1634 return LocalsTop - FixedEnd;
1635}
1636
1637/// emitPrologue - Push callee-saved registers onto the stack, which
1638/// automatically adjust the stack pointer. Adjust the stack pointer to allocate
1639/// space for local variables. Also emit labels used by the exception handler to
1640/// generate the exception handling frames.
1641
1642/*
1643 Here's a gist of what gets emitted:
1644
1645 ; Establish frame pointer, if needed
1646 [if needs FP]
1647 push %rbp
1648 .cfi_def_cfa_offset 16
1649 .cfi_offset %rbp, -16
1650 .seh_pushreg %rpb
1651 mov %rsp, %rbp
1652 .cfi_def_cfa_register %rbp
1653
1654 ; Spill general-purpose registers
1655 [for all callee-saved GPRs]
1656 pushq %<reg>
1657 [if not needs FP]
1658 .cfi_def_cfa_offset (offset from RETADDR)
1659 .seh_pushreg %<reg>
1660
1661 ; If the required stack alignment > default stack alignment
1662 ; rsp needs to be re-aligned. This creates a "re-alignment gap"
1663 ; of unknown size in the stack frame.
1664 [if stack needs re-alignment]
1665 and $MASK, %rsp
1666
1667 ; Allocate space for locals
1668 [if target is Windows and allocated space > 4096 bytes]
1669 ; Windows needs special care for allocations larger
1670 ; than one page.
1671 mov $NNN, %rax
1672 call ___chkstk_ms/___chkstk
1673 sub %rax, %rsp
1674 [else]
1675 sub $NNN, %rsp
1676
1677 [if needs FP]
1678 .seh_stackalloc (size of XMM spill slots)
1679 .seh_setframe %rbp, SEHFrameOffset ; = size of all spill slots
1680 [else]
1681 .seh_stackalloc NNN
1682
1683 ; Spill XMMs
1684 ; Note, that while only Windows 64 ABI specifies XMMs as callee-preserved,
1685 ; they may get spilled on any platform, if the current function
1686 ; calls @llvm.eh.unwind.init
1687 [if needs FP]
1688 [for all callee-saved XMM registers]
1689 movaps %<xmm reg>, -MMM(%rbp)
1690 [for all callee-saved XMM registers]
1691 .seh_savexmm %<xmm reg>, (-MMM + SEHFrameOffset)
1692 ; i.e. the offset relative to (%rbp - SEHFrameOffset)
1693 [else]
1694 [for all callee-saved XMM registers]
1695 movaps %<xmm reg>, KKK(%rsp)
1696 [for all callee-saved XMM registers]
1697 .seh_savexmm %<xmm reg>, KKK
1698
1699 .seh_endprologue
1700
1701 [if needs base pointer]
1702 mov %rsp, %rbx
1703 [if needs to restore base pointer]
1704 mov %rsp, -MMM(%rbp)
1705
1706 ; Emit CFI info
1707 [if needs FP]
1708 [for all callee-saved registers]
1709 .cfi_offset %<reg>, (offset from %rbp)
1710 [else]
1711 .cfi_def_cfa_offset (offset from RETADDR)
1712 [for all callee-saved registers]
1713 .cfi_offset %<reg>, (offset from %rsp)
1714
1715 Notes:
1716 - .seh directives are emitted only for Windows 64 ABI
1717 - .cv_fpo directives are emitted on win32 when emitting CodeView
1718 - .cfi directives are emitted for all other ABIs
1719 - for 32-bit code, substitute %e?? registers for %r??
1720*/
1721
1722void X86FrameLowering::emitPrologue(MachineFunction &MF,
1723 MachineBasicBlock &MBB) const {
1724 assert(&STI == &MF.getSubtarget<X86Subtarget>() &&
1725 "MF used frame lowering for wrong subtarget");
1726 MachineBasicBlock::iterator MBBI = MBB.begin();
1727 MachineFrameInfo &MFI = MF.getFrameInfo();
1728 const Function &Fn = MF.getFunction();
1729 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
1730 uint64_t MaxAlign = calculateMaxStackAlign(MF); // Desired stack alignment.
1731 uint64_t StackSize = MFI.getStackSize(); // Number of bytes to allocate.
1732 bool IsFunclet = MBB.isEHFuncletEntry();
1733 EHPersonality Personality = EHPersonality::Unknown;
1734 if (Fn.hasPersonalityFn())
1735 Personality = classifyEHPersonality(Pers: Fn.getPersonalityFn());
1736 bool FnHasClrFunclet =
1737 MF.hasEHFunclets() && Personality == EHPersonality::CoreCLR;
1738 bool IsClrFunclet = IsFunclet && FnHasClrFunclet;
1739 bool HasFP = hasFP(MF);
1740 bool IsWin64Prologue = MF.usesWindowsUnwindInfo();
1741 bool NeedsWin64CFI = IsWin64Prologue && Fn.needsUnwindTableEntry();
1742 // FIXME: Emit FPO data for EH funclets.
1743 bool NeedsWinFPO = !IsFunclet && STI.isTargetWin32() &&
1744 MF.getFunction().getParent()->getCodeViewFlag();
1745 bool NeedsWinCFI = NeedsWin64CFI || NeedsWinFPO;
1746 bool NeedsDwarfCFI = needsDwarfCFI(MF);
1747 bool IsWin64UnwindV3 = NeedsWin64CFI && requireWinX64UnwindV3(MF);
1748 Register FramePtr = TRI->getFrameRegister(MF);
1749 const Register MachineFramePtr =
1750 STI.isTarget64BitILP32() ? Register(getX86SubSuperRegister(Reg: FramePtr, Size: 64))
1751 : FramePtr;
1752 Register BasePtr = TRI->getBaseRegister();
1753 bool HasWinCFI = false;
1754
1755 // Helpers to emit Windows x64 unwind SEH pseudos with the correct placement.
1756 // V1/V2: pseudo goes after the real instruction.
1757 // V3: pseudo goes before the real instruction.
1758 // Usage:
1759 // EmitSEHBefore([&]{ BuildMI(...SEH_PushReg...); });
1760 // BuildMI(... real instruction ...);
1761 // EmitSEHAfter([&]{ BuildMI(...SEH_PushReg...); });
1762 auto EmitSEHBefore = [&](auto EmitFn) {
1763 if (NeedsWinCFI && IsWin64UnwindV3) {
1764 HasWinCFI = true;
1765 EmitFn();
1766 }
1767 };
1768 auto EmitSEHAfter = [&](auto EmitFn) {
1769 if (NeedsWinCFI && !IsWin64UnwindV3) {
1770 HasWinCFI = true;
1771 EmitFn();
1772 }
1773 };
1774
1775 // Debug location must be unknown since the first debug location is used
1776 // to determine the end of the prologue.
1777 DebugLoc DL;
1778 Register ArgBaseReg;
1779
1780 // Emit extra prolog for argument stack slot reference.
1781 if (auto *MI = X86FI->getStackPtrSaveMI()) {
1782 // MI is lea instruction that created in X86ArgumentStackSlotPass.
1783 // Creat extra prolog for stack realignment.
1784 ArgBaseReg = MI->getOperand(i: 0).getReg();
1785 // leal 4(%esp), %basereg
1786 // .cfi_def_cfa %basereg, 0
1787 // andl $-128, %esp
1788 // pushl -4(%basereg)
1789 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Is64Bit ? X86::LEA64r : X86::LEA32r),
1790 DestReg: ArgBaseReg)
1791 .addUse(RegNo: StackPtr)
1792 .addImm(Val: 1)
1793 .addUse(RegNo: X86::NoRegister)
1794 .addImm(Val: SlotSize)
1795 .addUse(RegNo: X86::NoRegister)
1796 .setMIFlag(MachineInstr::FrameSetup);
1797 if (NeedsDwarfCFI) {
1798 // .cfi_def_cfa %basereg, 0
1799 unsigned DwarfStackPtr = TRI->getDwarfRegNum(Reg: ArgBaseReg, isEH: true);
1800 BuildCFI(MBB, MBBI, DL,
1801 CFIInst: MCCFIInstruction::cfiDefCfa(L: nullptr, Register: DwarfStackPtr, Offset: 0),
1802 Flag: MachineInstr::FrameSetup);
1803 }
1804 BuildStackAlignAND(MBB, MBBI, DL, Reg: StackPtr, MaxAlign);
1805 int64_t Offset = -(int64_t)SlotSize;
1806 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Is64Bit ? X86::PUSH64rmm : X86::PUSH32rmm))
1807 .addReg(RegNo: ArgBaseReg)
1808 .addImm(Val: 1)
1809 .addReg(RegNo: X86::NoRegister)
1810 .addImm(Val: Offset)
1811 .addReg(RegNo: X86::NoRegister)
1812 .setMIFlag(MachineInstr::FrameSetup);
1813 }
1814
1815 // Space reserved for stack-based arguments when making a (ABI-guaranteed)
1816 // tail call.
1817 unsigned TailCallArgReserveSize = -X86FI->getTCReturnAddrDelta();
1818 if (TailCallArgReserveSize && IsWin64Prologue)
1819 report_fatal_error(reason: "Can't handle guaranteed tail call under win64 yet");
1820
1821 const bool EmitStackProbeCall =
1822 STI.getTargetLowering()->hasStackProbeSymbol(MF);
1823 unsigned StackProbeSize = STI.getTargetLowering()->getStackProbeSize(MF);
1824
1825 if (HasFP && X86FI->hasSwiftAsyncContext()) {
1826 switch (MF.getTarget().Options.SwiftAsyncFramePointer) {
1827 case SwiftAsyncFramePointerMode::DeploymentBased:
1828 if (STI.swiftAsyncContextIsDynamicallySet()) {
1829 // The special symbol below is absolute and has a *value* suitable to be
1830 // combined with the frame pointer directly.
1831 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::OR64rm), DestReg: MachineFramePtr)
1832 .addUse(RegNo: MachineFramePtr)
1833 .addUse(RegNo: X86::RIP)
1834 .addImm(Val: 1)
1835 .addUse(RegNo: X86::NoRegister)
1836 .addExternalSymbol(FnName: "swift_async_extendedFramePointerFlags",
1837 TargetFlags: X86II::MO_GOTPCREL)
1838 .addUse(RegNo: X86::NoRegister);
1839 break;
1840 }
1841 [[fallthrough]];
1842
1843 case SwiftAsyncFramePointerMode::Always:
1844 assert(
1845 !IsWin64Prologue &&
1846 "win64 prologue does not set the bit 60 in the saved frame pointer");
1847 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::BTS64ri8), DestReg: MachineFramePtr)
1848 .addUse(RegNo: MachineFramePtr)
1849 .addImm(Val: 60)
1850 .setMIFlag(MachineInstr::FrameSetup);
1851 break;
1852
1853 case SwiftAsyncFramePointerMode::Never:
1854 break;
1855 }
1856 }
1857
1858 // Re-align the stack on 64-bit if the x86-interrupt calling convention is
1859 // used and an error code was pushed, since the x86-64 ABI requires a 16-byte
1860 // stack alignment.
1861 if (Fn.getCallingConv() == CallingConv::X86_INTR && Is64Bit &&
1862 Fn.arg_size() == 2) {
1863 // Update the stack pointer by pushing a register. This is the instruction
1864 // emitted that would be end up being emitted by a call to `emitSPUpdate`.
1865 // Hard-coding the update to a push avoids emitting a second
1866 // `STACKALLOC_W_PROBING` instruction in the save block: We know that stack
1867 // probing isn't needed anyways for an 8-byte update.
1868 // Pushing a register leaves us in a similar situation to a regular
1869 // function call where we know that the address at (rsp-8) is writeable.
1870 // That way we avoid any off-by-ones with stack probing for additional
1871 // stack pointer updates later on.
1872 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH64r))
1873 .addReg(RegNo: X86::RAX, Flags: RegState::Undef)
1874 .setMIFlag(MachineInstr::FrameSetup);
1875 }
1876
1877 // If this is x86-64 and the Red Zone is not disabled, if we are a leaf
1878 // function, and use up to 128 bytes of stack space, don't have a frame
1879 // pointer, calls, or dynamic alloca then we do not need to adjust the
1880 // stack pointer (we fit in the Red Zone). We also check that we don't
1881 // push and pop from the stack.
1882 if (has128ByteRedZone(MF) && !TRI->hasStackRealignment(MF) &&
1883 !MFI.hasVarSizedObjects() && // No dynamic alloca.
1884 !MFI.adjustsStack() && // No calls.
1885 !EmitStackProbeCall && // No stack probes.
1886 !MFI.hasCopyImplyingStackAdjustment() && // Don't push and pop.
1887 !MF.shouldSplitStack()) { // Regular stack
1888 uint64_t MinSize =
1889 X86FI->getCalleeSavedFrameSize() - X86FI->getTCReturnAddrDelta();
1890 if (HasFP)
1891 MinSize += SlotSize;
1892 X86FI->setUsesRedZone(MinSize > 0 || StackSize > 0);
1893 StackSize = std::max(a: MinSize, b: StackSize > 128 ? StackSize - 128 : 0);
1894 MFI.setStackSize(StackSize);
1895 }
1896
1897 // Insert stack pointer adjustment for later moving of return addr. Only
1898 // applies to tail call optimized functions where the callee argument stack
1899 // size is bigger than the callers.
1900 if (TailCallArgReserveSize != 0) {
1901 BuildStackAdjustment(MBB, MBBI, DL, Offset: -(int)TailCallArgReserveSize,
1902 /*InEpilogue=*/false)
1903 .setMIFlag(MachineInstr::FrameSetup);
1904 }
1905
1906 // Mapping for machine moves:
1907 //
1908 // DST: VirtualFP AND
1909 // SRC: VirtualFP => DW_CFA_def_cfa_offset
1910 // ELSE => DW_CFA_def_cfa
1911 //
1912 // SRC: VirtualFP AND
1913 // DST: Register => DW_CFA_def_cfa_register
1914 //
1915 // ELSE
1916 // OFFSET < 0 => DW_CFA_offset_extended_sf
1917 // REG < 64 => DW_CFA_offset + Reg
1918 // ELSE => DW_CFA_offset_extended
1919
1920 uint64_t NumBytes = 0;
1921 int stackGrowth = -SlotSize;
1922
1923 // Find the funclet establisher parameter
1924 MCRegister Establisher;
1925 if (IsClrFunclet)
1926 Establisher = Uses64BitFramePtr ? X86::RCX : X86::ECX;
1927 else if (IsFunclet)
1928 Establisher = Uses64BitFramePtr ? X86::RDX : X86::EDX;
1929
1930 if (IsWin64Prologue && IsFunclet && !IsClrFunclet) {
1931 // Immediately spill establisher into the home slot.
1932 // The runtime cares about this.
1933 // MOV64mr %rdx, 16(%rsp)
1934 unsigned MOVmr = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr;
1935 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: MOVmr)), Reg: StackPtr, Offset: 16)
1936 .addReg(RegNo: Establisher)
1937 .setMIFlag(MachineInstr::FrameSetup);
1938 MBB.addLiveIn(PhysReg: Establisher);
1939 }
1940
1941 if (HasFP) {
1942 assert(MF.getRegInfo().isReserved(MachineFramePtr) && "FP reserved");
1943
1944 // Calculate required stack adjustment.
1945 uint64_t FrameSize = StackSize - SlotSize;
1946 NumBytes =
1947 FrameSize - (X86FI->getCalleeSavedFrameSize() + TailCallArgReserveSize);
1948
1949 // Callee-saved registers are pushed on stack before the stack is realigned,
1950 // and the realignment itself already provides the local objects' alignment,
1951 // so leave out the padding PEI put in front of them for it.
1952 if (TRI->hasStackRealignment(MF) && !IsWin64Prologue) {
1953 uint64_t Padding = getUnusedLocalAreaPadding(MFI);
1954 assert(Padding <= NumBytes && "Padding exceeds the local area");
1955 NumBytes = alignTo(Value: NumBytes - Padding, Align: MaxAlign);
1956 }
1957
1958 // Save EBP/RBP into the appropriate stack slot.
1959 auto EmitSEHPushFramePtr = [&]() {
1960 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_PushReg))
1961 .addImm(Val: FramePtr)
1962 .setMIFlag(MachineInstr::FrameSetup);
1963 };
1964 EmitSEHBefore(EmitSEHPushFramePtr);
1965 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
1966 MCID: TII.get(Opcode: getPUSHOpcode(ST: MF.getSubtarget<X86Subtarget>())))
1967 .addReg(RegNo: MachineFramePtr, Flags: RegState::Kill)
1968 .setMIFlag(MachineInstr::FrameSetup);
1969 EmitSEHAfter(EmitSEHPushFramePtr);
1970
1971 if (NeedsDwarfCFI && !ArgBaseReg.isValid()) {
1972 // Mark the place where EBP/RBP was saved.
1973 // Define the current CFA rule to use the provided offset.
1974 assert(StackSize);
1975 BuildCFI(MBB, MBBI, DL,
1976 CFIInst: MCCFIInstruction::cfiDefCfaOffset(
1977 L: nullptr, Offset: -2 * stackGrowth + (int)TailCallArgReserveSize),
1978 Flag: MachineInstr::FrameSetup);
1979
1980 // Change the rule for the FramePtr to be an "offset" rule.
1981 unsigned DwarfFramePtr = TRI->getDwarfRegNum(Reg: MachineFramePtr, isEH: true);
1982 BuildCFI(MBB, MBBI, DL,
1983 CFIInst: MCCFIInstruction::createOffset(L: nullptr, Register: DwarfFramePtr,
1984 Offset: 2 * stackGrowth -
1985 (int)TailCallArgReserveSize),
1986 Flag: MachineInstr::FrameSetup);
1987 }
1988
1989 if (!IsFunclet) {
1990 if (X86FI->hasSwiftAsyncContext()) {
1991 assert(!IsWin64Prologue &&
1992 "win64 prologue does not store async context right below rbp");
1993 const auto &Attrs = MF.getFunction().getAttributes();
1994
1995 // Before we update the live frame pointer we have to ensure there's a
1996 // valid (or null) asynchronous context in its slot just before FP in
1997 // the frame record, so store it now.
1998 auto EmitSEHPushR14 = [&]() {
1999 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_PushReg))
2000 .addImm(Val: X86::R14)
2001 .setMIFlag(MachineInstr::FrameSetup);
2002 };
2003 EmitSEHBefore(EmitSEHPushR14);
2004 if (Attrs.hasAttrSomewhere(Kind: Attribute::SwiftAsync)) {
2005 // We have an initial context in r14, store it just before the frame
2006 // pointer.
2007 MBB.addLiveIn(PhysReg: X86::R14);
2008 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH64r))
2009 .addReg(RegNo: X86::R14)
2010 .setMIFlag(MachineInstr::FrameSetup);
2011 } else {
2012 // No initial context, store null so that there's no pointer that
2013 // could be misused.
2014 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH64i32))
2015 .addImm(Val: 0)
2016 .setMIFlag(MachineInstr::FrameSetup);
2017 }
2018
2019 // Update CFA offset for the async-context push.
2020 if (NeedsDwarfCFI && !ArgBaseReg.isValid()) {
2021 BuildCFI(
2022 MBB, MBBI, DL,
2023 CFIInst: MCCFIInstruction::createAdjustCfaOffset(L: nullptr, Adjustment: -stackGrowth),
2024 Flag: MachineInstr::FrameSetup);
2025 }
2026
2027 EmitSEHAfter(EmitSEHPushR14);
2028
2029 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::LEA64r), DestReg: FramePtr)
2030 .addUse(RegNo: X86::RSP)
2031 .addImm(Val: 1)
2032 .addUse(RegNo: X86::NoRegister)
2033 .addImm(Val: 8)
2034 .addUse(RegNo: X86::NoRegister)
2035 .setMIFlag(MachineInstr::FrameSetup);
2036
2037 // Switch to an FP-relative CFA before adjusting RSP below.
2038 if (NeedsDwarfCFI && !ArgBaseReg.isValid()) {
2039 unsigned DwarfFramePtr = TRI->getDwarfRegNum(Reg: MachineFramePtr, isEH: true);
2040 BuildCFI(MBB, MBBI, DL,
2041 CFIInst: MCCFIInstruction::cfiDefCfa(L: nullptr, Register: DwarfFramePtr,
2042 Offset: -2 * stackGrowth +
2043 (int)TailCallArgReserveSize),
2044 Flag: MachineInstr::FrameSetup);
2045 }
2046
2047 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SUB64ri32), DestReg: X86::RSP)
2048 .addUse(RegNo: X86::RSP)
2049 .addImm(Val: 8)
2050 .setMIFlag(MachineInstr::FrameSetup);
2051 }
2052
2053 if (!IsWin64Prologue && !IsFunclet) {
2054 // Update EBP with the new base value.
2055 if (!X86FI->hasSwiftAsyncContext())
2056 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
2057 MCID: TII.get(Opcode: Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr),
2058 DestReg: FramePtr)
2059 .addReg(RegNo: StackPtr)
2060 .setMIFlag(MachineInstr::FrameSetup);
2061
2062 if (NeedsDwarfCFI) {
2063 if (ArgBaseReg.isValid()) {
2064 SmallString<64> CfaExpr;
2065 CfaExpr.push_back(Elt: dwarf::DW_CFA_expression);
2066 uint8_t buffer[16];
2067 unsigned DwarfReg = TRI->getDwarfRegNum(Reg: MachineFramePtr, isEH: true);
2068 CfaExpr.append(in_start: buffer, in_end: buffer + encodeULEB128(Value: DwarfReg, p: buffer));
2069 CfaExpr.push_back(Elt: 2);
2070 CfaExpr.push_back(Elt: (uint8_t)(dwarf::DW_OP_breg0 + DwarfReg));
2071 CfaExpr.push_back(Elt: 0);
2072 // DW_CFA_expression: reg5 DW_OP_breg5 +0
2073 BuildCFI(MBB, MBBI, DL,
2074 CFIInst: MCCFIInstruction::createEscape(L: nullptr, Vals: CfaExpr.str()),
2075 Flag: MachineInstr::FrameSetup);
2076 } else if (!X86FI->hasSwiftAsyncContext()) {
2077 // Mark effective beginning of when frame pointer becomes valid.
2078 // Define the current CFA to use the EBP/RBP register.
2079 unsigned DwarfFramePtr = TRI->getDwarfRegNum(Reg: MachineFramePtr, isEH: true);
2080 BuildCFI(
2081 MBB, MBBI, DL,
2082 CFIInst: MCCFIInstruction::createDefCfaRegister(L: nullptr, Register: DwarfFramePtr),
2083 Flag: MachineInstr::FrameSetup);
2084 }
2085 }
2086
2087 if (NeedsWinFPO) {
2088 // .cv_fpo_setframe $FramePtr
2089 // NeedsWinFPO is Win32 only, so we're never using Unwind v3, hence it
2090 // is always inserted afterwards.
2091 assert(!IsWin64UnwindV3);
2092 HasWinCFI = true;
2093 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_SetFrame))
2094 .addImm(Val: FramePtr)
2095 .addImm(Val: 0)
2096 .setMIFlag(MachineInstr::FrameSetup);
2097 }
2098 }
2099 }
2100 } else {
2101 assert(!IsFunclet && "funclets without FPs not yet implemented");
2102 NumBytes =
2103 StackSize - (X86FI->getCalleeSavedFrameSize() + TailCallArgReserveSize);
2104 }
2105
2106 // Update the offset adjustment, which is mainly used by codeview to translate
2107 // from ESP to VFRAME relative local variable offsets.
2108 if (!IsFunclet) {
2109 if (HasFP && TRI->hasStackRealignment(MF))
2110 MFI.setOffsetAdjustment(-NumBytes);
2111 else
2112 MFI.setOffsetAdjustment(-StackSize);
2113 }
2114
2115 // For EH funclets, only allocate enough space for outgoing calls. Save the
2116 // NumBytes value that we would've used for the parent frame.
2117 unsigned ParentFrameNumBytes = NumBytes;
2118 if (IsFunclet)
2119 NumBytes = getWinEHFuncletFrameSize(MF);
2120
2121 // Skip the callee-saved push instructions.
2122 bool PushedRegs = false;
2123 int StackOffset = 2 * stackGrowth;
2124 MachineBasicBlock::const_iterator LastCSPush = MBBI;
2125 auto IsCSPush = [&](const MachineBasicBlock::iterator &MBBI) {
2126 if (MBBI == MBB.end() || !MBBI->getFlag(Flag: MachineInstr::FrameSetup))
2127 return false;
2128 unsigned Opc = MBBI->getOpcode();
2129 return Opc == X86::PUSH32r || Opc == X86::PUSH64r || Opc == X86::PUSHP64r ||
2130 Opc == X86::PUSH2 || Opc == X86::PUSH2P;
2131 };
2132
2133 while (IsCSPush(MBBI)) {
2134 PushedRegs = true;
2135 Register Reg = MBBI->getOperand(i: 0).getReg();
2136 LastCSPush = MBBI;
2137 unsigned Opc = LastCSPush->getOpcode();
2138 bool IsPush2 = Opc == X86::PUSH2 || Opc == X86::PUSH2P;
2139
2140 // V3: emit SEH pseudo before the real instruction.
2141 EmitSEHBefore([&]() {
2142 if (IsPush2) {
2143 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_Push2Regs))
2144 .addImm(Val: Reg)
2145 .addImm(Val: LastCSPush->getOperand(i: 1).getReg())
2146 .setMIFlag(MachineInstr::FrameSetup);
2147 } else {
2148 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_PushReg))
2149 .addImm(Val: Reg)
2150 .setMIFlag(MachineInstr::FrameSetup);
2151 }
2152 });
2153 ++MBBI;
2154
2155 if (!HasFP && NeedsDwarfCFI) {
2156 // Mark callee-saved push instruction.
2157 // Define the current CFA rule to use the provided offset.
2158 assert(StackSize);
2159 // Compared to push, push2 introduces more stack offset (one more
2160 // register).
2161 if (IsPush2)
2162 StackOffset += stackGrowth;
2163 BuildCFI(MBB, MBBI, DL,
2164 CFIInst: MCCFIInstruction::cfiDefCfaOffset(L: nullptr, Offset: -StackOffset),
2165 Flag: MachineInstr::FrameSetup);
2166 StackOffset += stackGrowth;
2167 }
2168
2169 // V1/V2: emit SEH pseudo after the real instruction.
2170 EmitSEHAfter([&]() {
2171 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_PushReg))
2172 .addImm(Val: Reg)
2173 .setMIFlag(MachineInstr::FrameSetup);
2174 if (IsPush2)
2175 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_PushReg))
2176 .addImm(Val: LastCSPush->getOperand(i: 1).getReg())
2177 .setMIFlag(MachineInstr::FrameSetup);
2178 });
2179 }
2180
2181 // Realign stack after we pushed callee-saved registers (so that we'll be
2182 // able to calculate their offsets from the frame pointer).
2183 // Don't do this for Win64, it needs to realign the stack after the prologue.
2184 if (!IsWin64Prologue && !IsFunclet && TRI->hasStackRealignment(MF) &&
2185 !ArgBaseReg.isValid()) {
2186 assert(HasFP && "There should be a frame pointer if stack is realigned.");
2187 auto EmitSEHStackAlign = [&]() {
2188 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_StackAlign))
2189 .addImm(Val: MaxAlign)
2190 .setMIFlag(MachineInstr::FrameSetup);
2191 };
2192 EmitSEHBefore(EmitSEHStackAlign);
2193 BuildStackAlignAND(MBB, MBBI, DL, Reg: StackPtr, MaxAlign);
2194 EmitSEHAfter(EmitSEHStackAlign);
2195 }
2196
2197 // If there is an SUB32ri of ESP immediately before this instruction, merge
2198 // the two. This can be the case when tail call elimination is enabled and
2199 // the callee has more arguments than the caller.
2200 NumBytes = mergeSPUpdates(
2201 MBB, MBBI, CalcNewOffset: [NumBytes](int64_t Offset) { return NumBytes - Offset; },
2202 doMergeWithPrevious: true);
2203
2204 // Adjust stack pointer: ESP -= numbytes.
2205
2206 // Windows and cygwin/mingw require a prologue helper routine when allocating
2207 // more than 4K bytes on the stack. Windows uses __chkstk and cygwin/mingw
2208 // uses __alloca. __alloca and the 32-bit version of __chkstk will probe the
2209 // stack and adjust the stack pointer in one go. The 64-bit version of
2210 // __chkstk is only responsible for probing the stack. The 64-bit prologue is
2211 // responsible for adjusting the stack pointer. Touching the stack at 4K
2212 // increments is necessary to ensure that the guard pages used by the OS
2213 // virtual memory manager are allocated in correct sequence.
2214 uint64_t AlignedNumBytes = NumBytes;
2215 if (IsWin64Prologue && !IsFunclet && TRI->hasStackRealignment(MF))
2216 AlignedNumBytes = alignTo(Value: AlignedNumBytes, Align: MaxAlign);
2217
2218 auto EmitSEHStackAlloc = [&]() {
2219 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_StackAlloc))
2220 .addImm(Val: NumBytes)
2221 .setMIFlag(MachineInstr::FrameSetup);
2222 };
2223 if (NumBytes)
2224 EmitSEHBefore(EmitSEHStackAlloc);
2225
2226 if (AlignedNumBytes >= StackProbeSize && EmitStackProbeCall) {
2227 assert(!X86FI->getUsesRedZone() &&
2228 "The Red Zone is not accounted for in stack probes");
2229
2230 // Check whether EAX is livein for this block.
2231 bool isEAXAlive = isEAXLiveIn(MBB);
2232
2233 if (isEAXAlive) {
2234 if (Is64Bit) {
2235 // Save RAX
2236 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH64r))
2237 .addReg(RegNo: X86::RAX, Flags: RegState::Kill)
2238 .setMIFlag(MachineInstr::FrameSetup);
2239 } else {
2240 // Save EAX
2241 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH32r))
2242 .addReg(RegNo: X86::EAX, Flags: RegState::Kill)
2243 .setMIFlag(MachineInstr::FrameSetup);
2244 }
2245 }
2246
2247 if (Is64Bit) {
2248 // Handle the 64-bit Windows ABI case where we need to call __chkstk.
2249 // Function prologue is responsible for adjusting the stack pointer.
2250 int64_t Alloc = isEAXAlive ? NumBytes - 8 : NumBytes;
2251 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::getMOVriOpcode(Use64BitReg: Is64Bit, Imm: Alloc)),
2252 DestReg: X86::RAX)
2253 .addImm(Val: Alloc)
2254 .setMIFlag(MachineInstr::FrameSetup);
2255 } else {
2256 // Allocate NumBytes-4 bytes on stack in case of isEAXAlive.
2257 // We'll also use 4 already allocated bytes for EAX.
2258 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV32ri), DestReg: X86::EAX)
2259 .addImm(Val: isEAXAlive ? NumBytes - 4 : NumBytes)
2260 .setMIFlag(MachineInstr::FrameSetup);
2261 }
2262
2263 // Call __chkstk, __chkstk_ms, or __alloca.
2264 emitStackProbe(MF, MBB, MBBI, DL, InProlog: true);
2265
2266 if (isEAXAlive) {
2267 // Restore RAX/EAX
2268 MachineInstr *MI;
2269 if (Is64Bit)
2270 MI = addRegOffset(MIB: BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64rm), DestReg: X86::RAX),
2271 Reg: StackPtr, Offset: NumBytes - 8);
2272 else
2273 MI = addRegOffset(MIB: BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: X86::MOV32rm), DestReg: X86::EAX),
2274 Reg: StackPtr, Offset: NumBytes - 4);
2275 MI->setFlag(MachineInstr::FrameSetup);
2276 MBB.insert(I: MBBI, MI);
2277 }
2278 } else if (NumBytes) {
2279 emitSPUpdate(MBB, MBBI, DL, NumBytes: -(int64_t)NumBytes, /*InEpilogue=*/false);
2280 }
2281
2282 if (NumBytes)
2283 EmitSEHAfter(EmitSEHStackAlloc);
2284
2285 int SEHFrameOffset = 0;
2286 Register SPOrEstablisher;
2287 if (IsFunclet) {
2288 if (IsClrFunclet) {
2289 // The establisher parameter passed to a CLR funclet is actually a pointer
2290 // to the (mostly empty) frame of its nearest enclosing funclet; we have
2291 // to find the root function establisher frame by loading the PSPSym from
2292 // the intermediate frame.
2293 unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF);
2294 MachinePointerInfo NoInfo;
2295 MBB.addLiveIn(PhysReg: Establisher);
2296 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64rm), DestReg: Establisher),
2297 Reg: Establisher, Offset: PSPSlotOffset)
2298 .addMemOperand(MMO: MF.getMachineMemOperand(
2299 PtrInfo: NoInfo, F: MachineMemOperand::MOLoad, Size: SlotSize, BaseAlignment: Align(SlotSize)));
2300 ;
2301 // Save the root establisher back into the current funclet's (mostly
2302 // empty) frame, in case a sub-funclet or the GC needs it.
2303 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64mr)), Reg: StackPtr,
2304 Offset: PSPSlotOffset)
2305 .addReg(RegNo: Establisher)
2306 .addMemOperand(MMO: MF.getMachineMemOperand(
2307 PtrInfo: NoInfo,
2308 F: MachineMemOperand::MOStore | MachineMemOperand::MOVolatile,
2309 Size: SlotSize, BaseAlignment: Align(SlotSize)));
2310 }
2311 SPOrEstablisher = Establisher;
2312 } else {
2313 SPOrEstablisher = StackPtr;
2314 }
2315
2316 if (IsWin64Prologue && HasFP) {
2317 // Set RBP to a small fixed offset from RSP. In the funclet case, we base
2318 // this calculation on the incoming establisher, which holds the value of
2319 // RSP from the parent frame at the end of the prologue.
2320 SEHFrameOffset = calculateSetFPREG(SPAdjust: ParentFrameNumBytes);
2321
2322 // If this is not a funclet, emit the CFI describing our frame pointer.
2323 if (NeedsWinCFI && !IsFunclet) {
2324 assert(!NeedsWinFPO && "this setframe incompatible with FPO data");
2325 HasWinCFI = true;
2326 if (isAsynchronousEHPersonality(Pers: Personality) || MF.hasEHFunclets()) {
2327 if (TRI->hasBasePointer(MF))
2328 MF.getWinEHFuncInfo()->SEHSetFrameOffset =
2329 getWinEHParentFrameOffset(MF);
2330 else
2331 MF.getWinEHFuncInfo()->SEHSetFrameOffset = SEHFrameOffset;
2332 }
2333 }
2334
2335 auto EmitSEHSetFrame = [&]() {
2336 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_SetFrame))
2337 .addImm(Val: FramePtr)
2338 .addImm(Val: SEHFrameOffset)
2339 .setMIFlag(MachineInstr::FrameSetup);
2340 };
2341
2342 if (!IsFunclet)
2343 EmitSEHBefore(EmitSEHSetFrame);
2344
2345 if (SEHFrameOffset)
2346 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::LEA64r), DestReg: FramePtr),
2347 Reg: SPOrEstablisher, Offset: SEHFrameOffset);
2348 else
2349 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64rr), DestReg: FramePtr)
2350 .addReg(RegNo: SPOrEstablisher);
2351
2352 if (!IsFunclet)
2353 EmitSEHAfter(EmitSEHSetFrame);
2354 } else if (IsFunclet && STI.is32Bit()) {
2355 // Reset EBP / ESI to something good for funclets.
2356 MBBI = restoreWin32EHStackPointers(MBB, MBBI, DL);
2357 // If we're a catch funclet, we can be returned to via catchret. Save ESP
2358 // into the registration node so that the runtime will restore it for us.
2359 if (!MBB.isCleanupFuncletEntry()) {
2360 assert(Personality == EHPersonality::MSVC_CXX);
2361 Register FrameReg;
2362 int FI = MF.getWinEHFuncInfo()->EHRegNodeFrameIndex;
2363 int64_t EHRegOffset = getFrameIndexReference(MF, FI, FrameReg).getFixed();
2364 // ESP is the first field, so no extra displacement is needed.
2365 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV32mr)), Reg: FrameReg,
2366 Offset: EHRegOffset)
2367 .addReg(RegNo: X86::ESP);
2368 }
2369 }
2370
2371 while (MBBI != MBB.end() && MBBI->getFlag(Flag: MachineInstr::FrameSetup)) {
2372 const MachineInstr &FrameInstr = *MBBI;
2373
2374 if (NeedsWinCFI) {
2375 int FI;
2376 if (Register Reg = TII.isStoreToStackSlot(MI: FrameInstr, FrameIndex&: FI)) {
2377 if (X86::FR64RegClass.contains(Reg)) {
2378 int Offset;
2379 Register IgnoredFrameReg;
2380 if (IsWin64Prologue && IsFunclet)
2381 Offset = getWin64EHFrameIndexRef(MF, FI, SPReg&: IgnoredFrameReg);
2382 else
2383 Offset =
2384 getFrameIndexReference(MF, FI, FrameReg&: IgnoredFrameReg).getFixed() +
2385 SEHFrameOffset;
2386
2387 assert(!NeedsWinFPO && "SEH_SaveXMM incompatible with FPO data");
2388 auto EmitSEHSaveXMM = [&]() {
2389 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_SaveXMM))
2390 .addImm(Val: Reg)
2391 .addImm(Val: Offset)
2392 .setMIFlag(MachineInstr::FrameSetup);
2393 };
2394 EmitSEHBefore(EmitSEHSaveXMM);
2395 ++MBBI;
2396 EmitSEHAfter(EmitSEHSaveXMM);
2397 continue;
2398 }
2399 }
2400 }
2401 ++MBBI;
2402 }
2403
2404 if (NeedsWinCFI && HasWinCFI) {
2405 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_EndPrologue))
2406 .setMIFlag(MachineInstr::FrameSetup);
2407 }
2408
2409 if (FnHasClrFunclet && !IsFunclet) {
2410 // Save the so-called Initial-SP (i.e. the value of the stack pointer
2411 // immediately after the prolog) into the PSPSlot so that funclets
2412 // and the GC can recover it.
2413 unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF);
2414 auto PSPInfo = MachinePointerInfo::getFixedStack(
2415 MF, FI: MF.getWinEHFuncInfo()->PSPSymFrameIdx);
2416 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64mr)), Reg: StackPtr,
2417 Offset: PSPSlotOffset)
2418 .addReg(RegNo: StackPtr)
2419 .addMemOperand(MMO: MF.getMachineMemOperand(
2420 PtrInfo: PSPInfo, F: MachineMemOperand::MOStore | MachineMemOperand::MOVolatile,
2421 Size: SlotSize, BaseAlignment: Align(SlotSize)));
2422 }
2423
2424 // Realign stack after we spilled callee-saved registers (so that we'll be
2425 // able to calculate their offsets from the frame pointer).
2426 // Win64 requires aligning the stack after the prologue.
2427 if (IsWin64Prologue && TRI->hasStackRealignment(MF)) {
2428 assert(HasFP && "There should be a frame pointer if stack is realigned.");
2429 BuildStackAlignAND(MBB, MBBI, DL, Reg: SPOrEstablisher, MaxAlign);
2430 }
2431
2432 // We already dealt with stack realignment and funclets above.
2433 if (IsFunclet && STI.is32Bit())
2434 return;
2435
2436 // If we need a base pointer, set it up here. It's whatever the value
2437 // of the stack pointer is at this point. Any variable size objects
2438 // will be allocated after this, so we can still use the base pointer
2439 // to reference locals.
2440 if (TRI->hasBasePointer(MF)) {
2441 // Update the base pointer with the current stack pointer.
2442 unsigned Opc = Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr;
2443 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: BasePtr)
2444 .addReg(RegNo: SPOrEstablisher)
2445 .setMIFlag(MachineInstr::FrameSetup);
2446 if (X86FI->getRestoreBasePointer()) {
2447 // Stash value of base pointer. Saving RSP instead of EBP shortens
2448 // dependence chain. Used by SjLj EH.
2449 unsigned Opm = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr;
2450 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opm)), Reg: FramePtr,
2451 Offset: X86FI->getRestoreBasePointerOffset())
2452 .addReg(RegNo: SPOrEstablisher)
2453 .setMIFlag(MachineInstr::FrameSetup);
2454 }
2455
2456 if (X86FI->getHasSEHFramePtrSave() && !IsFunclet) {
2457 // Stash the value of the frame pointer relative to the base pointer for
2458 // Win32 EH. This supports Win32 EH, which does the inverse of the above:
2459 // it recovers the frame pointer from the base pointer rather than the
2460 // other way around.
2461 unsigned Opm = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr;
2462 Register UsedReg;
2463 int Offset =
2464 getFrameIndexReference(MF, FI: X86FI->getSEHFramePtrSaveIndex(), FrameReg&: UsedReg)
2465 .getFixed();
2466 assert(UsedReg == BasePtr);
2467 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opm)), Reg: UsedReg, Offset)
2468 .addReg(RegNo: FramePtr)
2469 .setMIFlag(MachineInstr::FrameSetup);
2470 }
2471 }
2472 if (ArgBaseReg.isValid()) {
2473 // Save argument base pointer.
2474 auto *MI = X86FI->getStackPtrSaveMI();
2475 int FI = MI->getOperand(i: 1).getIndex();
2476 unsigned MOVmr = Is64Bit ? X86::MOV64mr : X86::MOV32mr;
2477 // movl %basereg, offset(%ebp)
2478 addFrameReference(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: MOVmr)), FI)
2479 .addReg(RegNo: ArgBaseReg)
2480 .setMIFlag(MachineInstr::FrameSetup);
2481 }
2482
2483 if (((!HasFP && NumBytes) || PushedRegs) && NeedsDwarfCFI) {
2484 // Mark end of stack pointer adjustment.
2485 if (!HasFP && NumBytes) {
2486 // Define the current CFA rule to use the provided offset.
2487 assert(StackSize);
2488 BuildCFI(
2489 MBB, MBBI, DL,
2490 CFIInst: MCCFIInstruction::cfiDefCfaOffset(L: nullptr, Offset: StackSize - stackGrowth),
2491 Flag: MachineInstr::FrameSetup);
2492 }
2493
2494 // Emit DWARF info specifying the offsets of the callee-saved registers.
2495 emitCalleeSavedFrameMoves(MBB, MBBI, DL, IsPrologue: true);
2496 }
2497
2498 // X86 Interrupt handling function cannot assume anything about the direction
2499 // flag (DF in EFLAGS register). Clear this flag by creating "cld" instruction
2500 // in each prologue of interrupt handler function.
2501 //
2502 // Create "cld" instruction only in these cases:
2503 // 1. The interrupt handling function uses any of the "rep" instructions.
2504 // 2. Interrupt handling function calls another function.
2505 // 3. If there are any inline asm blocks, as we do not know what they do
2506 //
2507 // TODO: We should also emit cld if we detect the use of std, but as of now,
2508 // the compiler does not even emit that instruction or even define it, so in
2509 // practice, this would only happen with inline asm, which we cover anyway.
2510 if (Fn.getCallingConv() == CallingConv::X86_INTR) {
2511 bool NeedsCLD = false;
2512
2513 for (const MachineBasicBlock &B : MF) {
2514 for (const MachineInstr &MI : B) {
2515 if (MI.isCall()) {
2516 NeedsCLD = true;
2517 break;
2518 }
2519
2520 if (isOpcodeRep(Opcode: MI.getOpcode())) {
2521 NeedsCLD = true;
2522 break;
2523 }
2524
2525 if (MI.isInlineAsm()) {
2526 // TODO: Parse asm for rep instructions or call sites?
2527 // For now, let's play it safe and emit a cld instruction
2528 // just in case.
2529 NeedsCLD = true;
2530 break;
2531 }
2532 }
2533 }
2534
2535 if (NeedsCLD) {
2536 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::CLD))
2537 .setMIFlag(MachineInstr::FrameSetup);
2538 }
2539 }
2540
2541 // At this point we know if the function has WinCFI or not.
2542 MF.setHasWinCFI(HasWinCFI);
2543}
2544
2545bool X86FrameLowering::canUseLEAForSPInEpilogue(
2546 const MachineFunction &MF) const {
2547 // We can't use LEA instructions for adjusting the stack pointer if we don't
2548 // have a frame pointer in the Win64 ABI. Only ADD instructions may be used
2549 // to deallocate the stack.
2550 // This means that we can use LEA for SP in two situations:
2551 // 1. We *aren't* using the Win64 ABI which means we are free to use LEA.
2552 // 2. We *have* a frame pointer which means we are permitted to use LEA.
2553 return !MF.usesWindowsUnwindInfo() || hasFP(MF);
2554}
2555
2556static bool isFuncletReturnInstr(MachineInstr &MI) {
2557 switch (MI.getOpcode()) {
2558 case X86::CATCHRET:
2559 case X86::CLEANUPRET:
2560 return true;
2561 default:
2562 return false;
2563 }
2564 llvm_unreachable("impossible");
2565}
2566
2567// CLR funclets use a special "Previous Stack Pointer Symbol" slot on the
2568// stack. It holds a pointer to the bottom of the root function frame. The
2569// establisher frame pointer passed to a nested funclet may point to the
2570// (mostly empty) frame of its parent funclet, but it will need to find
2571// the frame of the root function to access locals. To facilitate this,
2572// every funclet copies the pointer to the bottom of the root function
2573// frame into a PSPSym slot in its own (mostly empty) stack frame. Using the
2574// same offset for the PSPSym in the root function frame that's used in the
2575// funclets' frames allows each funclet to dynamically accept any ancestor
2576// frame as its establisher argument (the runtime doesn't guarantee the
2577// immediate parent for some reason lost to history), and also allows the GC,
2578// which uses the PSPSym for some bookkeeping, to find it in any funclet's
2579// frame with only a single offset reported for the entire method.
2580unsigned
2581X86FrameLowering::getPSPSlotOffsetFromSP(const MachineFunction &MF) const {
2582 const WinEHFuncInfo &Info = *MF.getWinEHFuncInfo();
2583 Register SPReg;
2584 int Offset = getFrameIndexReferencePreferSP(MF, FI: Info.PSPSymFrameIdx, FrameReg&: SPReg,
2585 /*IgnoreSPUpdates*/ true)
2586 .getFixed();
2587 assert(Offset >= 0 && SPReg == TRI->getStackRegister());
2588 return static_cast<unsigned>(Offset);
2589}
2590
2591unsigned
2592X86FrameLowering::getWinEHFuncletFrameSize(const MachineFunction &MF) const {
2593 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
2594 // This is the size of the pushed CSRs.
2595 unsigned CSSize = X86FI->getCalleeSavedFrameSize();
2596 // This is the size of callee saved XMMs.
2597 const auto &WinEHXMMSlotInfo = X86FI->getWinEHXMMSlotInfo();
2598 unsigned XMMSize =
2599 WinEHXMMSlotInfo.size() * TRI->getSpillSize(RC: X86::VR128RegClass);
2600 // This is the amount of stack a funclet needs to allocate.
2601 unsigned UsedSize;
2602 EHPersonality Personality =
2603 classifyEHPersonality(Pers: MF.getFunction().getPersonalityFn());
2604 if (Personality == EHPersonality::CoreCLR) {
2605 // CLR funclets need to hold enough space to include the PSPSym, at the
2606 // same offset from the stack pointer (immediately after the prolog) as it
2607 // resides at in the main function.
2608 UsedSize = getPSPSlotOffsetFromSP(MF) + SlotSize;
2609 } else {
2610 // Other funclets just need enough stack for outgoing call arguments.
2611 UsedSize = MF.getFrameInfo().getMaxCallFrameSize();
2612 }
2613 // RBP is not included in the callee saved register block. After pushing RBP,
2614 // everything is 16 byte aligned. Everything we allocate before an outgoing
2615 // call must also be 16 byte aligned.
2616 unsigned FrameSizeMinusRBP = alignTo(Size: CSSize + UsedSize, A: getStackAlign());
2617 // Subtract out the size of the callee saved registers. This is how much stack
2618 // each funclet will allocate.
2619 return FrameSizeMinusRBP + XMMSize - CSSize;
2620}
2621
2622static bool isTailCallOpcode(unsigned Opc) {
2623 return Opc == X86::TCRETURNri || Opc == X86::TCRETURN_WIN64ri ||
2624 Opc == X86::TCRETURN_HIPE32ri || Opc == X86::TCRETURNdi ||
2625 Opc == X86::TCRETURNmi || Opc == X86::TCRETURNri64 ||
2626 Opc == X86::TCRETURNri64_ImpCall || Opc == X86::TCRETURNdi64 ||
2627 Opc == X86::TCRETURNmi64 || Opc == X86::TCRETURN_WINmi64;
2628}
2629
2630void X86FrameLowering::emitEpilogue(MachineFunction &MF,
2631 MachineBasicBlock &MBB) const {
2632 const MachineFrameInfo &MFI = MF.getFrameInfo();
2633 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
2634 MachineBasicBlock::iterator Terminator = MBB.getFirstTerminator();
2635 MachineBasicBlock::iterator MBBI = Terminator;
2636 DebugLoc DL;
2637 if (MBBI != MBB.end())
2638 DL = MBBI->getDebugLoc();
2639 // standard x86_64 uses 64-bit frame/stack pointers, x32 - 32-bit.
2640 const bool Is64BitILP32 = STI.isTarget64BitILP32();
2641 Register FramePtr = TRI->getFrameRegister(MF);
2642 Register MachineFramePtr =
2643 Is64BitILP32 ? Register(getX86SubSuperRegister(Reg: FramePtr, Size: 64)) : FramePtr;
2644
2645 bool IsWin64Prologue = MF.usesWindowsUnwindInfo();
2646 bool NeedsWin64CFI =
2647 IsWin64Prologue && MF.getFunction().needsUnwindTableEntry();
2648 // For V3 unwind, epilog SEH pseudos are emitted inline before each
2649 // unwind-effecting instruction.
2650 bool IsWin64UnwindV3 =
2651 NeedsWin64CFI && MF.hasWinCFI() && requireWinX64UnwindV3(MF);
2652 bool IsFunclet = MBBI == MBB.end() ? false : isFuncletReturnInstr(MI&: *MBBI);
2653
2654 // Get the number of bytes to allocate from the FrameInfo.
2655 uint64_t StackSize = MFI.getStackSize();
2656 uint64_t MaxAlign = calculateMaxStackAlign(MF);
2657 unsigned CSSize = X86FI->getCalleeSavedFrameSize();
2658 unsigned TailCallArgReserveSize = -X86FI->getTCReturnAddrDelta();
2659 bool HasFP = hasFP(MF);
2660 uint64_t NumBytes = 0;
2661
2662 bool NeedsDwarfCFI = (!MF.getTarget().getTargetTriple().isOSDarwin() &&
2663 !MF.getTarget().getTargetTriple().isOSWindows() &&
2664 !MF.getTarget().getTargetTriple().isUEFI()) &&
2665 MF.needsFrameMoves();
2666
2667 Register ArgBaseReg;
2668 if (auto *MI = X86FI->getStackPtrSaveMI()) {
2669 unsigned Opc = X86::LEA32r;
2670 Register StackReg = X86::ESP;
2671 ArgBaseReg = MI->getOperand(i: 0).getReg();
2672 if (STI.is64Bit()) {
2673 Opc = X86::LEA64r;
2674 StackReg = X86::RSP;
2675 }
2676 // leal -4(%basereg), %esp
2677 // .cfi_def_cfa %esp, 4
2678 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: StackReg)
2679 .addUse(RegNo: ArgBaseReg)
2680 .addImm(Val: 1)
2681 .addUse(RegNo: X86::NoRegister)
2682 .addImm(Val: -(int64_t)SlotSize)
2683 .addUse(RegNo: X86::NoRegister)
2684 .setMIFlag(MachineInstr::FrameDestroy);
2685 if (NeedsDwarfCFI) {
2686 unsigned DwarfStackPtr = TRI->getDwarfRegNum(Reg: StackReg, isEH: true);
2687 BuildCFI(MBB, MBBI, DL,
2688 CFIInst: MCCFIInstruction::cfiDefCfa(L: nullptr, Register: DwarfStackPtr, Offset: SlotSize),
2689 Flag: MachineInstr::FrameDestroy);
2690 --MBBI;
2691 }
2692 --MBBI;
2693 }
2694
2695 if (IsFunclet) {
2696 assert(HasFP && "EH funclets without FP not yet implemented");
2697 NumBytes = getWinEHFuncletFrameSize(MF);
2698 } else if (HasFP) {
2699 // Calculate required stack adjustment.
2700 uint64_t FrameSize = StackSize - SlotSize;
2701 NumBytes = FrameSize - CSSize - TailCallArgReserveSize;
2702
2703 // Callee-saved registers were pushed on stack before the stack was
2704 // realigned.
2705 if (TRI->hasStackRealignment(MF) && !IsWin64Prologue)
2706 NumBytes = alignTo(Value: FrameSize, Align: MaxAlign);
2707 } else {
2708 NumBytes = StackSize - CSSize - TailCallArgReserveSize;
2709 }
2710 uint64_t SEHStackAllocAmt = NumBytes;
2711
2712 unsigned SEHFrameOffset = 0;
2713 if (IsWin64Prologue && HasFP)
2714 SEHFrameOffset = calculateSetFPREG(SPAdjust: SEHStackAllocAmt);
2715
2716 // AfterPop is the position to insert .cfi_restore.
2717 MachineBasicBlock::iterator AfterPop = MBBI;
2718 if (HasFP) {
2719 if (X86FI->hasSwiftAsyncContext()) {
2720 // Discard the context.
2721 int64_t Offset = mergeSPAdd(MBB, MBBI, AddOffset: 16, doMergeWithPrevious: true);
2722 emitSPUpdate(MBB, MBBI, DL, NumBytes: Offset, /*InEpilogue*/ true);
2723 }
2724 // Pop EBP.
2725 if (IsWin64UnwindV3)
2726 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_PushReg))
2727 .addImm(Val: FramePtr)
2728 .setMIFlag(MachineInstr::FrameDestroy);
2729 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
2730 MCID: TII.get(Opcode: getPOPOpcode(ST: MF.getSubtarget<X86Subtarget>())),
2731 DestReg: MachineFramePtr)
2732 .setMIFlag(MachineInstr::FrameDestroy);
2733
2734 // We need to reset FP to its untagged state on return. Bit 60 is currently
2735 // used to show the presence of an extended frame.
2736 if (X86FI->hasSwiftAsyncContext()) {
2737 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::BTR64ri8), DestReg: MachineFramePtr)
2738 .addUse(RegNo: MachineFramePtr)
2739 .addImm(Val: 60)
2740 .setMIFlag(MachineInstr::FrameDestroy);
2741 }
2742
2743 if (NeedsDwarfCFI) {
2744 if (!ArgBaseReg.isValid()) {
2745 unsigned DwarfStackPtr =
2746 TRI->getDwarfRegNum(Reg: Is64Bit ? X86::RSP : X86::ESP, isEH: true);
2747 BuildCFI(MBB, MBBI, DL,
2748 CFIInst: MCCFIInstruction::cfiDefCfa(L: nullptr, Register: DwarfStackPtr, Offset: SlotSize),
2749 Flag: MachineInstr::FrameDestroy);
2750 }
2751 if (!MBB.succ_empty() && !MBB.isReturnBlock()) {
2752 unsigned DwarfFramePtr = TRI->getDwarfRegNum(Reg: MachineFramePtr, isEH: true);
2753 BuildCFI(MBB, MBBI: AfterPop, DL,
2754 CFIInst: MCCFIInstruction::createRestore(L: nullptr, Register: DwarfFramePtr),
2755 Flag: MachineInstr::FrameDestroy);
2756 --MBBI;
2757 --AfterPop;
2758 }
2759 --MBBI;
2760 }
2761 }
2762
2763 MachineBasicBlock::iterator FirstCSPop = MBBI;
2764 // Skip the callee-saved pop instructions.
2765 while (MBBI != MBB.begin()) {
2766 MachineBasicBlock::iterator PI = std::prev(x: MBBI);
2767 unsigned Opc = PI->getOpcode();
2768
2769 if (Opc != X86::DBG_VALUE && !PI->isTerminator()) {
2770 if (!PI->getFlag(Flag: MachineInstr::FrameDestroy) ||
2771 (Opc != X86::POP32r && Opc != X86::POP64r && Opc != X86::BTR64ri8 &&
2772 Opc != X86::ADD64ri32 && Opc != X86::POPP64r && Opc != X86::POP2 &&
2773 Opc != X86::POP2P && Opc != X86::LEA64r && Opc != X86::SEH_PushReg &&
2774 Opc != X86::SEH_Push2Regs && Opc != X86::SEH_StackAlloc &&
2775 Opc != X86::ADD64ri32_NF))
2776 break;
2777 FirstCSPop = PI;
2778 }
2779
2780 --MBBI;
2781 }
2782 if (ArgBaseReg.isValid()) {
2783 // Restore argument base pointer.
2784 auto *MI = X86FI->getStackPtrSaveMI();
2785 int FI = MI->getOperand(i: 1).getIndex();
2786 unsigned MOVrm = Is64Bit ? X86::MOV64rm : X86::MOV32rm;
2787 // movl offset(%ebp), %basereg
2788 addFrameReference(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: MOVrm), DestReg: ArgBaseReg), FI)
2789 .setMIFlag(MachineInstr::FrameDestroy);
2790 }
2791 MBBI = FirstCSPop;
2792
2793 if (IsFunclet && Terminator->getOpcode() == X86::CATCHRET)
2794 emitCatchRetReturnValue(MBB, MBBI: FirstCSPop, CatchRet: &*Terminator);
2795
2796 if (MBBI != MBB.end())
2797 DL = MBBI->getDebugLoc();
2798 // If there is an ADD32ri or SUB32ri of ESP immediately before this
2799 // instruction, merge the two instructions.
2800 if (NumBytes || MFI.hasVarSizedObjects())
2801 NumBytes = mergeSPAdd(MBB, MBBI, AddOffset: NumBytes, doMergeWithPrevious: true);
2802
2803 if (IsWin64UnwindV3 && NeedsWin64CFI && MF.hasWinCFI()) {
2804 // Find the XMM restores that were tagged with FrameDestroy, now that we
2805 // know the offset we can emit the SEH pseudos for them.
2806 auto EpilogStart = MBBI;
2807 {
2808 auto ScanIt = MBBI;
2809 while (ScanIt != MBB.begin()) {
2810 auto PI = std::prev(x: ScanIt);
2811 int FI;
2812 if (PI->getFlag(Flag: MachineInstr::FrameDestroy) &&
2813 TII.isLoadFromStackSlot(MI: *PI, FrameIndex&: FI)) {
2814 Register Reg = PI->getOperand(i: 0).getReg();
2815 if (X86::FR64RegClass.contains(Reg)) {
2816 Register IgnoredFrameReg;
2817 int Offset =
2818 getFrameIndexReference(MF, FI, FrameReg&: IgnoredFrameReg).getFixed() +
2819 SEHFrameOffset;
2820 BuildMI(BB&: MBB, I: PI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_SaveXMM))
2821 .addImm(Val: Reg)
2822 .addImm(Val: Offset)
2823 .setMIFlag(MachineInstr::FrameDestroy);
2824 // std::prev(PI) is the SEH_SaveXMM we just inserted (before PI).
2825 // We start ScanIt from that point so that the next
2826 // std::prev(ScanIt) will examine the instruction before the pseudo,
2827 // i.e. the next potential XMM restore further up the block.
2828 EpilogStart = std::prev(x: PI);
2829 ScanIt = EpilogStart;
2830 continue;
2831 }
2832 }
2833 break;
2834 }
2835 }
2836
2837 // For V3, SEH_BeginEpilogue must be emitted before any epilog SEH pseudos.
2838 BuildMI(BB&: MBB, I: EpilogStart, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_BeginEpilogue))
2839 .setMIFlag(MachineInstr::FrameDestroy);
2840 }
2841
2842 // If dynamic alloca is used, then reset esp to point to the last callee-saved
2843 // slot before popping them off! Same applies for the case, when stack was
2844 // realigned. Don't do this if this was a funclet epilogue, since the funclets
2845 // will not do realignment or dynamic stack allocation.
2846 if (((TRI->hasStackRealignment(MF)) || MFI.hasVarSizedObjects()) &&
2847 !IsFunclet) {
2848 if (TRI->hasStackRealignment(MF))
2849 MBBI = FirstCSPop;
2850 uint64_t LEAAmount =
2851 IsWin64Prologue ? SEHStackAllocAmt - SEHFrameOffset : -CSSize;
2852
2853 if (X86FI->hasSwiftAsyncContext())
2854 LEAAmount -= 16;
2855
2856 // There are only two legal forms of epilogue:
2857 // - add SEHAllocationSize, %rsp
2858 // - lea SEHAllocationSize(%FramePtr), %rsp
2859 //
2860 // 'mov %FramePtr, %rsp' will not be recognized as an epilogue sequence.
2861 // However, we may use this sequence if we have a frame pointer because the
2862 // effects of the prologue can safely be undone.
2863 if (IsWin64UnwindV3) {
2864 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_SetFrame))
2865 .addImm(Val: FramePtr)
2866 .addImm(Val: SEHFrameOffset)
2867 .setMIFlag(MachineInstr::FrameDestroy);
2868 if (SEHStackAllocAmt)
2869 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_StackAlloc))
2870 .addImm(Val: SEHStackAllocAmt)
2871 .setMIFlag(MachineInstr::FrameDestroy);
2872 }
2873 if (LEAAmount != 0) {
2874 unsigned Opc = getLEArOpcode(IsLP64: Uses64BitFramePtr);
2875 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: StackPtr), Reg: FramePtr,
2876 Offset: LEAAmount);
2877 --MBBI;
2878 } else {
2879 unsigned Opc = (Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr);
2880 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: StackPtr).addReg(RegNo: FramePtr);
2881 --MBBI;
2882 }
2883 } else if (NumBytes) {
2884 // Adjust stack pointer back: ESP += numbytes.
2885 if (IsWin64UnwindV3)
2886 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_StackAlloc))
2887 .addImm(Val: NumBytes)
2888 .setMIFlag(MachineInstr::FrameDestroy);
2889 emitSPUpdate(MBB, MBBI, DL, NumBytes, /*InEpilogue=*/true);
2890 if (!HasFP && NeedsDwarfCFI) {
2891 // Define the current CFA rule to use the provided offset.
2892 BuildCFI(MBB, MBBI, DL,
2893 CFIInst: MCCFIInstruction::cfiDefCfaOffset(
2894 L: nullptr, Offset: CSSize + TailCallArgReserveSize + SlotSize),
2895 Flag: MachineInstr::FrameDestroy);
2896 }
2897 --MBBI;
2898 }
2899
2900 // For V1/V2, emit SEH_BeginEpilogue after stack restore code.
2901 if (!IsWin64UnwindV3 && NeedsWin64CFI && MF.hasWinCFI())
2902 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_BeginEpilogue))
2903 .setMIFlag(MachineInstr::FrameDestroy);
2904
2905 if (!HasFP && NeedsDwarfCFI) {
2906 MBBI = FirstCSPop;
2907 int64_t Offset = -(int64_t)CSSize - SlotSize;
2908 // Mark callee-saved pop instruction.
2909 // Define the current CFA rule to use the provided offset.
2910 while (MBBI != MBB.end()) {
2911 MachineBasicBlock::iterator PI = MBBI;
2912 unsigned Opc = PI->getOpcode();
2913 ++MBBI;
2914 if (Opc == X86::POP32r || Opc == X86::POP64r || Opc == X86::POPP64r ||
2915 Opc == X86::POP2 || Opc == X86::POP2P) {
2916 Offset += SlotSize;
2917 // Compared to pop, pop2 introduces more stack offset (one more
2918 // register).
2919 if (Opc == X86::POP2 || Opc == X86::POP2P)
2920 Offset += SlotSize;
2921 BuildCFI(MBB, MBBI, DL,
2922 CFIInst: MCCFIInstruction::cfiDefCfaOffset(L: nullptr, Offset: -Offset),
2923 Flag: MachineInstr::FrameDestroy);
2924 }
2925 }
2926 }
2927
2928 // Emit DWARF info specifying the restores of the callee-saved registers.
2929 // For epilogue with return inside or being other block without successor,
2930 // no need to generate .cfi_restore for callee-saved registers.
2931 if (NeedsDwarfCFI && !MBB.succ_empty())
2932 emitCalleeSavedFrameMoves(MBB, MBBI: AfterPop, DL, IsPrologue: false);
2933
2934 if (Terminator == MBB.end() || !isTailCallOpcode(Opc: Terminator->getOpcode())) {
2935 // Add the return addr area delta back since we are not tail calling.
2936 int64_t Delta = X86FI->getTCReturnAddrDelta();
2937 assert(Delta <= 0 && "TCDelta should never be positive");
2938 if (Delta) {
2939 // Check for possible merge with preceding ADD instruction.
2940 int64_t Offset = mergeSPAdd(MBB, MBBI&: Terminator, AddOffset: -Delta, doMergeWithPrevious: true);
2941 emitSPUpdate(MBB, MBBI&: Terminator, DL, NumBytes: Offset, /*InEpilogue=*/true);
2942 }
2943 }
2944
2945 // Emit tilerelease for AMX kernel.
2946 if (X86FI->getAMXProgModel() == AMXProgModelEnum::ManagedRA)
2947 BuildMI(BB&: MBB, I: Terminator, MIMD: DL, MCID: TII.get(Opcode: X86::TILERELEASE));
2948
2949 if (NeedsWin64CFI && MF.hasWinCFI())
2950 BuildMI(BB&: MBB, I: Terminator, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_EndEpilogue))
2951 .setMIFlag(MachineInstr::FrameDestroy);
2952}
2953
2954StackOffset X86FrameLowering::getFrameIndexReference(const MachineFunction &MF,
2955 int FI,
2956 Register &FrameReg) const {
2957 const MachineFrameInfo &MFI = MF.getFrameInfo();
2958
2959 bool IsFixed = MFI.isFixedObjectIndex(ObjectIdx: FI);
2960 // We can't calculate offset from frame pointer if the stack is realigned,
2961 // so enforce usage of stack/base pointer. The base pointer is used when we
2962 // have dynamic allocas in addition to dynamic realignment.
2963 if (TRI->hasBasePointer(MF))
2964 FrameReg = IsFixed ? TRI->getFramePtr() : TRI->getBaseRegister();
2965 else if (TRI->hasStackRealignment(MF))
2966 FrameReg = IsFixed ? TRI->getFramePtr() : TRI->getStackRegister();
2967 else
2968 FrameReg = TRI->getFrameRegister(MF);
2969
2970 // Offset will hold the offset from the stack pointer at function entry to the
2971 // object.
2972 // We need to factor in additional offsets applied during the prologue to the
2973 // frame, base, and stack pointer depending on which is used.
2974 int64_t Offset = MFI.getObjectOffset(ObjectIdx: FI) - getOffsetOfLocalArea();
2975 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
2976 unsigned CSSize = X86FI->getCalleeSavedFrameSize();
2977 uint64_t StackSize = MFI.getStackSize();
2978 bool IsWin64Prologue = MF.usesWindowsUnwindInfo();
2979 int64_t FPDelta = 0;
2980
2981 // In an x86 interrupt, remove the offset we added to account for the return
2982 // address from any stack object allocated in the caller's frame. Interrupts
2983 // do not have a standard return address. Fixed objects in the current frame,
2984 // such as SSE register spills, should not get this treatment.
2985 if (MF.getFunction().getCallingConv() == CallingConv::X86_INTR &&
2986 Offset >= 0) {
2987 Offset += getOffsetOfLocalArea();
2988 }
2989
2990 if (IsWin64Prologue) {
2991 assert(!MFI.hasCalls() || (StackSize % 16) == 8);
2992
2993 // Calculate required stack adjustment.
2994 uint64_t FrameSize = StackSize - SlotSize;
2995 // If required, include space for extra hidden slot for stashing base
2996 // pointer.
2997 if (X86FI->getRestoreBasePointer())
2998 FrameSize += SlotSize;
2999 uint64_t NumBytes = FrameSize - CSSize;
3000
3001 uint64_t SEHFrameOffset = calculateSetFPREG(SPAdjust: NumBytes);
3002 if (FI && FI == X86FI->getFAIndex())
3003 return StackOffset::getFixed(Fixed: -SEHFrameOffset);
3004
3005 // FPDelta is the offset from the "traditional" FP location of the old base
3006 // pointer followed by return address and the location required by the
3007 // restricted Win64 prologue.
3008 // Add FPDelta to all offsets below that go through the frame pointer.
3009 FPDelta = FrameSize - SEHFrameOffset;
3010 assert((!MFI.hasCalls() || (FPDelta % 16) == 0) &&
3011 "FPDelta isn't aligned per the Win64 ABI!");
3012 }
3013
3014 if (FrameReg == TRI->getFramePtr()) {
3015 // Skip saved EBP/RBP
3016 Offset += SlotSize;
3017
3018 // Account for restricted Windows prologue.
3019 Offset += FPDelta;
3020
3021 // Skip the RETADDR move area
3022 int TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta();
3023 if (TailCallReturnAddrDelta < 0)
3024 Offset -= TailCallReturnAddrDelta;
3025
3026 return StackOffset::getFixed(Fixed: Offset);
3027 }
3028
3029 // FrameReg is either the stack pointer or a base pointer. But the base is
3030 // located at the end of the statically known StackSize so the distinction
3031 // doesn't really matter.
3032 if (TRI->hasStackRealignment(MF) || TRI->hasBasePointer(MF))
3033 assert(isAligned(MFI.getObjectAlign(FI), -(Offset + StackSize)));
3034 return StackOffset::getFixed(Fixed: Offset + StackSize);
3035}
3036
3037int X86FrameLowering::getWin64EHFrameIndexRef(const MachineFunction &MF, int FI,
3038 Register &FrameReg) const {
3039 const MachineFrameInfo &MFI = MF.getFrameInfo();
3040 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
3041 const auto &WinEHXMMSlotInfo = X86FI->getWinEHXMMSlotInfo();
3042 const auto it = WinEHXMMSlotInfo.find(Val: FI);
3043
3044 if (it == WinEHXMMSlotInfo.end())
3045 return getFrameIndexReference(MF, FI, FrameReg).getFixed();
3046
3047 FrameReg = TRI->getStackRegister();
3048 return alignDown(Value: MFI.getMaxCallFrameSize(), Align: getStackAlign().value()) +
3049 it->second;
3050}
3051
3052StackOffset
3053X86FrameLowering::getFrameIndexReferenceSP(const MachineFunction &MF, int FI,
3054 Register &FrameReg,
3055 int Adjustment) const {
3056 const MachineFrameInfo &MFI = MF.getFrameInfo();
3057 FrameReg = TRI->getStackRegister();
3058 return StackOffset::getFixed(Fixed: MFI.getObjectOffset(ObjectIdx: FI) -
3059 getOffsetOfLocalArea() + Adjustment);
3060}
3061
3062StackOffset
3063X86FrameLowering::getFrameIndexReferencePreferSP(const MachineFunction &MF,
3064 int FI, Register &FrameReg,
3065 bool IgnoreSPUpdates) const {
3066
3067 const MachineFrameInfo &MFI = MF.getFrameInfo();
3068 // Does not include any dynamic realign.
3069 const uint64_t StackSize = MFI.getStackSize();
3070 // LLVM arranges the stack as follows:
3071 // ...
3072 // ARG2
3073 // ARG1
3074 // RETADDR
3075 // PUSH RBP <-- RBP points here
3076 // PUSH CSRs
3077 // ~~~~~~~ <-- possible stack realignment (non-win64)
3078 // ...
3079 // STACK OBJECTS
3080 // ... <-- RSP after prologue points here
3081 // ~~~~~~~ <-- possible stack realignment (win64)
3082 //
3083 // if (hasVarSizedObjects()):
3084 // ... <-- "base pointer" (ESI/RBX) points here
3085 // DYNAMIC ALLOCAS
3086 // ... <-- RSP points here
3087 //
3088 // Case 1: In the simple case of no stack realignment and no dynamic
3089 // allocas, both "fixed" stack objects (arguments and CSRs) are addressable
3090 // with fixed offsets from RSP.
3091 //
3092 // Case 2: In the case of stack realignment with no dynamic allocas, fixed
3093 // stack objects are addressed with RBP and regular stack objects with RSP.
3094 //
3095 // Case 3: In the case of dynamic allocas and stack realignment, RSP is used
3096 // to address stack arguments for outgoing calls and nothing else. The "base
3097 // pointer" points to local variables, and RBP points to fixed objects.
3098 //
3099 // In cases 2 and 3, we can only answer for non-fixed stack objects, and the
3100 // answer we give is relative to the SP after the prologue, and not the
3101 // SP in the middle of the function.
3102
3103 if (MFI.isFixedObjectIndex(ObjectIdx: FI) && TRI->hasStackRealignment(MF) &&
3104 !STI.isTargetWin64())
3105 return getFrameIndexReference(MF, FI, FrameReg);
3106
3107 // If !hasReservedCallFrame the function might have SP adjustement in the
3108 // body. So, even though the offset is statically known, it depends on where
3109 // we are in the function.
3110 if (!IgnoreSPUpdates && !hasReservedCallFrame(MF))
3111 return getFrameIndexReference(MF, FI, FrameReg);
3112
3113 // We don't handle tail calls, and shouldn't be seeing them either.
3114 assert(MF.getInfo<X86MachineFunctionInfo>()->getTCReturnAddrDelta() >= 0 &&
3115 "we don't handle this case!");
3116
3117 // This is how the math works out:
3118 //
3119 // %rsp grows (i.e. gets lower) left to right. Each box below is
3120 // one word (eight bytes). Obj0 is the stack slot we're trying to
3121 // get to.
3122 //
3123 // ----------------------------------
3124 // | BP | Obj0 | Obj1 | ... | ObjN |
3125 // ----------------------------------
3126 // ^ ^ ^ ^
3127 // A B C E
3128 //
3129 // A is the incoming stack pointer.
3130 // (B - A) is the local area offset (-8 for x86-64) [1]
3131 // (C - A) is the Offset returned by MFI.getObjectOffset for Obj0 [2]
3132 //
3133 // |(E - B)| is the StackSize (absolute value, positive). For a
3134 // stack that grown down, this works out to be (B - E). [3]
3135 //
3136 // E is also the value of %rsp after stack has been set up, and we
3137 // want (C - E) -- the value we can add to %rsp to get to Obj0. Now
3138 // (C - E) == (C - A) - (B - A) + (B - E)
3139 // { Using [1], [2] and [3] above }
3140 // == getObjectOffset - LocalAreaOffset + StackSize
3141
3142 return getFrameIndexReferenceSP(MF, FI, FrameReg, Adjustment: StackSize);
3143}
3144
3145bool X86FrameLowering::assignCalleeSavedSpillSlots(
3146 MachineFunction &MF, const TargetRegisterInfo *TRI,
3147 std::vector<CalleeSavedInfo> &CSI) const {
3148 MachineFrameInfo &MFI = MF.getFrameInfo();
3149 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
3150
3151 unsigned CalleeSavedFrameSize = 0;
3152 unsigned XMMCalleeSavedFrameSize = 0;
3153 auto &WinEHXMMSlotInfo = X86FI->getWinEHXMMSlotInfo();
3154 int SpillSlotOffset = getOffsetOfLocalArea() + X86FI->getTCReturnAddrDelta();
3155
3156 int64_t TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta();
3157
3158 if (TailCallReturnAddrDelta < 0) {
3159 // create RETURNADDR area
3160 // arg
3161 // arg
3162 // RETADDR
3163 // { ...
3164 // RETADDR area
3165 // ...
3166 // }
3167 // [EBP]
3168 MFI.CreateFixedObject(Size: -TailCallReturnAddrDelta,
3169 SPOffset: TailCallReturnAddrDelta - SlotSize, IsImmutable: true);
3170 }
3171
3172 // Spill the BasePtr if it's used.
3173 if (this->TRI->hasBasePointer(MF)) {
3174 // Allocate a spill slot for EBP if we have a base pointer and EH funclets.
3175 if (MF.hasEHFunclets()) {
3176 int FI = MFI.CreateSpillStackObject(Size: SlotSize, Alignment: Align(SlotSize));
3177 X86FI->setHasSEHFramePtrSave(true);
3178 X86FI->setSEHFramePtrSaveIndex(FI);
3179 }
3180 }
3181
3182 bool IsFPRemovedFromCSI = false;
3183 if (hasFP(MF)) {
3184 // emitPrologue always spills frame register the first thing.
3185 SpillSlotOffset -= SlotSize;
3186 MFI.CreateFixedSpillStackObject(Size: SlotSize, SPOffset: SpillSlotOffset);
3187
3188 // The async context lives directly before the frame pointer, and we
3189 // allocate a second slot to preserve stack alignment.
3190 if (X86FI->hasSwiftAsyncContext()) {
3191 SpillSlotOffset -= SlotSize;
3192 MFI.CreateFixedSpillStackObject(Size: SlotSize, SPOffset: SpillSlotOffset);
3193 SpillSlotOffset -= SlotSize;
3194 }
3195
3196 // Since emitPrologue and emitEpilogue will handle spilling and restoring of
3197 // the frame register, we can delete it from CSI list and not have to worry
3198 // about avoiding it later.
3199 Register FPReg = TRI->getFrameRegister(MF);
3200 for (unsigned i = 0; i < CSI.size(); ++i) {
3201 if (TRI->regsOverlap(RegA: CSI[i].getReg(), RegB: FPReg)) {
3202 CSI.erase(position: CSI.begin() + i);
3203 IsFPRemovedFromCSI = true;
3204 break;
3205 }
3206 }
3207 }
3208
3209 // Strategy:
3210 // 1. Use push2 when
3211 // a) number of CSR > 1 if no need padding
3212 // b) number of CSR > 2 if need padding
3213 // c) stack alignment >= 16 bytes
3214 // 2. When the number of CSR push is odd
3215 // a. Start to use push2 from the 1st push if stack is 16B aligned.
3216 // b. Start to use push2 from the 2nd push if stack is not 16B aligned.
3217 // 3. When the number of CSR push is even, start to use push2 from the 1st
3218 // push and make the stack 16B aligned before the push
3219 unsigned NumRegsForPush2 = 0;
3220 if (STI.hasPush2Pop2() && getStackAlignment() >= 16) {
3221 unsigned NumCSGPR = llvm::count_if(Range&: CSI, P: [](const CalleeSavedInfo &I) {
3222 return X86::GR64RegClass.contains(Reg: I.getReg());
3223 });
3224 bool UsePush2Pop2 = !IsFPRemovedFromCSI ? NumCSGPR > 2 : NumCSGPR > 1;
3225 NumRegsForPush2 =
3226 UsePush2Pop2
3227 ? alignDown(Value: IsFPRemovedFromCSI ? NumCSGPR : NumCSGPR - 1, Align: 2)
3228 : 0;
3229 }
3230
3231 // Assign slots for GPRs. It increases frame size.
3232 for (CalleeSavedInfo &I : llvm::reverse(C&: CSI)) {
3233 MCRegister Reg = I.getReg();
3234
3235 if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg))
3236 continue;
3237
3238 // A CSR is a candidate for push2/pop2 when it's slot offset is 16B aligned
3239 // or only an odd number of registers in the candidates.
3240 if (X86FI->getNumCandidatesForPush2Pop2() < NumRegsForPush2 &&
3241 (SpillSlotOffset % 16 == 0 ||
3242 X86FI->getNumCandidatesForPush2Pop2() % 2))
3243 X86FI->addCandidateForPush2Pop2(Reg);
3244
3245 SpillSlotOffset -= SlotSize;
3246 CalleeSavedFrameSize += SlotSize;
3247
3248 int SlotIndex = MFI.CreateFixedSpillStackObject(Size: SlotSize, SPOffset: SpillSlotOffset);
3249 I.setFrameIdx(SlotIndex);
3250 }
3251
3252 // Adjust the offset of spill slot as we know the accurate callee saved frame
3253 // size.
3254 if (X86FI->getRestoreBasePointer()) {
3255 SpillSlotOffset -= SlotSize;
3256 CalleeSavedFrameSize += SlotSize;
3257
3258 MFI.CreateFixedSpillStackObject(Size: SlotSize, SPOffset: SpillSlotOffset);
3259 // TODO: saving the slot index is better?
3260 X86FI->setRestoreBasePointer(CalleeSavedFrameSize);
3261 }
3262 assert(X86FI->getNumCandidatesForPush2Pop2() % 2 == 0 &&
3263 "Expect even candidates for push2/pop2");
3264 if (X86FI->getNumCandidatesForPush2Pop2())
3265 ++NumFunctionUsingPush2Pop2;
3266 X86FI->setCalleeSavedFrameSize(CalleeSavedFrameSize);
3267 MFI.setCVBytesOfCalleeSavedRegisters(CalleeSavedFrameSize);
3268
3269 // Assign slots for XMMs.
3270 for (CalleeSavedInfo &I : llvm::reverse(C&: CSI)) {
3271 MCRegister Reg = I.getReg();
3272 if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg))
3273 continue;
3274
3275 const TargetRegisterClass *RC = getCalleeSavedSpillRC(Reg, STI, TRI: *TRI);
3276 unsigned Size = TRI->getSpillSize(RC: *RC);
3277 // The slot is fixed relative to the incoming stack pointer, so it cannot be
3278 // aligned beyond what that guarantees, even if the stack is realigned.
3279 Align Alignment = std::min(a: TRI->getSpillAlign(RC: *RC), b: getStackAlign());
3280 // ensure alignment
3281 assert(SpillSlotOffset < 0 && "SpillSlotOffset should always < 0 on X86");
3282 SpillSlotOffset = -alignTo(Size: -SpillSlotOffset, A: Alignment);
3283
3284 // spill into slot
3285 SpillSlotOffset -= Size;
3286 int SlotIndex = MFI.CreateFixedSpillStackObject(Size, SPOffset: SpillSlotOffset);
3287 I.setFrameIdx(SlotIndex);
3288 MFI.ensureMaxAlignment(Alignment);
3289
3290 // Save the start offset and size of XMM in stack frame for funclets.
3291 if (X86::VR128RegClass.contains(Reg)) {
3292 WinEHXMMSlotInfo[SlotIndex] = XMMCalleeSavedFrameSize;
3293 XMMCalleeSavedFrameSize += Size;
3294 }
3295 }
3296
3297 return true;
3298}
3299
3300bool X86FrameLowering::spillCalleeSavedRegisters(
3301 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
3302 ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
3303 DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
3304
3305 // Don't save CSRs in 32-bit EH funclets. The caller saves EBX, EBP, ESI, EDI
3306 // for us, and there are no XMM CSRs on Win32.
3307 if (MBB.isEHFuncletEntry() && STI.is32Bit() && STI.isOSWindows())
3308 return true;
3309
3310 // Push GPRs. It increases frame size.
3311 const MachineFunction &MF = *MBB.getParent();
3312 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
3313
3314 // Update LiveIn of the basic block and decide whether we can add a kill flag
3315 // to the use.
3316 auto UpdateLiveInCheckCanKill = [&](Register Reg) {
3317 const MachineRegisterInfo &MRI = MF.getRegInfo();
3318 // Do not set a kill flag on values that are also marked as live-in. This
3319 // happens with the @llvm-returnaddress intrinsic and with arguments
3320 // passed in callee saved registers.
3321 // Omitting the kill flags is conservatively correct even if the live-in
3322 // is not used after all.
3323 if (MRI.isLiveIn(Reg))
3324 return false;
3325 MBB.addLiveIn(PhysReg: Reg);
3326 // Check if any subregister is live-in
3327 for (MCRegAliasIterator AReg(Reg, TRI, false); AReg.isValid(); ++AReg)
3328 if (MRI.isLiveIn(Reg: *AReg))
3329 return false;
3330 return true;
3331 };
3332 auto UpdateLiveInGetKillRegState = [&](Register Reg) {
3333 return getKillRegState(B: UpdateLiveInCheckCanKill(Reg));
3334 };
3335
3336 for (auto RI = CSI.rbegin(), RE = CSI.rend(); RI != RE; ++RI) {
3337 MCRegister Reg = RI->getReg();
3338 if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg))
3339 continue;
3340
3341 if (X86FI->isCandidateForPush2Pop2(Reg)) {
3342 MCRegister Reg2 = (++RI)->getReg();
3343 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: getPUSH2Opcode(ST: STI)))
3344 .addReg(RegNo: Reg, Flags: UpdateLiveInGetKillRegState(Reg))
3345 .addReg(RegNo: Reg2, Flags: UpdateLiveInGetKillRegState(Reg2))
3346 .setMIFlag(MachineInstr::FrameSetup);
3347 } else {
3348 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: getPUSHOpcode(ST: STI)))
3349 .addReg(RegNo: Reg, Flags: UpdateLiveInGetKillRegState(Reg))
3350 .setMIFlag(MachineInstr::FrameSetup);
3351 }
3352 }
3353
3354 if (X86FI->getRestoreBasePointer()) {
3355 unsigned Opc = STI.is64Bit() ? X86::PUSH64r : X86::PUSH32r;
3356 Register BaseReg = this->TRI->getBaseRegister();
3357 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: Opc))
3358 .addReg(RegNo: BaseReg, Flags: getKillRegState(B: true))
3359 .setMIFlag(MachineInstr::FrameSetup);
3360 }
3361
3362 // Make XMM regs spilled. X86 does not have ability of push/pop XMM.
3363 // It can be done by spilling XMMs to stack frame.
3364 for (const CalleeSavedInfo &I : llvm::reverse(C&: CSI)) {
3365 MCRegister Reg = I.getReg();
3366 if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg))
3367 continue;
3368
3369 // Add the callee-saved register as live-in. It's killed at the spill.
3370 MBB.addLiveIn(PhysReg: Reg);
3371 const TargetRegisterClass *RC = getCalleeSavedSpillRC(Reg, STI, TRI: *TRI);
3372
3373 TII.storeRegToStackSlot(MBB, MI, SrcReg: Reg, isKill: true, FrameIndex: I.getFrameIdx(), RC, VReg: Register(),
3374 Flags: MachineInstr::FrameSetup);
3375 }
3376
3377 return true;
3378}
3379
3380void X86FrameLowering::emitCatchRetReturnValue(MachineBasicBlock &MBB,
3381 MachineBasicBlock::iterator MBBI,
3382 MachineInstr *CatchRet) const {
3383 // SEH shouldn't use catchret.
3384 assert(!isAsynchronousEHPersonality(classifyEHPersonality(
3385 MBB.getParent()->getFunction().getPersonalityFn())) &&
3386 "SEH should not use CATCHRET");
3387 const DebugLoc &DL = CatchRet->getDebugLoc();
3388 MachineBasicBlock *CatchRetTarget = CatchRet->getOperand(i: 0).getMBB();
3389
3390 // Fill EAX/RAX with the address of the target block.
3391 if (STI.is64Bit()) {
3392 // LEA64r CatchRetTarget(%rip), %rax
3393 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::LEA64r), DestReg: X86::RAX)
3394 .addReg(RegNo: X86::RIP)
3395 .addImm(Val: 0)
3396 .addReg(RegNo: 0)
3397 .addMBB(MBB: CatchRetTarget)
3398 .addReg(RegNo: 0);
3399 } else {
3400 // MOV32ri $CatchRetTarget, %eax
3401 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV32ri), DestReg: X86::EAX)
3402 .addMBB(MBB: CatchRetTarget);
3403 }
3404
3405 // Record that we've taken the address of CatchRetTarget and no longer just
3406 // reference it in a terminator.
3407 CatchRetTarget->setMachineBlockAddressTaken();
3408}
3409
3410bool X86FrameLowering::restoreCalleeSavedRegisters(
3411 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
3412 MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
3413 if (CSI.empty())
3414 return false;
3415
3416 if (MI != MBB.end() && isFuncletReturnInstr(MI&: *MI) && STI.isOSWindows()) {
3417 // Don't restore CSRs in 32-bit EH funclets. Matches
3418 // spillCalleeSavedRegisters.
3419 if (STI.is32Bit())
3420 return true;
3421 // Don't restore CSRs before an SEH catchret. SEH except blocks do not form
3422 // funclets. emitEpilogue transforms these to normal jumps.
3423 if (MI->getOpcode() == X86::CATCHRET) {
3424 const Function &F = MBB.getParent()->getFunction();
3425 bool IsSEH = isAsynchronousEHPersonality(
3426 Pers: classifyEHPersonality(Pers: F.getPersonalityFn()));
3427 if (IsSEH)
3428 return true;
3429 }
3430 }
3431
3432 DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
3433 MachineFunction &MF = *MBB.getParent();
3434 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
3435
3436 bool NeedsWin64CFI = MF.needsWinCFI();
3437 bool IsWin64UnwindV3 = NeedsWin64CFI && requireWinX64UnwindV3(MF);
3438
3439 // Reload XMMs from stack frame.
3440 for (const CalleeSavedInfo &I : CSI) {
3441 MCRegister Reg = I.getReg();
3442 if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg))
3443 continue;
3444
3445 const TargetRegisterClass *RC = getCalleeSavedSpillRC(Reg, STI, TRI: *TRI);
3446 TII.loadRegFromStackSlot(MBB, MI, DestReg: Reg, FrameIndex: I.getFrameIdx(), RC, VReg: Register(), SubReg: 0,
3447 Flags: MachineInstr::FrameDestroy);
3448 }
3449
3450 // Clear the stack slot for spill base pointer register.
3451 if (X86FI->getRestoreBasePointer()) {
3452 if (IsWin64UnwindV3)
3453 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_PushReg))
3454 .addImm(Val: this->TRI->getBaseRegister())
3455 .setMIFlag(MachineInstr::FrameDestroy);
3456 unsigned Opc = STI.is64Bit() ? X86::POP64r : X86::POP32r;
3457 Register BaseReg = this->TRI->getBaseRegister();
3458 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: BaseReg)
3459 .setMIFlag(MachineInstr::FrameDestroy);
3460 }
3461
3462 // POP GPRs.
3463 for (auto I = CSI.begin(), E = CSI.end(); I != E; ++I) {
3464 MCRegister Reg = I->getReg();
3465 if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg))
3466 continue;
3467
3468 if (X86FI->isCandidateForPush2Pop2(Reg)) {
3469 MCRegister Reg2 = (++I)->getReg();
3470 if (IsWin64UnwindV3) {
3471 // Name the pair in PUSH2 order: the epilog op must describe the PUSH2
3472 // that this POP2 undoes.
3473 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_Push2Regs))
3474 .addImm(Val: Reg2)
3475 .addImm(Val: Reg)
3476 .setMIFlag(MachineInstr::FrameDestroy);
3477 }
3478 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: getPOP2Opcode(ST: STI)), DestReg: Reg)
3479 .addReg(RegNo: Reg2, Flags: RegState::Define)
3480 .setMIFlag(MachineInstr::FrameDestroy);
3481 } else {
3482 if (IsWin64UnwindV3)
3483 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: X86::SEH_PushReg))
3484 .addImm(Val: Reg)
3485 .setMIFlag(MachineInstr::FrameDestroy);
3486 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: getPOPOpcode(ST: STI)), DestReg: Reg)
3487 .setMIFlag(MachineInstr::FrameDestroy);
3488 }
3489 }
3490
3491 return true;
3492}
3493
3494void X86FrameLowering::determineCalleeSaves(MachineFunction &MF,
3495 BitVector &SavedRegs,
3496 RegScavenger *RS) const {
3497 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
3498
3499 // Spill the BasePtr if it's used.
3500 if (TRI->hasBasePointer(MF)) {
3501 Register BasePtr = TRI->getBaseRegister();
3502 if (STI.isTarget64BitILP32())
3503 BasePtr = getX86SubSuperRegister(Reg: BasePtr, Size: 64);
3504 SavedRegs.set(BasePtr);
3505 }
3506 if (STI.hasUserReservedRegisters()) {
3507 for (int Reg = SavedRegs.find_first(); Reg != -1;
3508 Reg = SavedRegs.find_next(Prev: Reg)) {
3509 if (STI.isRegisterReservedByUser(i: Reg)) {
3510 SavedRegs.reset(Idx: Reg);
3511 }
3512 }
3513 }
3514}
3515
3516static bool HasNestArgument(const MachineFunction *MF) {
3517 const Function &F = MF->getFunction();
3518 for (Function::const_arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E;
3519 I++) {
3520 if (I->hasNestAttr() && !I->use_empty())
3521 return true;
3522 }
3523 return false;
3524}
3525
3526/// GetScratchRegister - Get a temp register for performing work in the
3527/// segmented stack and the Erlang/HiPE stack prologue. Depending on platform
3528/// and the properties of the function either one or two registers will be
3529/// needed. Set primary to true for the first register, false for the second.
3530static unsigned GetScratchRegister(bool Is64Bit, bool IsLP64,
3531 const MachineFunction &MF, bool Primary) {
3532 CallingConv::ID CallingConvention = MF.getFunction().getCallingConv();
3533
3534 // Erlang stuff.
3535 if (CallingConvention == CallingConv::HiPE) {
3536 if (Is64Bit)
3537 return Primary ? X86::R14 : X86::R13;
3538 else
3539 return Primary ? X86::EBX : X86::EDI;
3540 }
3541
3542 if (Is64Bit) {
3543 if (IsLP64)
3544 return Primary ? X86::R11 : X86::R12;
3545 else
3546 return Primary ? X86::R11D : X86::R12D;
3547 }
3548
3549 bool IsNested = HasNestArgument(MF: &MF);
3550
3551 if (CallingConvention == CallingConv::X86_FastCall ||
3552 CallingConvention == CallingConv::Fast ||
3553 CallingConvention == CallingConv::Tail) {
3554 if (IsNested)
3555 report_fatal_error(reason: "Segmented stacks does not support fastcall with "
3556 "nested function.");
3557 return Primary ? X86::EAX : X86::ECX;
3558 }
3559 if (IsNested)
3560 return Primary ? X86::EDX : X86::EAX;
3561 return Primary ? X86::ECX : X86::EAX;
3562}
3563
3564// The stack limit in the TCB is set to this many bytes above the actual stack
3565// limit.
3566static const uint64_t kSplitStackAvailable = 256;
3567
3568void X86FrameLowering::adjustForSegmentedStacks(
3569 MachineFunction &MF, MachineBasicBlock &PrologueMBB) const {
3570 MachineFrameInfo &MFI = MF.getFrameInfo();
3571 uint64_t StackSize;
3572 unsigned TlsReg, TlsOffset;
3573 DebugLoc DL;
3574
3575 // To support shrink-wrapping we would need to insert the new blocks
3576 // at the right place and update the branches to PrologueMBB.
3577 assert(&(*MF.begin()) == &PrologueMBB && "Shrink-wrapping not supported yet");
3578
3579 unsigned ScratchReg = GetScratchRegister(Is64Bit, IsLP64, MF, Primary: true);
3580 assert(!MF.getRegInfo().isLiveIn(ScratchReg) &&
3581 "Scratch register is live-in");
3582
3583 if (MF.getFunction().isVarArg())
3584 report_fatal_error(reason: "Segmented stacks do not support vararg functions.");
3585 if (!STI.isTargetLinux() && !STI.isTargetDarwin() && !STI.isTargetWin32() &&
3586 !STI.isTargetWin64() && !STI.isTargetFreeBSD() &&
3587 !STI.isTargetDragonFly())
3588 report_fatal_error(reason: "Segmented stacks not supported on this platform.");
3589
3590 // Eventually StackSize will be calculated by a link-time pass; which will
3591 // also decide whether checking code needs to be injected into this particular
3592 // prologue.
3593 StackSize = MFI.getStackSize();
3594
3595 if (!MFI.needsSplitStackProlog())
3596 return;
3597
3598 MachineBasicBlock *allocMBB = MF.CreateMachineBasicBlock();
3599 MachineBasicBlock *checkMBB = MF.CreateMachineBasicBlock();
3600 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
3601 bool IsNested = false;
3602
3603 // We need to know if the function has a nest argument only in 64 bit mode.
3604 if (Is64Bit)
3605 IsNested = HasNestArgument(MF: &MF);
3606
3607 // The MOV R10, RAX needs to be in a different block, since the RET we emit in
3608 // allocMBB needs to be last (terminating) instruction.
3609
3610 for (const auto &LI : PrologueMBB.liveins()) {
3611 allocMBB->addLiveIn(RegMaskPair: LI);
3612 checkMBB->addLiveIn(RegMaskPair: LI);
3613 }
3614
3615 if (IsNested)
3616 allocMBB->addLiveIn(PhysReg: IsLP64 ? X86::R10 : X86::R10D);
3617
3618 MF.push_front(MBB: allocMBB);
3619 MF.push_front(MBB: checkMBB);
3620
3621 // When the frame size is less than 256 we just compare the stack
3622 // boundary directly to the value of the stack pointer, per gcc.
3623 bool CompareStackPointer = StackSize < kSplitStackAvailable;
3624
3625 // Read the limit off the current stacklet off the stack_guard location.
3626 if (Is64Bit) {
3627 if (STI.isTargetLinux()) {
3628 TlsReg = X86::FS;
3629 TlsOffset = IsLP64 ? 0x70 : 0x40;
3630 } else if (STI.isTargetDarwin()) {
3631 TlsReg = X86::GS;
3632 TlsOffset = 0x60 + 90 * 8; // See pthread_machdep.h. Steal TLS slot 90.
3633 } else if (STI.isTargetWin64()) {
3634 TlsReg = X86::GS;
3635 TlsOffset = 0x28; // pvArbitrary, reserved for application use
3636 } else if (STI.isTargetFreeBSD()) {
3637 TlsReg = X86::FS;
3638 TlsOffset = 0x18;
3639 } else if (STI.isTargetDragonFly()) {
3640 TlsReg = X86::FS;
3641 TlsOffset = 0x20; // use tls_tcb.tcb_segstack
3642 } else {
3643 report_fatal_error(reason: "Segmented stacks not supported on this platform.");
3644 }
3645
3646 if (CompareStackPointer)
3647 ScratchReg = IsLP64 ? X86::RSP : X86::ESP;
3648 else
3649 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: IsLP64 ? X86::LEA64r : X86::LEA64_32r),
3650 DestReg: ScratchReg)
3651 .addReg(RegNo: X86::RSP)
3652 .addImm(Val: 1)
3653 .addReg(RegNo: 0)
3654 .addImm(Val: -StackSize)
3655 .addReg(RegNo: 0);
3656
3657 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: IsLP64 ? X86::CMP64rm : X86::CMP32rm))
3658 .addReg(RegNo: ScratchReg)
3659 .addReg(RegNo: 0)
3660 .addImm(Val: 1)
3661 .addReg(RegNo: 0)
3662 .addImm(Val: TlsOffset)
3663 .addReg(RegNo: TlsReg);
3664 } else {
3665 if (STI.isTargetLinux()) {
3666 TlsReg = X86::GS;
3667 TlsOffset = 0x30;
3668 } else if (STI.isTargetDarwin()) {
3669 TlsReg = X86::GS;
3670 TlsOffset = 0x48 + 90 * 4;
3671 } else if (STI.isTargetWin32()) {
3672 TlsReg = X86::FS;
3673 TlsOffset = 0x14; // pvArbitrary, reserved for application use
3674 } else if (STI.isTargetDragonFly()) {
3675 TlsReg = X86::FS;
3676 TlsOffset = 0x10; // use tls_tcb.tcb_segstack
3677 } else if (STI.isTargetFreeBSD()) {
3678 report_fatal_error(reason: "Segmented stacks not supported on FreeBSD i386.");
3679 } else {
3680 report_fatal_error(reason: "Segmented stacks not supported on this platform.");
3681 }
3682
3683 if (CompareStackPointer)
3684 ScratchReg = X86::ESP;
3685 else
3686 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: X86::LEA32r), DestReg: ScratchReg)
3687 .addReg(RegNo: X86::ESP)
3688 .addImm(Val: 1)
3689 .addReg(RegNo: 0)
3690 .addImm(Val: -StackSize)
3691 .addReg(RegNo: 0);
3692
3693 if (STI.isTargetLinux() || STI.isTargetWin32() || STI.isTargetWin64() ||
3694 STI.isTargetDragonFly()) {
3695 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: X86::CMP32rm))
3696 .addReg(RegNo: ScratchReg)
3697 .addReg(RegNo: 0)
3698 .addImm(Val: 0)
3699 .addReg(RegNo: 0)
3700 .addImm(Val: TlsOffset)
3701 .addReg(RegNo: TlsReg);
3702 } else if (STI.isTargetDarwin()) {
3703
3704 // TlsOffset doesn't fit into a mod r/m byte so we need an extra register.
3705 unsigned ScratchReg2;
3706 bool SaveScratch2;
3707 if (CompareStackPointer) {
3708 // The primary scratch register is available for holding the TLS offset.
3709 ScratchReg2 = GetScratchRegister(Is64Bit, IsLP64, MF, Primary: true);
3710 SaveScratch2 = false;
3711 } else {
3712 // Need to use a second register to hold the TLS offset
3713 ScratchReg2 = GetScratchRegister(Is64Bit, IsLP64, MF, Primary: false);
3714
3715 // Unfortunately, with fastcc the second scratch register may hold an
3716 // argument.
3717 SaveScratch2 = MF.getRegInfo().isLiveIn(Reg: ScratchReg2);
3718 }
3719
3720 // If Scratch2 is live-in then it needs to be saved.
3721 assert((!MF.getRegInfo().isLiveIn(ScratchReg2) || SaveScratch2) &&
3722 "Scratch register is live-in and not saved");
3723
3724 if (SaveScratch2)
3725 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH32r))
3726 .addReg(RegNo: ScratchReg2, Flags: RegState::Kill);
3727
3728 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: X86::MOV32ri), DestReg: ScratchReg2)
3729 .addImm(Val: TlsOffset);
3730 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: X86::CMP32rm))
3731 .addReg(RegNo: ScratchReg)
3732 .addReg(RegNo: ScratchReg2)
3733 .addImm(Val: 1)
3734 .addReg(RegNo: 0)
3735 .addImm(Val: 0)
3736 .addReg(RegNo: TlsReg);
3737
3738 if (SaveScratch2)
3739 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: X86::POP32r), DestReg: ScratchReg2);
3740 }
3741 }
3742
3743 // This jump is taken if SP >= (Stacklet Limit + Stack Space required).
3744 // It jumps to normal execution of the function body.
3745 BuildMI(BB: checkMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
3746 .addMBB(MBB: &PrologueMBB)
3747 .addImm(Val: X86::COND_A);
3748
3749 // On 32 bit we first push the arguments size and then the frame size. On 64
3750 // bit, we pass the stack frame size in r10 and the argument size in r11.
3751 if (Is64Bit) {
3752 // Functions with nested arguments use R10, so it needs to be saved across
3753 // the call to _morestack
3754
3755 const unsigned RegAX = IsLP64 ? X86::RAX : X86::EAX;
3756 const unsigned Reg10 = IsLP64 ? X86::R10 : X86::R10D;
3757 const unsigned Reg11 = IsLP64 ? X86::R11 : X86::R11D;
3758 const unsigned MOVrr = IsLP64 ? X86::MOV64rr : X86::MOV32rr;
3759
3760 if (IsNested)
3761 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: MOVrr), DestReg: RegAX).addReg(RegNo: Reg10);
3762
3763 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: X86::getMOVriOpcode(Use64BitReg: IsLP64, Imm: StackSize)),
3764 DestReg: Reg10)
3765 .addImm(Val: StackSize);
3766 BuildMI(BB: allocMBB, MIMD: DL,
3767 MCID: TII.get(Opcode: X86::getMOVriOpcode(Use64BitReg: IsLP64, Imm: X86FI->getArgumentStackSize())),
3768 DestReg: Reg11)
3769 .addImm(Val: X86FI->getArgumentStackSize());
3770 } else {
3771 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH32i))
3772 .addImm(Val: X86FI->getArgumentStackSize());
3773 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH32i)).addImm(Val: StackSize);
3774 }
3775
3776 // __morestack is in libgcc
3777 if (Is64Bit && MF.getTarget().getCodeModel() == CodeModel::Large) {
3778 // Under the large code model, we cannot assume that __morestack lives
3779 // within 2^31 bytes of the call site, so we cannot use pc-relative
3780 // addressing. We cannot perform the call via a temporary register,
3781 // as the rax register may be used to store the static chain, and all
3782 // other suitable registers may be either callee-save or used for
3783 // parameter passing. We cannot use the stack at this point either
3784 // because __morestack manipulates the stack directly.
3785 //
3786 // To avoid these issues, perform an indirect call via a read-only memory
3787 // location containing the address.
3788 //
3789 // This solution is not perfect, as it assumes that the .rodata section
3790 // is laid out within 2^31 bytes of each function body, but this seems
3791 // to be sufficient for JIT.
3792 // FIXME: Add retpoline support and remove the error here..
3793 if (STI.useIndirectThunkCalls())
3794 report_fatal_error(reason: "Emitting morestack calls on 64-bit with the large "
3795 "code model and thunks not yet implemented.");
3796 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: X86::CALL64m))
3797 .addReg(RegNo: X86::RIP)
3798 .addImm(Val: 0)
3799 .addReg(RegNo: 0)
3800 .addExternalSymbol(FnName: "__morestack_addr")
3801 .addReg(RegNo: 0);
3802 } else {
3803 if (Is64Bit)
3804 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: X86::CALL64pcrel32))
3805 .addExternalSymbol(FnName: "__morestack");
3806 else
3807 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: X86::CALLpcrel32))
3808 .addExternalSymbol(FnName: "__morestack");
3809 }
3810
3811 if (IsNested)
3812 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: X86::MORESTACK_RET_RESTORE_R10));
3813 else
3814 BuildMI(BB: allocMBB, MIMD: DL, MCID: TII.get(Opcode: X86::MORESTACK_RET));
3815
3816 allocMBB->addSuccessor(Succ: &PrologueMBB);
3817
3818 checkMBB->addSuccessor(Succ: allocMBB, Prob: BranchProbability::getZero());
3819 checkMBB->addSuccessor(Succ: &PrologueMBB, Prob: BranchProbability::getOne());
3820
3821#ifdef EXPENSIVE_CHECKS
3822 MF.verify();
3823#endif
3824}
3825
3826/// Lookup an ERTS parameter in the !hipe.literals named metadata node.
3827/// HiPE provides Erlang Runtime System-internal parameters, such as PCB offsets
3828/// to fields it needs, through a named metadata node "hipe.literals" containing
3829/// name-value pairs.
3830static unsigned getHiPELiteral(NamedMDNode *HiPELiteralsMD,
3831 const StringRef LiteralName) {
3832 for (int i = 0, e = HiPELiteralsMD->getNumOperands(); i != e; ++i) {
3833 MDNode *Node = HiPELiteralsMD->getOperand(i);
3834 if (Node->getNumOperands() != 2)
3835 continue;
3836 MDString *NodeName = dyn_cast<MDString>(Val: Node->getOperand(I: 0));
3837 ValueAsMetadata *NodeVal = dyn_cast<ValueAsMetadata>(Val: Node->getOperand(I: 1));
3838 if (!NodeName || !NodeVal)
3839 continue;
3840 ConstantInt *ValConst = dyn_cast_or_null<ConstantInt>(Val: NodeVal->getValue());
3841 if (ValConst && NodeName->getString() == LiteralName) {
3842 return ValConst->getZExtValue();
3843 }
3844 }
3845
3846 report_fatal_error(reason: "HiPE literal " + LiteralName +
3847 " required but not provided");
3848}
3849
3850// Return true if there are no non-ehpad successors to MBB and there are no
3851// non-meta instructions between MBBI and MBB.end().
3852static bool blockEndIsUnreachable(const MachineBasicBlock &MBB,
3853 MachineBasicBlock::const_iterator MBBI) {
3854 return llvm::all_of(
3855 Range: MBB.successors(),
3856 P: [](const MachineBasicBlock *Succ) { return Succ->isEHPad(); }) &&
3857 std::all_of(first: MBBI, last: MBB.end(), pred: [](const MachineInstr &MI) {
3858 return MI.isMetaInstruction();
3859 });
3860}
3861
3862/// Erlang programs may need a special prologue to handle the stack size they
3863/// might need at runtime. That is because Erlang/OTP does not implement a C
3864/// stack but uses a custom implementation of hybrid stack/heap architecture.
3865/// (for more information see Eric Stenman's Ph.D. thesis:
3866/// http://publications.uu.se/uu/fulltext/nbn_se_uu_diva-2688.pdf)
3867///
3868/// CheckStack:
3869/// temp0 = sp - MaxStack
3870/// if( temp0 < SP_LIMIT(P) ) goto IncStack else goto OldStart
3871/// OldStart:
3872/// ...
3873/// IncStack:
3874/// call inc_stack # doubles the stack space
3875/// temp0 = sp - MaxStack
3876/// if( temp0 < SP_LIMIT(P) ) goto IncStack else goto OldStart
3877void X86FrameLowering::adjustForHiPEPrologue(
3878 MachineFunction &MF, MachineBasicBlock &PrologueMBB) const {
3879 MachineFrameInfo &MFI = MF.getFrameInfo();
3880 DebugLoc DL;
3881
3882 // To support shrink-wrapping we would need to insert the new blocks
3883 // at the right place and update the branches to PrologueMBB.
3884 assert(&(*MF.begin()) == &PrologueMBB && "Shrink-wrapping not supported yet");
3885
3886 // HiPE-specific values
3887 NamedMDNode *HiPELiteralsMD =
3888 MF.getFunction().getParent()->getNamedMetadata(Name: "hipe.literals");
3889 if (!HiPELiteralsMD)
3890 report_fatal_error(
3891 reason: "Can't generate HiPE prologue without runtime parameters");
3892 const unsigned HipeLeafWords = getHiPELiteral(
3893 HiPELiteralsMD, LiteralName: Is64Bit ? "AMD64_LEAF_WORDS" : "X86_LEAF_WORDS");
3894 const unsigned CCRegisteredArgs = Is64Bit ? 6 : 5;
3895 const unsigned Guaranteed = HipeLeafWords * SlotSize;
3896 unsigned CallerStkArity = MF.getFunction().arg_size() > CCRegisteredArgs
3897 ? MF.getFunction().arg_size() - CCRegisteredArgs
3898 : 0;
3899 unsigned MaxStack = MFI.getStackSize() + CallerStkArity * SlotSize + SlotSize;
3900
3901 assert(STI.isTargetLinux() &&
3902 "HiPE prologue is only supported on Linux operating systems.");
3903
3904 // Compute the largest caller's frame that is needed to fit the callees'
3905 // frames. This 'MaxStack' is computed from:
3906 //
3907 // a) the fixed frame size, which is the space needed for all spilled temps,
3908 // b) outgoing on-stack parameter areas, and
3909 // c) the minimum stack space this function needs to make available for the
3910 // functions it calls (a tunable ABI property).
3911 if (MFI.hasCalls()) {
3912 unsigned MoreStackForCalls = 0;
3913
3914 for (auto &MBB : MF) {
3915 for (auto &MI : MBB) {
3916 if (!MI.isCall())
3917 continue;
3918
3919 // Get callee operand.
3920 const MachineOperand &MO = MI.getOperand(i: 0);
3921
3922 // Only take account of global function calls (no closures etc.).
3923 if (!MO.isGlobal())
3924 continue;
3925
3926 const Function *F = dyn_cast<Function>(Val: MO.getGlobal());
3927 if (!F)
3928 continue;
3929
3930 // Do not update 'MaxStack' for primitive and built-in functions
3931 // (encoded with names either starting with "erlang."/"bif_" or not
3932 // having a ".", such as a simple <Module>.<Function>.<Arity>, or an
3933 // "_", such as the BIF "suspend_0") as they are executed on another
3934 // stack.
3935 if (F->getName().contains(Other: "erlang.") || F->getName().contains(Other: "bif_") ||
3936 F->getName().find_first_of(Chars: "._") == StringRef::npos)
3937 continue;
3938
3939 unsigned CalleeStkArity = F->arg_size() > CCRegisteredArgs
3940 ? F->arg_size() - CCRegisteredArgs
3941 : 0;
3942 if (HipeLeafWords - 1 > CalleeStkArity)
3943 MoreStackForCalls =
3944 std::max(a: MoreStackForCalls,
3945 b: (HipeLeafWords - 1 - CalleeStkArity) * SlotSize);
3946 }
3947 }
3948 MaxStack += MoreStackForCalls;
3949 }
3950
3951 // If the stack frame needed is larger than the guaranteed then runtime checks
3952 // and calls to "inc_stack_0" BIF should be inserted in the assembly prologue.
3953 if (MaxStack > Guaranteed) {
3954 MachineBasicBlock *stackCheckMBB = MF.CreateMachineBasicBlock();
3955 MachineBasicBlock *incStackMBB = MF.CreateMachineBasicBlock();
3956
3957 for (const auto &LI : PrologueMBB.liveins()) {
3958 stackCheckMBB->addLiveIn(RegMaskPair: LI);
3959 incStackMBB->addLiveIn(RegMaskPair: LI);
3960 }
3961
3962 MF.push_front(MBB: incStackMBB);
3963 MF.push_front(MBB: stackCheckMBB);
3964
3965 unsigned ScratchReg, SPReg, PReg, SPLimitOffset;
3966 unsigned LEAop, CMPop, CALLop;
3967 SPLimitOffset = getHiPELiteral(HiPELiteralsMD, LiteralName: "P_NSP_LIMIT");
3968 if (Is64Bit) {
3969 SPReg = X86::RSP;
3970 PReg = X86::RBP;
3971 LEAop = X86::LEA64r;
3972 CMPop = X86::CMP64rm;
3973 CALLop = X86::CALL64pcrel32;
3974 } else {
3975 SPReg = X86::ESP;
3976 PReg = X86::EBP;
3977 LEAop = X86::LEA32r;
3978 CMPop = X86::CMP32rm;
3979 CALLop = X86::CALLpcrel32;
3980 }
3981
3982 ScratchReg = GetScratchRegister(Is64Bit, IsLP64, MF, Primary: true);
3983 assert(!MF.getRegInfo().isLiveIn(ScratchReg) &&
3984 "HiPE prologue scratch register is live-in");
3985
3986 // Create new MBB for StackCheck:
3987 addRegOffset(MIB: BuildMI(BB: stackCheckMBB, MIMD: DL, MCID: TII.get(Opcode: LEAop), DestReg: ScratchReg), Reg: SPReg,
3988 Offset: -MaxStack);
3989 // SPLimitOffset is in a fixed heap location (pointed by BP).
3990 addRegOffset(MIB: BuildMI(BB: stackCheckMBB, MIMD: DL, MCID: TII.get(Opcode: CMPop)).addReg(RegNo: ScratchReg),
3991 Reg: PReg, Offset: SPLimitOffset);
3992 BuildMI(BB: stackCheckMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
3993 .addMBB(MBB: &PrologueMBB)
3994 .addImm(Val: X86::COND_AE);
3995
3996 // Create new MBB for IncStack:
3997 BuildMI(BB: incStackMBB, MIMD: DL, MCID: TII.get(Opcode: CALLop)).addExternalSymbol(FnName: "inc_stack_0");
3998 addRegOffset(MIB: BuildMI(BB: incStackMBB, MIMD: DL, MCID: TII.get(Opcode: LEAop), DestReg: ScratchReg), Reg: SPReg,
3999 Offset: -MaxStack);
4000 addRegOffset(MIB: BuildMI(BB: incStackMBB, MIMD: DL, MCID: TII.get(Opcode: CMPop)).addReg(RegNo: ScratchReg),
4001 Reg: PReg, Offset: SPLimitOffset);
4002 BuildMI(BB: incStackMBB, MIMD: DL, MCID: TII.get(Opcode: X86::JCC_1))
4003 .addMBB(MBB: incStackMBB)
4004 .addImm(Val: X86::COND_LE);
4005
4006 stackCheckMBB->addSuccessor(Succ: &PrologueMBB, Prob: {99, 100});
4007 stackCheckMBB->addSuccessor(Succ: incStackMBB, Prob: {1, 100});
4008 incStackMBB->addSuccessor(Succ: &PrologueMBB, Prob: {99, 100});
4009 incStackMBB->addSuccessor(Succ: incStackMBB, Prob: {1, 100});
4010 }
4011#ifdef EXPENSIVE_CHECKS
4012 MF.verify();
4013#endif
4014}
4015
4016bool X86FrameLowering::adjustStackWithPops(MachineBasicBlock &MBB,
4017 MachineBasicBlock::iterator MBBI,
4018 const DebugLoc &DL,
4019 int Offset) const {
4020 if (Offset <= 0)
4021 return false;
4022
4023 if (Offset % SlotSize)
4024 return false;
4025
4026 int NumPops = Offset / SlotSize;
4027 // This is only worth it if we have at most 2 pops.
4028 if (NumPops != 1 && NumPops != 2)
4029 return false;
4030
4031 // Handle only the trivial case where the adjustment follows a call, ignoring
4032 // debug instructions. This is the most common one, anyway.
4033 if (MBBI == MBB.begin())
4034 return false;
4035 MachineBasicBlock::iterator Prev = prev_nodbg(It: MBBI, Begin: MBB.begin());
4036 if (!Prev->isCall() || !Prev->getOperand(i: 1).isRegMask())
4037 return false;
4038
4039 unsigned Regs[2];
4040 unsigned FoundRegs = 0;
4041
4042 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4043 const MachineOperand &RegMask = Prev->getOperand(i: 1);
4044
4045 auto &RegClass =
4046 Is64Bit ? X86::GR64_NOREX_NOSPRegClass : X86::GR32_NOREX_NOSPRegClass;
4047 // Try to find up to NumPops free registers.
4048 for (auto Candidate : RegClass) {
4049 // Poor man's liveness:
4050 // Since we're immediately after a call, any register that is clobbered
4051 // by the call and not defined by it can be considered dead.
4052 if (!RegMask.clobbersPhysReg(PhysReg: Candidate))
4053 continue;
4054
4055 // Don't clobber reserved registers
4056 if (MRI.isReserved(PhysReg: Candidate))
4057 continue;
4058
4059 bool IsDef = false;
4060 for (const MachineOperand &MO : Prev->implicit_operands()) {
4061 if (MO.isReg() && MO.isDef() &&
4062 TRI->isSuperOrSubRegisterEq(RegA: MO.getReg(), RegB: Candidate)) {
4063 IsDef = true;
4064 break;
4065 }
4066 }
4067
4068 if (IsDef)
4069 continue;
4070
4071 Regs[FoundRegs++] = Candidate;
4072 if (FoundRegs == (unsigned)NumPops)
4073 break;
4074 }
4075
4076 if (FoundRegs == 0)
4077 return false;
4078
4079 // If we found only one free register, but need two, reuse the same one twice.
4080 while (FoundRegs < (unsigned)NumPops)
4081 Regs[FoundRegs++] = Regs[0];
4082
4083 for (int i = 0; i < NumPops; ++i)
4084 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: STI.is64Bit() ? X86::POP64r : X86::POP32r),
4085 DestReg: Regs[i]);
4086
4087 return true;
4088}
4089
4090MachineBasicBlock::iterator X86FrameLowering::eliminateCallFramePseudoInstr(
4091 MachineFunction &MF, MachineBasicBlock &MBB,
4092 MachineBasicBlock::iterator I) const {
4093 bool reserveCallFrame = hasReservedCallFrame(MF);
4094 unsigned Opcode = I->getOpcode();
4095 bool isDestroy = Opcode == TII.getCallFrameDestroyOpcode();
4096 DebugLoc DL = I->getDebugLoc(); // copy DebugLoc as I will be erased.
4097 uint64_t Amount = TII.getFrameSize(I: *I);
4098 uint64_t InternalAmt = (isDestroy || Amount) ? TII.getFrameAdjustment(I: *I) : 0;
4099 I = MBB.erase(I);
4100 auto InsertPos = skipDebugInstructionsForward(It: I, End: MBB.end());
4101
4102 // Try to avoid emitting dead SP adjustments if the block end is unreachable,
4103 // typically because the function is marked noreturn (abort, throw,
4104 // assert_fail, etc).
4105 if (isDestroy && blockEndIsUnreachable(MBB, MBBI: I))
4106 return I;
4107
4108 if (!reserveCallFrame) {
4109 // If the stack pointer can be changed after prologue, turn the
4110 // adjcallstackup instruction into a 'sub ESP, <amt>' and the
4111 // adjcallstackdown instruction into 'add ESP, <amt>'
4112
4113 // We need to keep the stack aligned properly. To do this, we round the
4114 // amount of space needed for the outgoing arguments up to the next
4115 // alignment boundary.
4116 Amount = alignTo(Size: Amount, A: getStackAlign());
4117
4118 const Function &F = MF.getFunction();
4119 bool WindowsCFI = MF.usesWindowsUnwindInfo();
4120 bool DwarfCFI = !WindowsCFI && MF.needsFrameMoves();
4121
4122 // If we have any exception handlers in this function, and we adjust
4123 // the SP before calls, we may need to indicate this to the unwinder
4124 // using GNU_ARGS_SIZE. Note that this may be necessary even when
4125 // Amount == 0, because the preceding function may have set a non-0
4126 // GNU_ARGS_SIZE.
4127 // TODO: We don't need to reset this between subsequent functions,
4128 // if it didn't change.
4129 bool HasDwarfEHHandlers = !WindowsCFI && !MF.getLandingPads().empty();
4130
4131 if (HasDwarfEHHandlers && !isDestroy &&
4132 MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences())
4133 BuildCFI(MBB, MBBI: InsertPos, DL,
4134 CFIInst: MCCFIInstruction::createGnuArgsSize(L: nullptr, Size: Amount));
4135
4136 if (Amount == 0)
4137 return I;
4138
4139 // Factor out the amount that gets handled inside the sequence
4140 // (Pushes of argument for frame setup, callee pops for frame destroy)
4141 Amount -= InternalAmt;
4142
4143 // TODO: This is needed only if we require precise CFA.
4144 // If this is a callee-pop calling convention, emit a CFA adjust for
4145 // the amount the callee popped.
4146 if (isDestroy && InternalAmt && DwarfCFI && !hasFP(MF))
4147 BuildCFI(MBB, MBBI: InsertPos, DL,
4148 CFIInst: MCCFIInstruction::createAdjustCfaOffset(L: nullptr, Adjustment: -InternalAmt));
4149
4150 // Add Amount to SP to destroy a frame, or subtract to setup.
4151 int64_t StackAdjustment = isDestroy ? Amount : -Amount;
4152 int64_t CfaAdjustment = StackAdjustment;
4153
4154 if (StackAdjustment) {
4155 // Merge with any previous or following adjustment instruction. Note: the
4156 // instructions merged with here do not have CFI, so their stack
4157 // adjustments do not feed into CfaAdjustment
4158
4159 auto CalcCfaAdjust = [&CfaAdjustment](MachineBasicBlock::iterator PI,
4160 int64_t Offset) {
4161 CfaAdjustment += Offset;
4162 };
4163 auto CalcNewOffset = [&StackAdjustment](int64_t Offset) {
4164 return StackAdjustment + Offset;
4165 };
4166 StackAdjustment =
4167 mergeSPUpdates(MBB, MBBI&: InsertPos, FoundStackAdjust: CalcCfaAdjust, CalcNewOffset, doMergeWithPrevious: true);
4168 StackAdjustment =
4169 mergeSPUpdates(MBB, MBBI&: InsertPos, FoundStackAdjust: CalcCfaAdjust, CalcNewOffset, doMergeWithPrevious: false);
4170
4171 if (StackAdjustment) {
4172 if (!(F.hasMinSize() &&
4173 adjustStackWithPops(MBB, MBBI: InsertPos, DL, Offset: StackAdjustment)))
4174 BuildStackAdjustment(MBB, MBBI: InsertPos, DL, Offset: StackAdjustment,
4175 /*InEpilogue=*/false);
4176 }
4177 }
4178
4179 if (DwarfCFI && !hasFP(MF) && CfaAdjustment) {
4180 // If we don't have FP, but need to generate unwind information,
4181 // we need to set the correct CFA offset after the stack adjustment.
4182 // How much we adjust the CFA offset depends on whether we're emitting
4183 // CFI only for EH purposes or for debugging. EH only requires the CFA
4184 // offset to be correct at each call site, while for debugging we want
4185 // it to be more precise.
4186
4187 // TODO: When not using precise CFA, we also need to adjust for the
4188 // InternalAmt here.
4189 BuildCFI(
4190 MBB, MBBI: InsertPos, DL,
4191 CFIInst: MCCFIInstruction::createAdjustCfaOffset(L: nullptr, Adjustment: -CfaAdjustment));
4192 }
4193
4194 return I;
4195 }
4196
4197 if (InternalAmt) {
4198 MachineBasicBlock::iterator CI = I;
4199 MachineBasicBlock::iterator B = MBB.begin();
4200 while (CI != B && !std::prev(x: CI)->isCall())
4201 --CI;
4202 BuildStackAdjustment(MBB, MBBI: CI, DL, Offset: -InternalAmt, /*InEpilogue=*/false);
4203 }
4204
4205 return I;
4206}
4207
4208bool X86FrameLowering::canUseAsPrologue(const MachineBasicBlock &MBB) const {
4209 assert(MBB.getParent() && "Block is not attached to a function!");
4210 const MachineFunction &MF = *MBB.getParent();
4211 if (!MBB.isLiveIn(Reg: X86::EFLAGS))
4212 return true;
4213
4214 // If stack probes have to loop inline or call, that will clobber EFLAGS.
4215 // FIXME: we could allow cases that will use emitStackProbeInlineGenericBlock.
4216 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
4217 const X86TargetLowering &TLI = *STI.getTargetLowering();
4218 if (TLI.hasInlineStackProbe(MF) || TLI.hasStackProbeSymbol(MF))
4219 return false;
4220
4221 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
4222 return !TRI->hasStackRealignment(MF) && !X86FI->hasSwiftAsyncContext();
4223}
4224
4225bool X86FrameLowering::canUseAsEpilogue(const MachineBasicBlock &MBB) const {
4226 assert(MBB.getParent() && "Block is not attached to a function!");
4227
4228 // Win64 has strict requirements in terms of epilogue and we are
4229 // not taking a chance at messing with them.
4230 // I.e., unless this block is already an exit block, we can't use
4231 // it as an epilogue.
4232 if (STI.isTargetWin64() && !MBB.succ_empty() && !MBB.isReturnBlock())
4233 return false;
4234
4235 // Swift async context epilogue has a BTR instruction that clobbers parts of
4236 // EFLAGS.
4237 const MachineFunction &MF = *MBB.getParent();
4238 if (MF.getInfo<X86MachineFunctionInfo>()->hasSwiftAsyncContext())
4239 return !flagsNeedToBePreservedBeforeTheTerminators(MBB);
4240
4241 if (canUseLEAForSPInEpilogue(MF: *MBB.getParent()))
4242 return true;
4243
4244 // If we cannot use LEA to adjust SP, we may need to use ADD, which
4245 // clobbers the EFLAGS. Check that we do not need to preserve it,
4246 // otherwise, conservatively assume this is not
4247 // safe to insert the epilogue here.
4248 return !flagsNeedToBePreservedBeforeTheTerminators(MBB);
4249}
4250
4251bool X86FrameLowering::enableShrinkWrapping(const MachineFunction &MF) const {
4252 // If we may need to emit frameless compact unwind information, give
4253 // up as this is currently broken: PR25614.
4254 bool CompactUnwind =
4255 MF.getContext().getObjectFileInfo()->getCompactUnwindSection() != nullptr;
4256 return (MF.getFunction().hasFnAttribute(Kind: Attribute::NoUnwind) || hasFP(MF) ||
4257 !CompactUnwind) &&
4258 // The lowering of segmented stack and HiPE only support entry
4259 // blocks as prologue blocks: PR26107. This limitation may be
4260 // lifted if we fix:
4261 // - adjustForSegmentedStacks
4262 // - adjustForHiPEPrologue
4263 MF.getFunction().getCallingConv() != CallingConv::HiPE &&
4264 !MF.shouldSplitStack();
4265}
4266
4267MachineBasicBlock::iterator X86FrameLowering::restoreWin32EHStackPointers(
4268 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
4269 const DebugLoc &DL, bool RestoreSP) const {
4270 assert(STI.isTargetWindowsMSVC() && "funclets only supported in MSVC env");
4271 assert(STI.isTargetWin32() && "EBP/ESI restoration only required on win32");
4272 assert(STI.is32Bit() && !Uses64BitFramePtr &&
4273 "restoring EBP/ESI on non-32-bit target");
4274
4275 MachineFunction &MF = *MBB.getParent();
4276 Register FramePtr = TRI->getFrameRegister(MF);
4277 Register BasePtr = TRI->getBaseRegister();
4278 WinEHFuncInfo &FuncInfo = *MF.getWinEHFuncInfo();
4279 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
4280 MachineFrameInfo &MFI = MF.getFrameInfo();
4281
4282 // FIXME: Don't set FrameSetup flag in catchret case.
4283
4284 int FI = FuncInfo.EHRegNodeFrameIndex;
4285 int EHRegSize = MFI.getObjectSize(ObjectIdx: FI);
4286
4287 if (RestoreSP) {
4288 // MOV32rm -EHRegSize(%ebp), %esp
4289 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV32rm), DestReg: X86::ESP),
4290 Reg: X86::EBP, Offset: -EHRegSize)
4291 .setMIFlag(MachineInstr::FrameSetup);
4292 }
4293
4294 Register UsedReg;
4295 int EHRegOffset = getFrameIndexReference(MF, FI, FrameReg&: UsedReg).getFixed();
4296 int EndOffset = -EHRegOffset - EHRegSize;
4297 FuncInfo.EHRegNodeEndOffset = EndOffset;
4298
4299 if (UsedReg == FramePtr) {
4300 // ADD $offset, %ebp
4301 unsigned ADDri = getADDriOpcode(IsLP64: false);
4302 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: ADDri), DestReg: FramePtr)
4303 .addReg(RegNo: FramePtr)
4304 .addImm(Val: EndOffset)
4305 .setMIFlag(MachineInstr::FrameSetup)
4306 ->getOperand(i: 3)
4307 .setIsDead();
4308 assert(EndOffset >= 0 &&
4309 "end of registration object above normal EBP position!");
4310 } else if (UsedReg == BasePtr) {
4311 // LEA offset(%ebp), %esi
4312 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::LEA32r), DestReg: BasePtr),
4313 Reg: FramePtr, Offset: EndOffset)
4314 .setMIFlag(MachineInstr::FrameSetup);
4315 // MOV32rm SavedEBPOffset(%esi), %ebp
4316 assert(X86FI->getHasSEHFramePtrSave());
4317 int Offset =
4318 getFrameIndexReference(MF, FI: X86FI->getSEHFramePtrSaveIndex(), FrameReg&: UsedReg)
4319 .getFixed();
4320 assert(UsedReg == BasePtr);
4321 addRegOffset(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV32rm), DestReg: FramePtr),
4322 Reg: UsedReg, Offset)
4323 .setMIFlag(MachineInstr::FrameSetup);
4324 } else {
4325 llvm_unreachable("32-bit frames with WinEH must use FramePtr or BasePtr");
4326 }
4327 return MBBI;
4328}
4329
4330int X86FrameLowering::getInitialCFAOffset(const MachineFunction &MF) const {
4331 return TRI->getSlotSize();
4332}
4333
4334Register
4335X86FrameLowering::getInitialCFARegister(const MachineFunction &MF) const {
4336 return StackPtr;
4337}
4338
4339TargetFrameLowering::DwarfFrameBase
4340X86FrameLowering::getDwarfFrameBase(const MachineFunction &MF) const {
4341 const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
4342 Register FrameRegister = RI->getFrameRegister(MF);
4343 if (getInitialCFARegister(MF) == FrameRegister &&
4344 MF.getInfo<X86MachineFunctionInfo>()->hasCFIAdjustCfa()) {
4345 DwarfFrameBase FrameBase;
4346 FrameBase.Kind = DwarfFrameBase::CFA;
4347 FrameBase.Location.Offset =
4348 -MF.getFrameInfo().getStackSize() - getInitialCFAOffset(MF);
4349 return FrameBase;
4350 }
4351
4352 return DwarfFrameBase{.Kind: DwarfFrameBase::Register, .Location: {.Reg: FrameRegister}};
4353}
4354
4355namespace {
4356// Struct used by orderFrameObjects to help sort the stack objects.
4357struct X86FrameSortingObject {
4358 bool IsValid = false; // true if we care about this Object.
4359 unsigned ObjectIndex = 0; // Index of Object into MFI list.
4360 unsigned ObjectSize = 0; // Size of Object in bytes.
4361 Align ObjectAlignment = Align(1); // Alignment of Object in bytes.
4362 unsigned ObjectNumUses = 0; // Object static number of uses.
4363};
4364
4365// The comparison function we use for std::sort to order our local
4366// stack symbols. The current algorithm is to use an estimated
4367// "density". This takes into consideration the size and number of
4368// uses each object has in order to roughly minimize code size.
4369// So, for example, an object of size 16B that is referenced 5 times
4370// will get higher priority than 4 4B objects referenced 1 time each.
4371// It's not perfect and we may be able to squeeze a few more bytes out of
4372// it (for example : 0(esp) requires fewer bytes, symbols allocated at the
4373// fringe end can have special consideration, given their size is less
4374// important, etc.), but the algorithmic complexity grows too much to be
4375// worth the extra gains we get. This gets us pretty close.
4376// The final order leaves us with objects with highest priority going
4377// at the end of our list.
4378struct X86FrameSortingComparator {
4379 inline bool operator()(const X86FrameSortingObject &A,
4380 const X86FrameSortingObject &B) const {
4381 uint64_t DensityAScaled, DensityBScaled;
4382
4383 // For consistency in our comparison, all invalid objects are placed
4384 // at the end. This also allows us to stop walking when we hit the
4385 // first invalid item after it's all sorted.
4386 if (!A.IsValid)
4387 return false;
4388 if (!B.IsValid)
4389 return true;
4390
4391 // The density is calculated by doing :
4392 // (double)DensityA = A.ObjectNumUses / A.ObjectSize
4393 // (double)DensityB = B.ObjectNumUses / B.ObjectSize
4394 // Since this approach may cause inconsistencies in
4395 // the floating point <, >, == comparisons, depending on the floating
4396 // point model with which the compiler was built, we're going
4397 // to scale both sides by multiplying with
4398 // A.ObjectSize * B.ObjectSize. This ends up factoring away
4399 // the division and, with it, the need for any floating point
4400 // arithmetic.
4401 DensityAScaled = static_cast<uint64_t>(A.ObjectNumUses) *
4402 static_cast<uint64_t>(B.ObjectSize);
4403 DensityBScaled = static_cast<uint64_t>(B.ObjectNumUses) *
4404 static_cast<uint64_t>(A.ObjectSize);
4405
4406 // If the two densities are equal, prioritize highest alignment
4407 // objects. This allows for similar alignment objects
4408 // to be packed together (given the same density).
4409 // There's room for improvement here, also, since we can pack
4410 // similar alignment (different density) objects next to each
4411 // other to save padding. This will also require further
4412 // complexity/iterations, and the overall gain isn't worth it,
4413 // in general. Something to keep in mind, though.
4414 if (DensityAScaled == DensityBScaled)
4415 return A.ObjectAlignment < B.ObjectAlignment;
4416
4417 return DensityAScaled < DensityBScaled;
4418 }
4419};
4420} // namespace
4421
4422// Order the symbols in the local stack.
4423// We want to place the local stack objects in some sort of sensible order.
4424// The heuristic we use is to try and pack them according to static number
4425// of uses and size of object in order to minimize code size.
4426void X86FrameLowering::orderFrameObjects(
4427 const MachineFunction &MF, SmallVectorImpl<int> &ObjectsToAllocate) const {
4428 const MachineFrameInfo &MFI = MF.getFrameInfo();
4429
4430 // Don't waste time if there's nothing to do.
4431 if (ObjectsToAllocate.empty())
4432 return;
4433
4434 // Create an array of all MFI objects. We won't need all of these
4435 // objects, but we're going to create a full array of them to make
4436 // it easier to index into when we're counting "uses" down below.
4437 // We want to be able to easily/cheaply access an object by simply
4438 // indexing into it, instead of having to search for it every time.
4439 std::vector<X86FrameSortingObject> SortingObjects(MFI.getObjectIndexEnd());
4440
4441 // Walk the objects we care about and mark them as such in our working
4442 // struct.
4443 for (auto &Obj : ObjectsToAllocate) {
4444 SortingObjects[Obj].IsValid = true;
4445 SortingObjects[Obj].ObjectIndex = Obj;
4446 SortingObjects[Obj].ObjectAlignment = MFI.getObjectAlign(ObjectIdx: Obj);
4447 // Set the size.
4448 int ObjectSize = MFI.getObjectSize(ObjectIdx: Obj);
4449 if (ObjectSize == 0)
4450 // Variable size. Just use 4.
4451 SortingObjects[Obj].ObjectSize = 4;
4452 else
4453 SortingObjects[Obj].ObjectSize = ObjectSize;
4454 }
4455
4456 // Count the number of uses for each object.
4457 for (auto &MBB : MF) {
4458 for (auto &MI : MBB) {
4459 if (MI.isDebugInstr())
4460 continue;
4461 for (const MachineOperand &MO : MI.operands()) {
4462 // Check to see if it's a local stack symbol.
4463 if (!MO.isFI())
4464 continue;
4465 int Index = MO.getIndex();
4466 // Check to see if it falls within our range, and is tagged
4467 // to require ordering.
4468 if (Index >= 0 && Index < MFI.getObjectIndexEnd() &&
4469 SortingObjects[Index].IsValid)
4470 SortingObjects[Index].ObjectNumUses++;
4471 }
4472 }
4473 }
4474
4475 // Sort the objects using X86FrameSortingAlgorithm (see its comment for
4476 // info).
4477 llvm::stable_sort(Range&: SortingObjects, C: X86FrameSortingComparator());
4478
4479 // Now modify the original list to represent the final order that
4480 // we want. The order will depend on whether we're going to access them
4481 // from the stack pointer or the frame pointer. For SP, the list should
4482 // end up with the END containing objects that we want with smaller offsets.
4483 // For FP, it should be flipped.
4484 int i = 0;
4485 for (auto &Obj : SortingObjects) {
4486 // All invalid items are sorted at the end, so it's safe to stop.
4487 if (!Obj.IsValid)
4488 break;
4489 ObjectsToAllocate[i++] = Obj.ObjectIndex;
4490 }
4491
4492 // Flip it if we're accessing off of the FP.
4493 if (!TRI->hasStackRealignment(MF) && hasFP(MF))
4494 std::reverse(first: ObjectsToAllocate.begin(), last: ObjectsToAllocate.end());
4495}
4496
4497unsigned
4498X86FrameLowering::getWinEHParentFrameOffset(const MachineFunction &MF) const {
4499 // RDX, the parent frame pointer, is homed into 16(%rsp) in the prologue.
4500 unsigned Offset = 16;
4501 // RBP is immediately pushed.
4502 Offset += SlotSize;
4503 // All callee-saved registers are then pushed.
4504 Offset += MF.getInfo<X86MachineFunctionInfo>()->getCalleeSavedFrameSize();
4505 // Every funclet allocates enough stack space for the largest outgoing call.
4506 Offset += getWinEHFuncletFrameSize(MF);
4507 return Offset;
4508}
4509
4510void X86FrameLowering::processFunctionBeforeFrameFinalized(
4511 MachineFunction &MF, RegScavenger *RS) const {
4512 // Mark the function as not having WinCFI. We will set it back to true in
4513 // emitPrologue if it gets called and emits CFI.
4514 MF.setHasWinCFI(false);
4515
4516 MachineFrameInfo &MFI = MF.getFrameInfo();
4517 // If the frame is big enough that we might need to scavenge a register to
4518 // handle huge offsets, reserve a stack slot for that now.
4519 if (!isInt<32>(x: MFI.estimateStackSize(MF))) {
4520 int FI = MFI.CreateStackObject(Size: SlotSize, Alignment: Align(SlotSize), isSpillSlot: false);
4521 RS->addScavengingFrameIndex(FI);
4522 }
4523
4524 // If we are using Windows x64 CFI, ensure that the stack is always 8 byte
4525 // aligned. The format doesn't support misaligned stack adjustments.
4526 if (MF.usesWindowsUnwindInfo())
4527 MF.getFrameInfo().ensureMaxAlignment(Alignment: Align(SlotSize));
4528
4529 // If this function isn't doing Win64-style C++ EH, we don't need to do
4530 // anything.
4531 if (STI.is64Bit() && MF.hasEHFunclets() &&
4532 classifyEHPersonality(Pers: MF.getFunction().getPersonalityFn()) ==
4533 EHPersonality::MSVC_CXX) {
4534 adjustFrameForMsvcCxxEh(MF);
4535 }
4536}
4537
4538void X86FrameLowering::adjustFrameForMsvcCxxEh(MachineFunction &MF) const {
4539 // Win64 C++ EH needs to allocate the UnwindHelp object at some fixed offset
4540 // relative to RSP after the prologue. Find the offset of the last fixed
4541 // object, so that we can allocate a slot immediately following it. If there
4542 // were no fixed objects, use offset -SlotSize, which is immediately after the
4543 // return address. Fixed objects have negative frame indices.
4544 MachineFrameInfo &MFI = MF.getFrameInfo();
4545 WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo();
4546 int64_t MinFixedObjOffset = -SlotSize;
4547 for (int I = MFI.getObjectIndexBegin(); I < 0; ++I)
4548 MinFixedObjOffset = std::min(a: MinFixedObjOffset, b: MFI.getObjectOffset(ObjectIdx: I));
4549
4550 for (WinEHTryBlockMapEntry &TBME : EHInfo.TryBlockMap) {
4551 for (WinEHHandlerType &H : TBME.HandlerArray) {
4552 int FrameIndex = H.CatchObj.FrameIndex;
4553 if ((FrameIndex != INT_MAX) && MFI.getObjectOffset(ObjectIdx: FrameIndex) == 0) {
4554 // Ensure alignment.
4555 unsigned Align = MFI.getObjectAlign(ObjectIdx: FrameIndex).value();
4556 MinFixedObjOffset -= std::abs(i: MinFixedObjOffset) % Align;
4557 MinFixedObjOffset -= MFI.getObjectSize(ObjectIdx: FrameIndex);
4558 MFI.setObjectOffset(ObjectIdx: FrameIndex, SPOffset: MinFixedObjOffset);
4559 }
4560 }
4561 }
4562
4563 // Ensure alignment.
4564 MinFixedObjOffset -= std::abs(i: MinFixedObjOffset) % 8;
4565 int64_t UnwindHelpOffset = MinFixedObjOffset - SlotSize;
4566 int UnwindHelpFI =
4567 MFI.CreateFixedObject(Size: SlotSize, SPOffset: UnwindHelpOffset, /*IsImmutable=*/false);
4568 EHInfo.UnwindHelpFrameIdx = UnwindHelpFI;
4569
4570 // Store -2 into UnwindHelp on function entry. We have to scan forwards past
4571 // other frame setup instructions.
4572 MachineBasicBlock &MBB = MF.front();
4573 auto MBBI = MBB.begin();
4574 while (MBBI != MBB.end() && MBBI->getFlag(Flag: MachineInstr::FrameSetup))
4575 ++MBBI;
4576
4577 DebugLoc DL = MBB.findDebugLoc(MBBI);
4578 addFrameReference(MIB: BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64mi32)),
4579 FI: UnwindHelpFI)
4580 .addImm(Val: -2);
4581}
4582
4583void X86FrameLowering::processFunctionBeforeFrameIndicesReplaced(
4584 MachineFunction &MF, RegScavenger *RS) const {
4585 auto *X86FI = MF.getInfo<X86MachineFunctionInfo>();
4586
4587 if (STI.is32Bit() && MF.hasEHFunclets())
4588 restoreWinEHStackPointersInParent(MF);
4589 // We have emitted prolog and epilog. Don't need stack pointer saving
4590 // instruction any more.
4591 if (MachineInstr *MI = X86FI->getStackPtrSaveMI()) {
4592 MI->eraseFromParent();
4593 X86FI->setStackPtrSaveMI(nullptr);
4594 }
4595}
4596
4597void X86FrameLowering::restoreWinEHStackPointersInParent(
4598 MachineFunction &MF) const {
4599 // 32-bit functions have to restore stack pointers when control is transferred
4600 // back to the parent function. These blocks are identified as eh pads that
4601 // are not funclet entries.
4602 bool IsSEH = isAsynchronousEHPersonality(
4603 Pers: classifyEHPersonality(Pers: MF.getFunction().getPersonalityFn()));
4604 for (MachineBasicBlock &MBB : MF) {
4605 bool NeedsRestore = MBB.isEHPad() && !MBB.isEHFuncletEntry();
4606 if (NeedsRestore)
4607 restoreWin32EHStackPointers(MBB, MBBI: MBB.begin(), DL: DebugLoc(),
4608 /*RestoreSP=*/IsSEH);
4609 }
4610}
4611
4612// Compute the alignment gap between current SP after spilling FP/BP and the
4613// next properly aligned stack offset.
4614static int computeFPBPAlignmentGap(MachineFunction &MF,
4615 const TargetRegisterClass *RC,
4616 unsigned NumSpilledRegs) {
4617 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
4618 unsigned AllocSize = TRI->getSpillSize(RC: *RC) * NumSpilledRegs;
4619 Align StackAlign = MF.getSubtarget().getFrameLowering()->getStackAlign();
4620 unsigned AlignedSize = alignTo(Size: AllocSize, A: StackAlign);
4621 return AlignedSize - AllocSize;
4622}
4623
4624void X86FrameLowering::spillFPBPUsingSP(MachineFunction &MF,
4625 MachineBasicBlock::iterator BeforeMI,
4626 Register FP, Register BP,
4627 int SPAdjust) const {
4628 assert(FP.isValid() || BP.isValid());
4629
4630 MachineBasicBlock *MBB = BeforeMI->getParent();
4631 DebugLoc DL = BeforeMI->getDebugLoc();
4632
4633 // Spill FP.
4634 if (FP.isValid()) {
4635 BuildMI(BB&: *MBB, I: BeforeMI, MIMD: DL,
4636 MCID: TII.get(Opcode: getPUSHOpcode(ST: MF.getSubtarget<X86Subtarget>())))
4637 .addReg(RegNo: FP);
4638 }
4639
4640 // Spill BP.
4641 if (BP.isValid()) {
4642 BuildMI(BB&: *MBB, I: BeforeMI, MIMD: DL,
4643 MCID: TII.get(Opcode: getPUSHOpcode(ST: MF.getSubtarget<X86Subtarget>())))
4644 .addReg(RegNo: BP);
4645 }
4646
4647 // Make sure SP is aligned.
4648 if (SPAdjust)
4649 emitSPUpdate(MBB&: *MBB, MBBI&: BeforeMI, DL, NumBytes: -SPAdjust, InEpilogue: false);
4650
4651 // Emit unwinding information.
4652 if (FP.isValid() && needsDwarfCFI(MF)) {
4653 // Emit .cfi_remember_state to remember old frame.
4654 unsigned CFIIndex =
4655 MF.addFrameInst(Inst: MCCFIInstruction::createRememberState(L: nullptr));
4656 BuildMI(BB&: *MBB, I: BeforeMI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::CFI_INSTRUCTION))
4657 .addCFIIndex(CFIIndex);
4658
4659 // Setup new CFA value with DW_CFA_def_cfa_expression:
4660 // DW_OP_breg7+offset, DW_OP_deref, DW_OP_consts 16, DW_OP_plus
4661 SmallString<64> CfaExpr;
4662 uint8_t buffer[16];
4663 int Offset = SPAdjust;
4664 if (BP.isValid())
4665 Offset += TRI->getSpillSize(RC: *TRI->getMinimalPhysRegClass(Reg: BP));
4666 // If BeforeMI is a frame setup instruction, we need to adjust the position
4667 // and offset of the new cfi instruction.
4668 if (TII.isFrameSetup(I: *BeforeMI)) {
4669 Offset += alignTo(Size: TII.getFrameSize(I: *BeforeMI), A: getStackAlign());
4670 BeforeMI = std::next(x: BeforeMI);
4671 }
4672 Register StackPtr = TRI->getStackRegister();
4673 if (STI.isTarget64BitILP32())
4674 StackPtr = Register(getX86SubSuperRegister(Reg: StackPtr, Size: 64));
4675 unsigned DwarfStackPtr = TRI->getDwarfRegNum(Reg: StackPtr, isEH: true);
4676 CfaExpr.push_back(Elt: (uint8_t)(dwarf::DW_OP_breg0 + DwarfStackPtr));
4677 CfaExpr.append(in_start: buffer, in_end: buffer + encodeSLEB128(Value: Offset, p: buffer));
4678 CfaExpr.push_back(Elt: dwarf::DW_OP_deref);
4679 CfaExpr.push_back(Elt: dwarf::DW_OP_consts);
4680 CfaExpr.append(in_start: buffer, in_end: buffer + encodeSLEB128(Value: SlotSize * 2, p: buffer));
4681 CfaExpr.push_back(Elt: (uint8_t)dwarf::DW_OP_plus);
4682
4683 SmallString<64> DefCfaExpr;
4684 DefCfaExpr.push_back(Elt: dwarf::DW_CFA_def_cfa_expression);
4685 DefCfaExpr.append(in_start: buffer, in_end: buffer + encodeSLEB128(Value: CfaExpr.size(), p: buffer));
4686 DefCfaExpr.append(RHS: CfaExpr.str());
4687 BuildCFI(MBB&: *MBB, MBBI: BeforeMI, DL,
4688 CFIInst: MCCFIInstruction::createEscape(L: nullptr, Vals: DefCfaExpr.str()),
4689 Flag: MachineInstr::FrameSetup);
4690 }
4691}
4692
4693void X86FrameLowering::restoreFPBPUsingSP(MachineFunction &MF,
4694 MachineBasicBlock::iterator AfterMI,
4695 Register FP, Register BP,
4696 int SPAdjust) const {
4697 assert(FP.isValid() || BP.isValid());
4698
4699 // Adjust SP so it points to spilled FP or BP.
4700 MachineBasicBlock *MBB = AfterMI->getParent();
4701 MachineBasicBlock::iterator Pos = std::next(x: AfterMI);
4702 DebugLoc DL = AfterMI->getDebugLoc();
4703 if (SPAdjust)
4704 emitSPUpdate(MBB&: *MBB, MBBI&: Pos, DL, NumBytes: SPAdjust, InEpilogue: false);
4705
4706 // Restore BP.
4707 if (BP.isValid()) {
4708 BuildMI(BB&: *MBB, I: Pos, MIMD: DL,
4709 MCID: TII.get(Opcode: getPOPOpcode(ST: MF.getSubtarget<X86Subtarget>())), DestReg: BP);
4710 }
4711
4712 // Restore FP.
4713 if (FP.isValid()) {
4714 BuildMI(BB&: *MBB, I: Pos, MIMD: DL,
4715 MCID: TII.get(Opcode: getPOPOpcode(ST: MF.getSubtarget<X86Subtarget>())), DestReg: FP);
4716
4717 // Emit unwinding information.
4718 if (needsDwarfCFI(MF)) {
4719 // Restore original frame with .cfi_restore_state.
4720 unsigned CFIIndex =
4721 MF.addFrameInst(Inst: MCCFIInstruction::createRestoreState(L: nullptr));
4722 BuildMI(BB&: *MBB, I: Pos, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::CFI_INSTRUCTION))
4723 .addCFIIndex(CFIIndex);
4724 }
4725 }
4726}
4727
4728void X86FrameLowering::saveAndRestoreFPBPUsingSP(
4729 MachineFunction &MF, MachineBasicBlock::iterator BeforeMI,
4730 MachineBasicBlock::iterator AfterMI, bool SpillFP, bool SpillBP) const {
4731 assert(SpillFP || SpillBP);
4732
4733 Register FP, BP;
4734 const TargetRegisterClass *RC;
4735 unsigned NumRegs = 0;
4736
4737 if (SpillFP) {
4738 FP = TRI->getFrameRegister(MF);
4739 if (STI.isTarget64BitILP32())
4740 FP = Register(getX86SubSuperRegister(Reg: FP, Size: 64));
4741 RC = TRI->getMinimalPhysRegClass(Reg: FP);
4742 ++NumRegs;
4743 }
4744 if (SpillBP) {
4745 BP = TRI->getBaseRegister();
4746 if (STI.isTarget64BitILP32())
4747 BP = Register(getX86SubSuperRegister(Reg: BP, Size: 64));
4748 RC = TRI->getMinimalPhysRegClass(Reg: BP);
4749 ++NumRegs;
4750 }
4751 int SPAdjust = computeFPBPAlignmentGap(MF, RC, NumSpilledRegs: NumRegs);
4752
4753 spillFPBPUsingSP(MF, BeforeMI, FP, BP, SPAdjust);
4754 restoreFPBPUsingSP(MF, AfterMI, FP, BP, SPAdjust);
4755}
4756
4757bool X86FrameLowering::skipSpillFPBP(
4758 MachineFunction &MF, MachineBasicBlock::reverse_iterator &MI) const {
4759 if (MI->getOpcode() == X86::LCMPXCHG16B_SAVE_RBX) {
4760 // The pseudo instruction LCMPXCHG16B_SAVE_RBX is generated in the form
4761 // SaveRbx = COPY RBX
4762 // SaveRbx = LCMPXCHG16B_SAVE_RBX ..., SaveRbx, implicit-def rbx
4763 // And later LCMPXCHG16B_SAVE_RBX is expanded to restore RBX from SaveRbx.
4764 // We should skip this instruction sequence.
4765 int FI;
4766 Register Reg;
4767 while (!(MI->getOpcode() == TargetOpcode::COPY &&
4768 MI->getOperand(i: 1).getReg() == X86::RBX) &&
4769 !((Reg = TII.isStoreToStackSlot(MI: *MI, FrameIndex&: FI)) && Reg == X86::RBX))
4770 ++MI;
4771 return true;
4772 }
4773 return false;
4774}
4775
4776static bool isFPBPAccess(const MachineInstr &MI, Register FP, Register BP,
4777 const TargetRegisterInfo *TRI, bool &AccessFP,
4778 bool &AccessBP) {
4779 AccessFP = AccessBP = false;
4780 if (FP) {
4781 if (MI.findRegisterUseOperandIdx(Reg: FP, TRI, isKill: false) != -1 ||
4782 MI.findRegisterDefOperandIdx(Reg: FP, TRI, isDead: false, Overlap: true) != -1)
4783 AccessFP = true;
4784 }
4785 if (BP) {
4786 if (MI.findRegisterUseOperandIdx(Reg: BP, TRI, isKill: false) != -1 ||
4787 MI.findRegisterDefOperandIdx(Reg: BP, TRI, isDead: false, Overlap: true) != -1)
4788 AccessBP = true;
4789 }
4790 return AccessFP || AccessBP;
4791}
4792
4793// Invoke instruction has been lowered to normal function call. We try to figure
4794// out if MI comes from Invoke.
4795// Do we have any better method?
4796static bool isInvoke(const MachineInstr &MI, bool InsideEHLabels) {
4797 if (!MI.isCall())
4798 return false;
4799 if (InsideEHLabels)
4800 return true;
4801
4802 const MachineBasicBlock *MBB = MI.getParent();
4803 if (!MBB->hasEHPadSuccessor())
4804 return false;
4805
4806 // Check if there is another call instruction from MI to the end of MBB.
4807 MachineBasicBlock::const_iterator MBBI = MI, ME = MBB->end();
4808 for (++MBBI; MBBI != ME; ++MBBI)
4809 if (MBBI->isCall())
4810 return false;
4811 return true;
4812}
4813
4814/// Given the live range of FP or BP (DefMI, KillMI), check if there is any
4815/// interfered stack access in the range, usually generated by register spill.
4816void X86FrameLowering::checkInterferedAccess(
4817 MachineFunction &MF, MachineBasicBlock::reverse_iterator DefMI,
4818 MachineBasicBlock::reverse_iterator KillMI, bool SpillFP,
4819 bool SpillBP) const {
4820 if (DefMI == KillMI)
4821 return;
4822 if (TRI->hasBasePointer(MF)) {
4823 if (!SpillBP)
4824 return;
4825 } else {
4826 if (!SpillFP)
4827 return;
4828 }
4829
4830 auto MI = KillMI;
4831 while (MI != DefMI) {
4832 if (any_of(Range: MI->operands(),
4833 P: [](const MachineOperand &MO) { return MO.isFI(); }))
4834 MF.getContext().reportError(L: SMLoc(),
4835 Msg: "Interference usage of base pointer/frame "
4836 "pointer.");
4837 MI++;
4838 }
4839}
4840
4841/// If a function uses base pointer and the base pointer is clobbered by inline
4842/// asm, RA doesn't detect this case, and after the inline asm, the base pointer
4843/// contains garbage value.
4844/// For example if a 32b x86 function uses base pointer esi, and esi is
4845/// clobbered by following inline asm
4846/// asm("rep movsb" : "+D"(ptr), "+S"(x), "+c"(c)::"memory");
4847/// We need to save esi before the asm and restore it after the asm.
4848///
4849/// The problem can also occur to frame pointer if there is a function call, and
4850/// the callee uses a different calling convention and clobbers the fp.
4851///
4852/// Because normal frame objects (spill slots) are accessed through fp/bp
4853/// register, so we can't spill fp/bp to normal spill slots.
4854///
4855/// FIXME: There are 2 possible enhancements:
4856/// 1. In many cases there are different physical registers not clobbered by
4857/// inline asm, we can use one of them as base pointer. Or use a virtual
4858/// register as base pointer and let RA allocate a physical register to it.
4859/// 2. If there is no other instructions access stack with fp/bp from the
4860/// inline asm to the epilog, and no cfi requirement for a correct fp, we can
4861/// skip the save and restore operations.
4862void X86FrameLowering::spillFPBP(MachineFunction &MF) const {
4863 Register FP, BP;
4864 const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering();
4865 if (TFI.hasFP(MF))
4866 FP = TRI->getFrameRegister(MF);
4867 if (TRI->hasBasePointer(MF))
4868 BP = TRI->getBaseRegister();
4869
4870 // Currently only inline asm and function call can clobbers fp/bp. So we can
4871 // do some quick test and return early.
4872 if (!MF.hasInlineAsm()) {
4873 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
4874 if (!X86FI->getFPClobberedByCall())
4875 FP = 0;
4876 if (!X86FI->getBPClobberedByCall())
4877 BP = 0;
4878 }
4879 if (!FP && !BP)
4880 return;
4881
4882 for (MachineBasicBlock &MBB : MF) {
4883 bool InsideEHLabels = false;
4884 auto MI = MBB.rbegin(), ME = MBB.rend();
4885 auto TermMI = MBB.getFirstTerminator();
4886 if (TermMI == MBB.begin())
4887 continue;
4888 MI = *(std::prev(x: TermMI));
4889
4890 while (MI != ME) {
4891 // Skip frame setup/destroy instructions.
4892 // Skip Invoke (call inside try block) instructions.
4893 // Skip instructions handled by target.
4894 if (MI->getFlag(Flag: MachineInstr::MIFlag::FrameSetup) ||
4895 MI->getFlag(Flag: MachineInstr::MIFlag::FrameDestroy) ||
4896 isInvoke(MI: *MI, InsideEHLabels) || skipSpillFPBP(MF, MI)) {
4897 ++MI;
4898 continue;
4899 }
4900
4901 if (MI->getOpcode() == TargetOpcode::EH_LABEL) {
4902 InsideEHLabels = !InsideEHLabels;
4903 ++MI;
4904 continue;
4905 }
4906
4907 bool AccessFP, AccessBP;
4908 // Check if fp or bp is used in MI.
4909 if (!isFPBPAccess(MI: *MI, FP, BP, TRI, AccessFP, AccessBP)) {
4910 ++MI;
4911 continue;
4912 }
4913
4914 // Look for the range [DefMI, KillMI] in which fp or bp is defined and
4915 // used.
4916 bool FPLive = false, BPLive = false;
4917 bool SpillFP = false, SpillBP = false;
4918 auto DefMI = MI, KillMI = MI;
4919 do {
4920 SpillFP |= AccessFP;
4921 SpillBP |= AccessBP;
4922
4923 // Maintain FPLive and BPLive.
4924 if (FPLive && MI->findRegisterDefOperandIdx(Reg: FP, TRI, isDead: false, Overlap: true) != -1)
4925 FPLive = false;
4926 if (FP && MI->findRegisterUseOperandIdx(Reg: FP, TRI, isKill: false) != -1)
4927 FPLive = true;
4928 if (BPLive && MI->findRegisterDefOperandIdx(Reg: BP, TRI, isDead: false, Overlap: true) != -1)
4929 BPLive = false;
4930 if (BP && MI->findRegisterUseOperandIdx(Reg: BP, TRI, isKill: false) != -1)
4931 BPLive = true;
4932
4933 DefMI = MI++;
4934 } while ((MI != ME) &&
4935 (FPLive || BPLive ||
4936 isFPBPAccess(MI: *MI, FP, BP, TRI, AccessFP, AccessBP)));
4937
4938 // Don't need to save/restore if FP is accessed through llvm.frameaddress.
4939 if (FPLive && !SpillBP)
4940 continue;
4941
4942 // If the bp is clobbered by a call, we should save and restore outside of
4943 // the frame setup instructions.
4944 if (KillMI->isCall() && DefMI != ME) {
4945 auto FrameSetup = std::next(x: DefMI);
4946 // Look for frame setup instruction toward the start of the BB.
4947 // If we reach another call instruction, it means no frame setup
4948 // instruction for the current call instruction.
4949 while (FrameSetup != ME && !TII.isFrameSetup(I: *FrameSetup) &&
4950 !FrameSetup->isCall())
4951 ++FrameSetup;
4952 // If a frame setup instruction is found, we need to find out the
4953 // corresponding frame destroy instruction.
4954 if (FrameSetup != ME && TII.isFrameSetup(I: *FrameSetup) &&
4955 (TII.getFrameSize(I: *FrameSetup) ||
4956 TII.getFrameAdjustment(I: *FrameSetup))) {
4957 while (!TII.isFrameInstr(I: *KillMI))
4958 --KillMI;
4959 DefMI = FrameSetup;
4960 MI = DefMI;
4961 ++MI;
4962 }
4963 }
4964
4965 checkInterferedAccess(MF, DefMI, KillMI, SpillFP, SpillBP);
4966
4967 // Call target function to spill and restore FP and BP registers.
4968 saveAndRestoreFPBPUsingSP(MF, BeforeMI: &(*DefMI), AfterMI: &(*KillMI), SpillFP, SpillBP);
4969 }
4970 }
4971}
4972