1//===----- X86DynAllocaExpander.cpp - Expand DynAlloca pseudo instruction -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines a pass that expands DynAlloca pseudo-instructions.
10//
11// It performs a conservative analysis to determine whether each allocation
12// falls within a region of the stack that is safe to use, or whether stack
13// probes must be emitted.
14//
15//===----------------------------------------------------------------------===//
16
17#include "X86.h"
18#include "X86InstrInfo.h"
19#include "X86MachineFunctionInfo.h"
20#include "X86Subtarget.h"
21#include "llvm/ADT/MapVector.h"
22#include "llvm/ADT/PostOrderIterator.h"
23#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
24#include "llvm/CodeGen/MachineFunctionPass.h"
25#include "llvm/CodeGen/MachineInstrBuilder.h"
26#include "llvm/CodeGen/MachineRegisterInfo.h"
27#include "llvm/CodeGen/Passes.h"
28#include "llvm/CodeGen/RegisterClassInfo.h"
29#include "llvm/CodeGen/TargetInstrInfo.h"
30#include "llvm/IR/Analysis.h"
31#include "llvm/IR/Function.h"
32
33using namespace llvm;
34
35namespace {
36
37class X86DynAllocaExpander {
38public:
39 bool run(MachineFunction &MF);
40
41private:
42 /// Strategies for lowering a DynAlloca.
43 enum Lowering { TouchAndSub, Sub, Probe };
44
45 /// Deterministic-order map from DynAlloca instruction to desired lowering.
46 typedef MapVector<MachineInstr*, Lowering> LoweringMap;
47
48 /// Compute which lowering to use for each DynAlloca instruction.
49 void computeLowerings(MachineFunction &MF, LoweringMap& Lowerings);
50
51 /// Get the appropriate lowering based on current offset and amount.
52 Lowering getLowering(int64_t CurrentOffset, int64_t AllocaAmount);
53
54 /// Lower a DynAlloca instruction.
55 void lower(MachineInstr* MI, Lowering L);
56
57 MachineRegisterInfo *MRI = nullptr;
58 const X86Subtarget *STI = nullptr;
59 const TargetInstrInfo *TII = nullptr;
60 const X86RegisterInfo *TRI = nullptr;
61 Register StackPtr;
62 unsigned SlotSize = 0;
63 int64_t StackProbeSize = 0;
64 bool NoStackArgProbe = false;
65};
66
67class X86DynAllocaExpanderLegacy : public MachineFunctionPass {
68public:
69 X86DynAllocaExpanderLegacy() : MachineFunctionPass(ID) {}
70
71 bool runOnMachineFunction(MachineFunction &MF) override;
72
73 void getAnalysisUsage(AnalysisUsage &AU) const override {
74 AU.addPreserved<MachineRegisterClassInfoWrapperPass>();
75 MachineFunctionPass::getAnalysisUsage(AU);
76 }
77
78private:
79 StringRef getPassName() const override { return "X86 DynAlloca Expander"; }
80
81public:
82 static char ID;
83};
84
85char X86DynAllocaExpanderLegacy::ID = 0;
86
87} // end anonymous namespace
88
89INITIALIZE_PASS(X86DynAllocaExpanderLegacy, "x86-dyn-alloca-expander",
90 "X86 DynAlloca Expander", false, false)
91
92FunctionPass *llvm::createX86DynAllocaExpanderLegacyPass() {
93 return new X86DynAllocaExpanderLegacy();
94}
95
96/// Return the allocation amount for a DynAlloca instruction, or -1 if unknown.
97static int64_t getDynAllocaAmount(MachineInstr *MI, MachineRegisterInfo *MRI) {
98 assert(MI->getOpcode() == X86::DYN_ALLOCA_32 ||
99 MI->getOpcode() == X86::DYN_ALLOCA_64);
100 assert(MI->getOperand(0).isReg());
101
102 Register AmountReg = MI->getOperand(i: 0).getReg();
103 MachineInstr *Def = MRI->getUniqueVRegDef(Reg: AmountReg);
104
105 if (!Def ||
106 (Def->getOpcode() != X86::MOV32ri && Def->getOpcode() != X86::MOV64ri) ||
107 !Def->getOperand(i: 1).isImm())
108 return -1;
109
110 return Def->getOperand(i: 1).getImm();
111}
112
113X86DynAllocaExpander::Lowering
114X86DynAllocaExpander::getLowering(int64_t CurrentOffset,
115 int64_t AllocaAmount) {
116 // For a non-constant amount or a large amount, we have to probe.
117 if (AllocaAmount < 0 || AllocaAmount > StackProbeSize)
118 return Probe;
119
120 // If it fits within the safe region of the stack, just subtract.
121 if (CurrentOffset + AllocaAmount <= StackProbeSize)
122 return Sub;
123
124 // Otherwise, touch the current tip of the stack, then subtract.
125 return TouchAndSub;
126}
127
128static bool isPushPop(const MachineInstr &MI) {
129 switch (MI.getOpcode()) {
130 case X86::PUSH32r:
131 case X86::PUSH32rmm:
132 case X86::PUSH32rmr:
133 case X86::PUSH32i:
134 case X86::PUSH64r:
135 case X86::PUSH64rmm:
136 case X86::PUSH64rmr:
137 case X86::PUSH64i32:
138 case X86::POP32r:
139 case X86::POP64r:
140 return true;
141 default:
142 return false;
143 }
144}
145
146void X86DynAllocaExpander::computeLowerings(MachineFunction &MF,
147 LoweringMap &Lowerings) {
148 // Do a one-pass reverse post-order walk of the CFG to conservatively estimate
149 // the offset between the stack pointer and the lowest touched part of the
150 // stack, and use that to decide how to lower each DynAlloca instruction.
151
152 // Initialize OutOffset[B], the stack offset at exit from B, to something big.
153 DenseMap<MachineBasicBlock *, int64_t> OutOffset;
154 for (MachineBasicBlock &MBB : MF)
155 OutOffset[&MBB] = INT32_MAX;
156
157 // Note: we don't know the offset at the start of the entry block since the
158 // prologue hasn't been inserted yet, and how much that will adjust the stack
159 // pointer depends on register spills, which have not been computed yet.
160
161 // Compute the reverse post-order.
162 ReversePostOrderTraversal<MachineFunction*> RPO(&MF);
163
164 for (MachineBasicBlock *MBB : RPO) {
165 int64_t Offset = -1;
166 for (MachineBasicBlock *Pred : MBB->predecessors())
167 Offset = std::max(a: Offset, b: OutOffset[Pred]);
168 if (Offset == -1) Offset = INT32_MAX;
169
170 for (MachineInstr &MI : *MBB) {
171 if (MI.getOpcode() == X86::DYN_ALLOCA_32 ||
172 MI.getOpcode() == X86::DYN_ALLOCA_64) {
173 // A DynAlloca moves StackPtr, and potentially touches it.
174 int64_t Amount = getDynAllocaAmount(MI: &MI, MRI);
175 Lowering L = getLowering(CurrentOffset: Offset, AllocaAmount: Amount);
176 Lowerings[&MI] = L;
177 switch (L) {
178 case Sub:
179 Offset += Amount;
180 break;
181 case TouchAndSub:
182 Offset = Amount;
183 break;
184 case Probe:
185 Offset = 0;
186 break;
187 }
188 } else if (MI.isCall() || isPushPop(MI)) {
189 // Calls, pushes and pops touch the tip of the stack.
190 Offset = 0;
191 } else if (MI.getOpcode() == X86::ADJCALLSTACKUP32 ||
192 MI.getOpcode() == X86::ADJCALLSTACKUP64) {
193 Offset -= MI.getOperand(i: 0).getImm();
194 } else if (MI.getOpcode() == X86::ADJCALLSTACKDOWN32 ||
195 MI.getOpcode() == X86::ADJCALLSTACKDOWN64) {
196 Offset += MI.getOperand(i: 0).getImm();
197 } else if (MI.modifiesRegister(Reg: StackPtr, TRI)) {
198 // Any other modification of SP means we've lost track of it.
199 Offset = INT32_MAX;
200 }
201 }
202
203 OutOffset[MBB] = Offset;
204 }
205}
206
207static unsigned getSubOpcode(bool Is64Bit) {
208 if (Is64Bit)
209 return X86::SUB64ri32;
210 return X86::SUB32ri;
211}
212
213void X86DynAllocaExpander::lower(MachineInstr *MI, Lowering L) {
214 const DebugLoc &DL = MI->getDebugLoc();
215 MachineBasicBlock *MBB = MI->getParent();
216 MachineBasicBlock::iterator I = *MI;
217
218 int64_t Amount = getDynAllocaAmount(MI, MRI);
219 if (Amount == 0) {
220 MI->eraseFromParent();
221 return;
222 }
223
224 // These two variables differ on x32, which is a 64-bit target with a
225 // 32-bit alloca.
226 bool Is64Bit = STI->is64Bit();
227 bool Is64BitAlloca = MI->getOpcode() == X86::DYN_ALLOCA_64;
228 bool DeadEFLAGS = MI->registerDefIsDead(Reg: X86::EFLAGS, /*TRI=*/nullptr);
229 assert(SlotSize == 4 || SlotSize == 8);
230
231 std::optional<MachineFunction::DebugInstrOperandPair> InstrNum;
232 if (unsigned Num = MI->peekDebugInstrNum()) {
233 // Operand 2 of DYN_ALLOCAs contains the stack def.
234 InstrNum = {Num, 2};
235 }
236
237 switch (L) {
238 case TouchAndSub: {
239 assert(Amount >= SlotSize);
240
241 // Use a push to touch the top of the stack.
242 unsigned RegA = Is64Bit ? X86::RAX : X86::EAX;
243 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII->get(Opcode: Is64Bit ? X86::PUSH64r : X86::PUSH32r))
244 .addReg(RegNo: RegA, Flags: RegState::Undef);
245 Amount -= SlotSize;
246 if (!Amount)
247 break;
248
249 // Fall through to make any remaining adjustment.
250 [[fallthrough]];
251 }
252 case Sub:
253 assert(Amount > 0);
254 if (Amount == SlotSize) {
255 // Use push to save size.
256 unsigned RegA = Is64Bit ? X86::RAX : X86::EAX;
257 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII->get(Opcode: Is64Bit ? X86::PUSH64r : X86::PUSH32r))
258 .addReg(RegNo: RegA, Flags: RegState::Undef);
259 } else {
260 // Sub.
261 MachineInstrBuilder Sub =
262 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII->get(Opcode: getSubOpcode(Is64Bit: Is64BitAlloca)), DestReg: StackPtr)
263 .addReg(RegNo: StackPtr)
264 .addImm(Val: Amount);
265 if (DeadEFLAGS)
266 Sub.setOperandDead(3); // implicit-def $eflags
267 }
268 break;
269 case Probe:
270 if (!NoStackArgProbe) {
271 // The probe lowering expects the amount in RAX/EAX.
272 unsigned RegA = Is64BitAlloca ? X86::RAX : X86::EAX;
273 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: RegA)
274 .addReg(RegNo: MI->getOperand(i: 0).getReg());
275
276 // Do the probe.
277 STI->getFrameLowering()->emitStackProbe(MF&: *MBB->getParent(), MBB&: *MBB, MBBI: MI, DL,
278 /*InProlog=*/false, InstrNum);
279 } else {
280 // Sub
281 MachineInstrBuilder Sub =
282 BuildMI(BB&: *MBB, I, MIMD: DL,
283 MCID: TII->get(Opcode: Is64BitAlloca ? X86::SUB64rr : X86::SUB32rr),
284 DestReg: StackPtr)
285 .addReg(RegNo: StackPtr)
286 .addReg(RegNo: MI->getOperand(i: 0).getReg());
287 if (DeadEFLAGS)
288 Sub.setOperandDead(3); // implicit-def $eflags
289 }
290 break;
291 }
292
293 Register AmountReg = MI->getOperand(i: 0).getReg();
294 MI->eraseFromParent();
295
296 // Delete the definition of AmountReg.
297 if (MRI->use_empty(RegNo: AmountReg))
298 if (MachineInstr *AmountDef = MRI->getUniqueVRegDef(Reg: AmountReg))
299 AmountDef->eraseFromParent();
300}
301
302bool X86DynAllocaExpander::run(MachineFunction &MF) {
303 if (!MF.getInfo<X86MachineFunctionInfo>()->hasDynAlloca())
304 return false;
305
306 MRI = &MF.getRegInfo();
307 STI = &MF.getSubtarget<X86Subtarget>();
308 TII = STI->getInstrInfo();
309 TRI = STI->getRegisterInfo();
310 StackPtr = TRI->getStackRegister();
311 SlotSize = TRI->getSlotSize();
312 StackProbeSize = STI->getTargetLowering()->getStackProbeSize(MF);
313 NoStackArgProbe = MF.getFunction().hasFnAttribute(Kind: "no-stack-arg-probe");
314 if (NoStackArgProbe)
315 StackProbeSize = INT64_MAX;
316
317 LoweringMap Lowerings;
318 computeLowerings(MF, Lowerings);
319 for (auto &P : Lowerings)
320 lower(MI: P.first, L: P.second);
321
322 return true;
323}
324
325bool X86DynAllocaExpanderLegacy::runOnMachineFunction(MachineFunction &MF) {
326 return X86DynAllocaExpander().run(MF);
327}
328
329PreservedAnalyses
330X86DynAllocaExpanderPass::run(MachineFunction &MF,
331 MachineFunctionAnalysisManager &MFAM) {
332 bool Changed = X86DynAllocaExpander().run(MF);
333 if (!Changed)
334 return PreservedAnalyses::all();
335
336 return getMachineFunctionPassPreservedAnalyses().preserveSet<CFGAnalyses>();
337}
338