1//===----- X86DynAllocaExpander.cpp - Expand DynAlloca pseudo instruction -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines a pass that expands DynAlloca pseudo-instructions.
10//
11// It performs a conservative analysis to determine whether each allocation
12// falls within a region of the stack that is safe to use, or whether stack
13// probes must be emitted.
14//
15//===----------------------------------------------------------------------===//
16
17#include "X86.h"
18#include "X86InstrInfo.h"
19#include "X86MachineFunctionInfo.h"
20#include "X86Subtarget.h"
21#include "llvm/ADT/MapVector.h"
22#include "llvm/ADT/PostOrderIterator.h"
23#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
24#include "llvm/CodeGen/MachineFunctionPass.h"
25#include "llvm/CodeGen/MachineInstrBuilder.h"
26#include "llvm/CodeGen/MachineRegisterInfo.h"
27#include "llvm/CodeGen/Passes.h"
28#include "llvm/CodeGen/TargetInstrInfo.h"
29#include "llvm/IR/Analysis.h"
30#include "llvm/IR/Function.h"
31
32using namespace llvm;
33
34namespace {
35
36class X86DynAllocaExpander {
37public:
38 bool run(MachineFunction &MF);
39
40private:
41 /// Strategies for lowering a DynAlloca.
42 enum Lowering { TouchAndSub, Sub, Probe };
43
44 /// Deterministic-order map from DynAlloca instruction to desired lowering.
45 typedef MapVector<MachineInstr*, Lowering> LoweringMap;
46
47 /// Compute which lowering to use for each DynAlloca instruction.
48 void computeLowerings(MachineFunction &MF, LoweringMap& Lowerings);
49
50 /// Get the appropriate lowering based on current offset and amount.
51 Lowering getLowering(int64_t CurrentOffset, int64_t AllocaAmount);
52
53 /// Lower a DynAlloca instruction.
54 void lower(MachineInstr* MI, Lowering L);
55
56 MachineRegisterInfo *MRI = nullptr;
57 const X86Subtarget *STI = nullptr;
58 const TargetInstrInfo *TII = nullptr;
59 const X86RegisterInfo *TRI = nullptr;
60 Register StackPtr;
61 unsigned SlotSize = 0;
62 int64_t StackProbeSize = 0;
63 bool NoStackArgProbe = false;
64};
65
66class X86DynAllocaExpanderLegacy : public MachineFunctionPass {
67public:
68 X86DynAllocaExpanderLegacy() : MachineFunctionPass(ID) {}
69
70 bool runOnMachineFunction(MachineFunction &MF) override;
71
72private:
73 StringRef getPassName() const override { return "X86 DynAlloca Expander"; }
74
75public:
76 static char ID;
77};
78
79char X86DynAllocaExpanderLegacy::ID = 0;
80
81} // end anonymous namespace
82
83INITIALIZE_PASS(X86DynAllocaExpanderLegacy, "x86-dyn-alloca-expander",
84 "X86 DynAlloca Expander", false, false)
85
86FunctionPass *llvm::createX86DynAllocaExpanderLegacyPass() {
87 return new X86DynAllocaExpanderLegacy();
88}
89
90/// Return the allocation amount for a DynAlloca instruction, or -1 if unknown.
91static int64_t getDynAllocaAmount(MachineInstr *MI, MachineRegisterInfo *MRI) {
92 assert(MI->getOpcode() == X86::DYN_ALLOCA_32 ||
93 MI->getOpcode() == X86::DYN_ALLOCA_64);
94 assert(MI->getOperand(0).isReg());
95
96 Register AmountReg = MI->getOperand(i: 0).getReg();
97 MachineInstr *Def = MRI->getUniqueVRegDef(Reg: AmountReg);
98
99 if (!Def ||
100 (Def->getOpcode() != X86::MOV32ri && Def->getOpcode() != X86::MOV64ri) ||
101 !Def->getOperand(i: 1).isImm())
102 return -1;
103
104 return Def->getOperand(i: 1).getImm();
105}
106
107X86DynAllocaExpander::Lowering
108X86DynAllocaExpander::getLowering(int64_t CurrentOffset,
109 int64_t AllocaAmount) {
110 // For a non-constant amount or a large amount, we have to probe.
111 if (AllocaAmount < 0 || AllocaAmount > StackProbeSize)
112 return Probe;
113
114 // If it fits within the safe region of the stack, just subtract.
115 if (CurrentOffset + AllocaAmount <= StackProbeSize)
116 return Sub;
117
118 // Otherwise, touch the current tip of the stack, then subtract.
119 return TouchAndSub;
120}
121
122static bool isPushPop(const MachineInstr &MI) {
123 switch (MI.getOpcode()) {
124 case X86::PUSH32r:
125 case X86::PUSH32rmm:
126 case X86::PUSH32rmr:
127 case X86::PUSH32i:
128 case X86::PUSH64r:
129 case X86::PUSH64rmm:
130 case X86::PUSH64rmr:
131 case X86::PUSH64i32:
132 case X86::POP32r:
133 case X86::POP64r:
134 return true;
135 default:
136 return false;
137 }
138}
139
140void X86DynAllocaExpander::computeLowerings(MachineFunction &MF,
141 LoweringMap &Lowerings) {
142 // Do a one-pass reverse post-order walk of the CFG to conservatively estimate
143 // the offset between the stack pointer and the lowest touched part of the
144 // stack, and use that to decide how to lower each DynAlloca instruction.
145
146 // Initialize OutOffset[B], the stack offset at exit from B, to something big.
147 DenseMap<MachineBasicBlock *, int64_t> OutOffset;
148 for (MachineBasicBlock &MBB : MF)
149 OutOffset[&MBB] = INT32_MAX;
150
151 // Note: we don't know the offset at the start of the entry block since the
152 // prologue hasn't been inserted yet, and how much that will adjust the stack
153 // pointer depends on register spills, which have not been computed yet.
154
155 // Compute the reverse post-order.
156 ReversePostOrderTraversal<MachineFunction*> RPO(&MF);
157
158 for (MachineBasicBlock *MBB : RPO) {
159 int64_t Offset = -1;
160 for (MachineBasicBlock *Pred : MBB->predecessors())
161 Offset = std::max(a: Offset, b: OutOffset[Pred]);
162 if (Offset == -1) Offset = INT32_MAX;
163
164 for (MachineInstr &MI : *MBB) {
165 if (MI.getOpcode() == X86::DYN_ALLOCA_32 ||
166 MI.getOpcode() == X86::DYN_ALLOCA_64) {
167 // A DynAlloca moves StackPtr, and potentially touches it.
168 int64_t Amount = getDynAllocaAmount(MI: &MI, MRI);
169 Lowering L = getLowering(CurrentOffset: Offset, AllocaAmount: Amount);
170 Lowerings[&MI] = L;
171 switch (L) {
172 case Sub:
173 Offset += Amount;
174 break;
175 case TouchAndSub:
176 Offset = Amount;
177 break;
178 case Probe:
179 Offset = 0;
180 break;
181 }
182 } else if (MI.isCall() || isPushPop(MI)) {
183 // Calls, pushes and pops touch the tip of the stack.
184 Offset = 0;
185 } else if (MI.getOpcode() == X86::ADJCALLSTACKUP32 ||
186 MI.getOpcode() == X86::ADJCALLSTACKUP64) {
187 Offset -= MI.getOperand(i: 0).getImm();
188 } else if (MI.getOpcode() == X86::ADJCALLSTACKDOWN32 ||
189 MI.getOpcode() == X86::ADJCALLSTACKDOWN64) {
190 Offset += MI.getOperand(i: 0).getImm();
191 } else if (MI.modifiesRegister(Reg: StackPtr, TRI)) {
192 // Any other modification of SP means we've lost track of it.
193 Offset = INT32_MAX;
194 }
195 }
196
197 OutOffset[MBB] = Offset;
198 }
199}
200
201static unsigned getSubOpcode(bool Is64Bit) {
202 if (Is64Bit)
203 return X86::SUB64ri32;
204 return X86::SUB32ri;
205}
206
207void X86DynAllocaExpander::lower(MachineInstr *MI, Lowering L) {
208 const DebugLoc &DL = MI->getDebugLoc();
209 MachineBasicBlock *MBB = MI->getParent();
210 MachineBasicBlock::iterator I = *MI;
211
212 int64_t Amount = getDynAllocaAmount(MI, MRI);
213 if (Amount == 0) {
214 MI->eraseFromParent();
215 return;
216 }
217
218 // These two variables differ on x32, which is a 64-bit target with a
219 // 32-bit alloca.
220 bool Is64Bit = STI->is64Bit();
221 bool Is64BitAlloca = MI->getOpcode() == X86::DYN_ALLOCA_64;
222 assert(SlotSize == 4 || SlotSize == 8);
223
224 std::optional<MachineFunction::DebugInstrOperandPair> InstrNum;
225 if (unsigned Num = MI->peekDebugInstrNum()) {
226 // Operand 2 of DYN_ALLOCAs contains the stack def.
227 InstrNum = {Num, 2};
228 }
229
230 switch (L) {
231 case TouchAndSub: {
232 assert(Amount >= SlotSize);
233
234 // Use a push to touch the top of the stack.
235 unsigned RegA = Is64Bit ? X86::RAX : X86::EAX;
236 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII->get(Opcode: Is64Bit ? X86::PUSH64r : X86::PUSH32r))
237 .addReg(RegNo: RegA, Flags: RegState::Undef);
238 Amount -= SlotSize;
239 if (!Amount)
240 break;
241
242 // Fall through to make any remaining adjustment.
243 [[fallthrough]];
244 }
245 case Sub:
246 assert(Amount > 0);
247 if (Amount == SlotSize) {
248 // Use push to save size.
249 unsigned RegA = Is64Bit ? X86::RAX : X86::EAX;
250 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII->get(Opcode: Is64Bit ? X86::PUSH64r : X86::PUSH32r))
251 .addReg(RegNo: RegA, Flags: RegState::Undef);
252 } else {
253 // Sub.
254 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII->get(Opcode: getSubOpcode(Is64Bit: Is64BitAlloca)), DestReg: StackPtr)
255 .addReg(RegNo: StackPtr)
256 .addImm(Val: Amount);
257 }
258 break;
259 case Probe:
260 if (!NoStackArgProbe) {
261 // The probe lowering expects the amount in RAX/EAX.
262 unsigned RegA = Is64BitAlloca ? X86::RAX : X86::EAX;
263 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: RegA)
264 .addReg(RegNo: MI->getOperand(i: 0).getReg());
265
266 // Do the probe.
267 STI->getFrameLowering()->emitStackProbe(MF&: *MBB->getParent(), MBB&: *MBB, MBBI: MI, DL,
268 /*InProlog=*/false, InstrNum);
269 } else {
270 // Sub
271 BuildMI(BB&: *MBB, I, MIMD: DL,
272 MCID: TII->get(Opcode: Is64BitAlloca ? X86::SUB64rr : X86::SUB32rr), DestReg: StackPtr)
273 .addReg(RegNo: StackPtr)
274 .addReg(RegNo: MI->getOperand(i: 0).getReg());
275 }
276 break;
277 }
278
279 Register AmountReg = MI->getOperand(i: 0).getReg();
280 MI->eraseFromParent();
281
282 // Delete the definition of AmountReg.
283 if (MRI->use_empty(RegNo: AmountReg))
284 if (MachineInstr *AmountDef = MRI->getUniqueVRegDef(Reg: AmountReg))
285 AmountDef->eraseFromParent();
286}
287
288bool X86DynAllocaExpander::run(MachineFunction &MF) {
289 if (!MF.getInfo<X86MachineFunctionInfo>()->hasDynAlloca())
290 return false;
291
292 MRI = &MF.getRegInfo();
293 STI = &MF.getSubtarget<X86Subtarget>();
294 TII = STI->getInstrInfo();
295 TRI = STI->getRegisterInfo();
296 StackPtr = TRI->getStackRegister();
297 SlotSize = TRI->getSlotSize();
298 StackProbeSize = STI->getTargetLowering()->getStackProbeSize(MF);
299 NoStackArgProbe = MF.getFunction().hasFnAttribute(Kind: "no-stack-arg-probe");
300 if (NoStackArgProbe)
301 StackProbeSize = INT64_MAX;
302
303 LoweringMap Lowerings;
304 computeLowerings(MF, Lowerings);
305 for (auto &P : Lowerings)
306 lower(MI: P.first, L: P.second);
307
308 return true;
309}
310
311bool X86DynAllocaExpanderLegacy::runOnMachineFunction(MachineFunction &MF) {
312 return X86DynAllocaExpander().run(MF);
313}
314
315PreservedAnalyses
316X86DynAllocaExpanderPass::run(MachineFunction &MF,
317 MachineFunctionAnalysisManager &MFAM) {
318 bool Changed = X86DynAllocaExpander().run(MF);
319 if (!Changed)
320 return PreservedAnalyses::all();
321
322 return getMachineFunctionPassPreservedAnalyses().preserveSet<CFGAnalyses>();
323}
324