1//===- X86VZeroUpper.cpp - AVX vzeroupper instruction inserter ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the pass which inserts x86 AVX vzeroupper instructions
10// before calls to SSE encoded functions. This avoids transition latency
11// penalty when transferring control between AVX encoded instructions and old
12// SSE encoding mode.
13//
14//===----------------------------------------------------------------------===//
15
16#include "X86.h"
17#include "X86InstrInfo.h"
18#include "X86Subtarget.h"
19#include "llvm/ADT/SmallVector.h"
20#include "llvm/ADT/Statistic.h"
21#include "llvm/CodeGen/MachineBasicBlock.h"
22#include "llvm/CodeGen/MachineFunction.h"
23#include "llvm/CodeGen/MachineFunctionPass.h"
24#include "llvm/CodeGen/MachineInstr.h"
25#include "llvm/CodeGen/MachineInstrBuilder.h"
26#include "llvm/CodeGen/MachineOperand.h"
27#include "llvm/CodeGen/MachineRegisterInfo.h"
28#include "llvm/CodeGen/TargetInstrInfo.h"
29#include "llvm/CodeGen/TargetRegisterInfo.h"
30#include "llvm/IR/CallingConv.h"
31#include "llvm/IR/DebugLoc.h"
32#include "llvm/IR/Function.h"
33#include "llvm/Support/Debug.h"
34#include "llvm/Support/ErrorHandling.h"
35#include "llvm/Support/raw_ostream.h"
36#include <cassert>
37
38using namespace llvm;
39
40#define DEBUG_TYPE "x86-vzeroupper"
41
42static cl::opt<bool>
43UseVZeroUpper("x86-use-vzeroupper", cl::Hidden,
44 cl::desc("Minimize AVX to SSE transition penalty"),
45 cl::init(Val: true));
46
47STATISTIC(NumVZU, "Number of vzeroupper instructions inserted");
48
49namespace {
50
51 class VZeroUpperInserter : public MachineFunctionPass {
52 public:
53 VZeroUpperInserter() : MachineFunctionPass(ID) {}
54
55 bool runOnMachineFunction(MachineFunction &MF) override;
56
57 MachineFunctionProperties getRequiredProperties() const override {
58 return MachineFunctionProperties().set(
59 MachineFunctionProperties::Property::NoVRegs);
60 }
61
62 StringRef getPassName() const override { return "X86 vzeroupper inserter"; }
63
64 private:
65 void processBasicBlock(MachineBasicBlock &MBB);
66 void insertVZeroUpper(MachineBasicBlock::iterator I,
67 MachineBasicBlock &MBB);
68 void addDirtySuccessor(MachineBasicBlock &MBB);
69
70 using BlockExitState = enum { PASS_THROUGH, EXITS_CLEAN, EXITS_DIRTY };
71
72 static const char* getBlockExitStateName(BlockExitState ST);
73
74 // Core algorithm state:
75 // BlockState - Each block is either:
76 // - PASS_THROUGH: There are neither YMM/ZMM dirtying instructions nor
77 // vzeroupper instructions in this block.
78 // - EXITS_CLEAN: There is (or will be) a vzeroupper instruction in this
79 // block that will ensure that YMM/ZMM is clean on exit.
80 // - EXITS_DIRTY: An instruction in the block dirties YMM/ZMM and no
81 // subsequent vzeroupper in the block clears it.
82 //
83 // AddedToDirtySuccessors - This flag is raised when a block is added to the
84 // DirtySuccessors list to ensure that it's not
85 // added multiple times.
86 //
87 // FirstUnguardedCall - Records the location of the first unguarded call in
88 // each basic block that may need to be guarded by a
89 // vzeroupper. We won't know whether it actually needs
90 // to be guarded until we discover a predecessor that
91 // is DIRTY_OUT.
92 struct BlockState {
93 BlockExitState ExitState = PASS_THROUGH;
94 bool AddedToDirtySuccessors = false;
95 MachineBasicBlock::iterator FirstUnguardedCall;
96
97 BlockState() = default;
98 };
99
100 using BlockStateMap = SmallVector<BlockState, 8>;
101 using DirtySuccessorsWorkList = SmallVector<MachineBasicBlock *, 8>;
102
103 BlockStateMap BlockStates;
104 DirtySuccessorsWorkList DirtySuccessors;
105 bool EverMadeChange;
106 bool IsX86INTR;
107 const TargetInstrInfo *TII;
108
109 static char ID;
110 };
111
112} // end anonymous namespace
113
114char VZeroUpperInserter::ID = 0;
115
116FunctionPass *llvm::createX86IssueVZeroUpperPass() {
117 return new VZeroUpperInserter();
118}
119
120#ifndef NDEBUG
121const char* VZeroUpperInserter::getBlockExitStateName(BlockExitState ST) {
122 switch (ST) {
123 case PASS_THROUGH: return "Pass-through";
124 case EXITS_DIRTY: return "Exits-dirty";
125 case EXITS_CLEAN: return "Exits-clean";
126 }
127 llvm_unreachable("Invalid block exit state.");
128}
129#endif
130
131/// VZEROUPPER cleans state that is related to Y/ZMM0-15 only.
132/// Thus, there is no need to check for Y/ZMM16 and above.
133static bool isYmmOrZmmReg(unsigned Reg) {
134 return (Reg >= X86::YMM0 && Reg <= X86::YMM15) ||
135 (Reg >= X86::ZMM0 && Reg <= X86::ZMM15);
136}
137
138static bool checkFnHasLiveInYmmOrZmm(MachineRegisterInfo &MRI) {
139 for (std::pair<unsigned, unsigned> LI : MRI.liveins())
140 if (isYmmOrZmmReg(Reg: LI.first))
141 return true;
142
143 return false;
144}
145
146static bool clobbersAllYmmAndZmmRegs(const MachineOperand &MO) {
147 for (unsigned reg = X86::YMM0; reg <= X86::YMM15; ++reg) {
148 if (!MO.clobbersPhysReg(PhysReg: reg))
149 return false;
150 }
151 for (unsigned reg = X86::ZMM0; reg <= X86::ZMM15; ++reg) {
152 if (!MO.clobbersPhysReg(PhysReg: reg))
153 return false;
154 }
155 return true;
156}
157
158static bool hasYmmOrZmmReg(MachineInstr &MI) {
159 for (const MachineOperand &MO : MI.operands()) {
160 if (MI.isCall() && MO.isRegMask() && !clobbersAllYmmAndZmmRegs(MO))
161 return true;
162 if (!MO.isReg())
163 continue;
164 if (MO.isDebug())
165 continue;
166 if (isYmmOrZmmReg(Reg: MO.getReg()))
167 return true;
168 }
169 return false;
170}
171
172/// Check if given call instruction has a RegMask operand.
173static bool callHasRegMask(MachineInstr &MI) {
174 assert(MI.isCall() && "Can only be called on call instructions.");
175 for (const MachineOperand &MO : MI.operands()) {
176 if (MO.isRegMask())
177 return true;
178 }
179 return false;
180}
181
182/// Insert a vzeroupper instruction before I.
183void VZeroUpperInserter::insertVZeroUpper(MachineBasicBlock::iterator I,
184 MachineBasicBlock &MBB) {
185 BuildMI(BB&: MBB, I, MIMD: I->getDebugLoc(), MCID: TII->get(Opcode: X86::VZEROUPPER));
186 ++NumVZU;
187 EverMadeChange = true;
188}
189
190/// Add MBB to the DirtySuccessors list if it hasn't already been added.
191void VZeroUpperInserter::addDirtySuccessor(MachineBasicBlock &MBB) {
192 if (!BlockStates[MBB.getNumber()].AddedToDirtySuccessors) {
193 DirtySuccessors.push_back(Elt: &MBB);
194 BlockStates[MBB.getNumber()].AddedToDirtySuccessors = true;
195 }
196}
197
198/// Loop over all of the instructions in the basic block, inserting vzeroupper
199/// instructions before function calls.
200void VZeroUpperInserter::processBasicBlock(MachineBasicBlock &MBB) {
201 // Start by assuming that the block is PASS_THROUGH which implies no unguarded
202 // calls.
203 BlockExitState CurState = PASS_THROUGH;
204 BlockStates[MBB.getNumber()].FirstUnguardedCall = MBB.end();
205
206 for (MachineInstr &MI : MBB) {
207 bool IsCall = MI.isCall();
208 bool IsReturn = MI.isReturn();
209 bool IsControlFlow = IsCall || IsReturn;
210
211 // No need for vzeroupper before iret in interrupt handler function,
212 // epilogue will restore YMM/ZMM registers if needed.
213 if (IsX86INTR && IsReturn)
214 continue;
215
216 // An existing VZERO* instruction resets the state.
217 if (MI.getOpcode() == X86::VZEROALL || MI.getOpcode() == X86::VZEROUPPER) {
218 CurState = EXITS_CLEAN;
219 continue;
220 }
221
222 // Shortcut: don't need to check regular instructions in dirty state.
223 if (!IsControlFlow && CurState == EXITS_DIRTY)
224 continue;
225
226 if (hasYmmOrZmmReg(MI)) {
227 // We found a ymm/zmm-using instruction; this could be an AVX/AVX512
228 // instruction, or it could be control flow.
229 CurState = EXITS_DIRTY;
230 continue;
231 }
232
233 // Check for control-flow out of the current function (which might
234 // indirectly execute SSE instructions).
235 if (!IsControlFlow)
236 continue;
237
238 // If the call has no RegMask, skip it as well. It usually happens on
239 // helper function calls (such as '_chkstk', '_ftol2') where standard
240 // calling convention is not used (RegMask is not used to mark register
241 // clobbered and register usage (def/implicit-def/use) is well-defined and
242 // explicitly specified.
243 if (IsCall && !callHasRegMask(MI))
244 continue;
245
246 // The VZEROUPPER instruction resets the upper 128 bits of YMM0-YMM15
247 // registers. In addition, the processor changes back to Clean state, after
248 // which execution of SSE instructions or AVX instructions has no transition
249 // penalty. Add the VZEROUPPER instruction before any function call/return
250 // that might execute SSE code.
251 // FIXME: In some cases, we may want to move the VZEROUPPER into a
252 // predecessor block.
253 if (CurState == EXITS_DIRTY) {
254 // After the inserted VZEROUPPER the state becomes clean again, but
255 // other YMM/ZMM may appear before other subsequent calls or even before
256 // the end of the BB.
257 insertVZeroUpper(I: MI, MBB);
258 CurState = EXITS_CLEAN;
259 } else if (CurState == PASS_THROUGH) {
260 // If this block is currently in pass-through state and we encounter a
261 // call then whether we need a vzeroupper or not depends on whether this
262 // block has successors that exit dirty. Record the location of the call,
263 // and set the state to EXITS_CLEAN, but do not insert the vzeroupper yet.
264 // It will be inserted later if necessary.
265 BlockStates[MBB.getNumber()].FirstUnguardedCall = MI;
266 CurState = EXITS_CLEAN;
267 }
268 }
269
270 LLVM_DEBUG(dbgs() << "MBB #" << MBB.getNumber() << " exit state: "
271 << getBlockExitStateName(CurState) << '\n');
272
273 if (CurState == EXITS_DIRTY)
274 for (MachineBasicBlock *Succ : MBB.successors())
275 addDirtySuccessor(MBB&: *Succ);
276
277 BlockStates[MBB.getNumber()].ExitState = CurState;
278}
279
280/// Loop over all of the basic blocks, inserting vzeroupper instructions before
281/// function calls.
282bool VZeroUpperInserter::runOnMachineFunction(MachineFunction &MF) {
283 if (!UseVZeroUpper)
284 return false;
285
286 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
287 if (!ST.hasAVX() || !ST.insertVZEROUPPER())
288 return false;
289 TII = ST.getInstrInfo();
290 MachineRegisterInfo &MRI = MF.getRegInfo();
291 EverMadeChange = false;
292 IsX86INTR = MF.getFunction().getCallingConv() == CallingConv::X86_INTR;
293
294 bool FnHasLiveInYmmOrZmm = checkFnHasLiveInYmmOrZmm(MRI);
295
296 // Fast check: if the function doesn't use any ymm/zmm registers, we don't
297 // need to insert any VZEROUPPER instructions. This is constant-time, so it
298 // is cheap in the common case of no ymm/zmm use.
299 bool YmmOrZmmUsed = FnHasLiveInYmmOrZmm;
300 for (const auto *RC : {&X86::VR256RegClass, &X86::VR512_0_15RegClass}) {
301 if (!YmmOrZmmUsed) {
302 for (MCPhysReg R : *RC) {
303 if (!MRI.reg_nodbg_empty(RegNo: R)) {
304 YmmOrZmmUsed = true;
305 break;
306 }
307 }
308 }
309 }
310 if (!YmmOrZmmUsed)
311 return false;
312
313 assert(BlockStates.empty() && DirtySuccessors.empty() &&
314 "X86VZeroUpper state should be clear");
315 BlockStates.resize(N: MF.getNumBlockIDs());
316
317 // Process all blocks. This will compute block exit states, record the first
318 // unguarded call in each block, and add successors of dirty blocks to the
319 // DirtySuccessors list.
320 for (MachineBasicBlock &MBB : MF)
321 processBasicBlock(MBB);
322
323 // If any YMM/ZMM regs are live-in to this function, add the entry block to
324 // the DirtySuccessors list
325 if (FnHasLiveInYmmOrZmm)
326 addDirtySuccessor(MBB&: MF.front());
327
328 // Re-visit all blocks that are successors of EXITS_DIRTY blocks. Add
329 // vzeroupper instructions to unguarded calls, and propagate EXITS_DIRTY
330 // through PASS_THROUGH blocks.
331 while (!DirtySuccessors.empty()) {
332 MachineBasicBlock &MBB = *DirtySuccessors.back();
333 DirtySuccessors.pop_back();
334 BlockState &BBState = BlockStates[MBB.getNumber()];
335
336 // MBB is a successor of a dirty block, so its first call needs to be
337 // guarded.
338 if (BBState.FirstUnguardedCall != MBB.end())
339 insertVZeroUpper(I: BBState.FirstUnguardedCall, MBB);
340
341 // If this successor was a pass-through block, then it is now dirty. Its
342 // successors need to be added to the worklist (if they haven't been
343 // already).
344 if (BBState.ExitState == PASS_THROUGH) {
345 LLVM_DEBUG(dbgs() << "MBB #" << MBB.getNumber()
346 << " was Pass-through, is now Dirty-out.\n");
347 for (MachineBasicBlock *Succ : MBB.successors())
348 addDirtySuccessor(MBB&: *Succ);
349 }
350 }
351
352 BlockStates.clear();
353 return EverMadeChange;
354}
355