1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file defines the pass which inserts x86 AVX vzeroupper instructions
11/// before calls to SSE encoded functions. This avoids transition latency
12/// penalty when transferring control between AVX encoded instructions and old
13/// SSE encoding mode.
14///
15//===----------------------------------------------------------------------===//
16
17#include "X86.h"
18#include "X86InstrInfo.h"
19#include "X86Subtarget.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/ADT/Statistic.h"
22#include "llvm/CodeGen/MachineBasicBlock.h"
23#include "llvm/CodeGen/MachineFunction.h"
24#include "llvm/CodeGen/MachineFunctionPass.h"
25#include "llvm/CodeGen/MachineInstr.h"
26#include "llvm/CodeGen/MachineInstrBuilder.h"
27#include "llvm/CodeGen/MachineOperand.h"
28#include "llvm/CodeGen/MachinePassManager.h"
29#include "llvm/CodeGen/MachineRegisterInfo.h"
30#include "llvm/CodeGen/TargetInstrInfo.h"
31#include "llvm/CodeGen/TargetRegisterInfo.h"
32#include "llvm/IR/Analysis.h"
33#include "llvm/IR/CallingConv.h"
34#include "llvm/IR/DebugLoc.h"
35#include "llvm/IR/Function.h"
36#include "llvm/Support/Debug.h"
37#include "llvm/Support/ErrorHandling.h"
38#include "llvm/Support/raw_ostream.h"
39#include <cassert>
40
41using namespace llvm;
42
43#define DEBUG_TYPE "x86-insert-vzeroupper"
44
45STATISTIC(NumVZU, "Number of vzeroupper instructions inserted");
46
47namespace {
48class X86InsertVZeroUpperLegacy : public MachineFunctionPass {
49public:
50 static char ID;
51
52 X86InsertVZeroUpperLegacy() : MachineFunctionPass(ID) {}
53
54 StringRef getPassName() const override { return "X86 vzeroupper inserter"; }
55
56 bool runOnMachineFunction(MachineFunction &MF) override;
57
58 MachineFunctionProperties getRequiredProperties() const override {
59 return MachineFunctionProperties().setNoVRegs();
60 }
61};
62
63enum BlockExitState { PASS_THROUGH, EXITS_CLEAN, EXITS_DIRTY };
64
65// Core algorithm state:
66// BlockState - Each block is either:
67// - PASS_THROUGH: There are neither YMM/ZMM dirtying instructions nor
68// vzeroupper instructions in this block.
69// - EXITS_CLEAN: There is (or will be) a vzeroupper instruction in this
70// block that will ensure that YMM/ZMM is clean on exit.
71// - EXITS_DIRTY: An instruction in the block dirties YMM/ZMM and no
72// subsequent vzeroupper in the block clears it.
73//
74// AddedToDirtySuccessors - This flag is raised when a block is added to the
75// DirtySuccessors list to ensure that it's not
76// added multiple times.
77//
78// FirstUnguardedCall - Records the location of the first unguarded call in
79// each basic block that may need to be guarded by a
80// vzeroupper. We won't know whether it actually needs
81// to be guarded until we discover a predecessor that
82// is DIRTY_OUT.
83struct BlockState {
84 BlockExitState ExitState = PASS_THROUGH;
85 bool AddedToDirtySuccessors = false;
86 MachineBasicBlock::iterator FirstUnguardedCall;
87
88 BlockState() = default;
89};
90
91using BlockStateMap = SmallVector<BlockState, 8>;
92using DirtySuccessorsWorkList = SmallVector<MachineBasicBlock *, 8>;
93} // end anonymous namespace
94
95char X86InsertVZeroUpperLegacy::ID = 0;
96
97FunctionPass *llvm::createX86InsertVZeroUpperLegacyPass() {
98 return new X86InsertVZeroUpperLegacy();
99}
100
101#ifndef NDEBUG
102static const char *getBlockExitStateName(BlockExitState ST) {
103 switch (ST) {
104 case PASS_THROUGH:
105 return "Pass-through";
106 case EXITS_DIRTY:
107 return "Exits-dirty";
108 case EXITS_CLEAN:
109 return "Exits-clean";
110 }
111 llvm_unreachable("Invalid block exit state.");
112}
113#endif
114
115/// VZEROUPPER cleans state that is related to Y/ZMM0-15 only.
116/// Thus, there is no need to check for Y/ZMM16 and above.
117static bool isYmmOrZmmReg(MCRegister Reg) {
118 return (Reg >= X86::YMM0 && Reg <= X86::YMM15) ||
119 (Reg >= X86::ZMM0 && Reg <= X86::ZMM15);
120}
121
122static bool checkFnHasLiveInYmmOrZmm(MachineRegisterInfo &MRI) {
123 for (std::pair<MCRegister, Register> LI : MRI.liveins())
124 if (isYmmOrZmmReg(Reg: LI.first))
125 return true;
126
127 return false;
128}
129
130static bool clobbersAllYmmAndZmmRegs(const MachineOperand &MO) {
131 for (unsigned reg = X86::YMM0; reg <= X86::YMM15; ++reg) {
132 if (!MO.clobbersPhysReg(PhysReg: reg))
133 return false;
134 }
135 for (unsigned reg = X86::ZMM0; reg <= X86::ZMM15; ++reg) {
136 if (!MO.clobbersPhysReg(PhysReg: reg))
137 return false;
138 }
139 return true;
140}
141
142static bool hasYmmOrZmmReg(MachineInstr &MI) {
143 for (const MachineOperand &MO : MI.operands()) {
144 if (MI.isCall() && MO.isRegMask() && !clobbersAllYmmAndZmmRegs(MO))
145 return true;
146 if (!MO.isReg())
147 continue;
148 if (MO.isDebug())
149 continue;
150 if (isYmmOrZmmReg(Reg: MO.getReg().asMCReg()))
151 return true;
152 }
153 return false;
154}
155
156/// Check if given call instruction has a RegMask operand.
157static bool callHasRegMask(MachineInstr &MI) {
158 assert(MI.isCall() && "Can only be called on call instructions.");
159 for (const MachineOperand &MO : MI.operands()) {
160 if (MO.isRegMask())
161 return true;
162 }
163 return false;
164}
165
166/// Insert a vzeroupper instruction before I.
167static bool insertVZeroUpper(MachineBasicBlock::iterator I,
168 MachineBasicBlock &MBB,
169 const TargetInstrInfo *TII) {
170 BuildMI(BB&: MBB, I, MIMD: I->getDebugLoc(), MCID: TII->get(Opcode: X86::VZEROUPPER));
171 ++NumVZU;
172 return true;
173}
174
175/// Add MBB to the DirtySuccessors list if it hasn't already been added.
176static void addDirtySuccessor(MachineBasicBlock &MBB,
177 BlockStateMap &BlockStates,
178 DirtySuccessorsWorkList &DirtySuccessors) {
179 if (!BlockStates[MBB.getNumber()].AddedToDirtySuccessors) {
180 DirtySuccessors.push_back(Elt: &MBB);
181 BlockStates[MBB.getNumber()].AddedToDirtySuccessors = true;
182 }
183}
184
185/// Loop over all of the instructions in the basic block, inserting vzeroupper
186/// instructions before function calls.
187static bool processBasicBlock(MachineBasicBlock &MBB,
188 BlockStateMap &BlockStates,
189 DirtySuccessorsWorkList &DirtySuccessors,
190 bool IsX86INTR, const TargetInstrInfo *TII) {
191 // Start by assuming that the block is PASS_THROUGH which implies no unguarded
192 // calls.
193 BlockExitState CurState = PASS_THROUGH;
194 BlockStates[MBB.getNumber()].FirstUnguardedCall = MBB.end();
195 bool MadeChange = false;
196
197 for (MachineInstr &MI : MBB) {
198 bool IsCall = MI.isCall();
199 bool IsReturn = MI.isReturn();
200 bool IsControlFlow = IsCall || IsReturn;
201
202 // No need for vzeroupper before iret in interrupt handler function,
203 // epilogue will restore YMM/ZMM registers if needed.
204 if (IsX86INTR && IsReturn)
205 continue;
206
207 // An existing VZERO* instruction resets the state.
208 if (MI.getOpcode() == X86::VZEROALL || MI.getOpcode() == X86::VZEROUPPER) {
209 CurState = EXITS_CLEAN;
210 continue;
211 }
212
213 // Shortcut: don't need to check regular instructions in dirty state.
214 if (!IsControlFlow && CurState == EXITS_DIRTY)
215 continue;
216
217 if (hasYmmOrZmmReg(MI)) {
218 // We found a ymm/zmm-using instruction; this could be an AVX/AVX512
219 // instruction, or it could be control flow.
220 CurState = EXITS_DIRTY;
221 continue;
222 }
223
224 // Check for control-flow out of the current function (which might
225 // indirectly execute SSE instructions).
226 if (!IsControlFlow)
227 continue;
228
229 // If the call has no RegMask, skip it as well. It usually happens on
230 // helper function calls (such as '_chkstk', '_ftol2') where standard
231 // calling convention is not used (RegMask is not used to mark register
232 // clobbered and register usage (def/implicit-def/use) is well-defined and
233 // explicitly specified.
234 if (IsCall && !callHasRegMask(MI))
235 continue;
236
237 // The VZEROUPPER instruction resets the upper 128 bits of YMM0-YMM15
238 // registers. In addition, the processor changes back to Clean state, after
239 // which execution of SSE instructions or AVX instructions has no transition
240 // penalty. Add the VZEROUPPER instruction before any function call/return
241 // that might execute SSE code.
242 // FIXME: In some cases, we may want to move the VZEROUPPER into a
243 // predecessor block.
244 if (CurState == EXITS_DIRTY) {
245 // After the inserted VZEROUPPER the state becomes clean again, but
246 // other YMM/ZMM may appear before other subsequent calls or even before
247 // the end of the BB.
248 MadeChange |= insertVZeroUpper(I: MI, MBB, TII);
249 CurState = EXITS_CLEAN;
250 } else if (CurState == PASS_THROUGH) {
251 // If this block is currently in pass-through state and we encounter a
252 // call then whether we need a vzeroupper or not depends on whether this
253 // block has successors that exit dirty. Record the location of the call,
254 // and set the state to EXITS_CLEAN, but do not insert the vzeroupper yet.
255 // It will be inserted later if necessary.
256 BlockStates[MBB.getNumber()].FirstUnguardedCall = MI;
257 CurState = EXITS_CLEAN;
258 }
259 }
260
261 LLVM_DEBUG(dbgs() << "MBB #" << MBB.getNumber() << " exit state: "
262 << getBlockExitStateName(CurState) << '\n');
263
264 if (CurState == EXITS_DIRTY)
265 for (MachineBasicBlock *Succ : MBB.successors())
266 addDirtySuccessor(MBB&: *Succ, BlockStates, DirtySuccessors);
267
268 BlockStates[MBB.getNumber()].ExitState = CurState;
269 return MadeChange;
270}
271
272/// Loop over all of the basic blocks, inserting vzeroupper instructions before
273/// function calls.
274static bool insertVZeroUpper(MachineFunction &MF) {
275 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
276 if (!ST.getCLOpts().use_vzeroupper || !ST.hasAVX() || !ST.insertVZEROUPPER())
277 return false;
278
279 MachineRegisterInfo &MRI = MF.getRegInfo();
280
281 bool FnHasLiveInYmmOrZmm = checkFnHasLiveInYmmOrZmm(MRI);
282
283 // Fast check: if the function doesn't use any ymm/zmm registers, we don't
284 // need to insert any VZEROUPPER instructions. This is constant-time, so it
285 // is cheap in the common case of no ymm/zmm use.
286 bool YmmOrZmmUsed = FnHasLiveInYmmOrZmm;
287 for (const auto *RC : {&X86::VR256RegClass, &X86::VR512_0_15RegClass}) {
288 if (!YmmOrZmmUsed) {
289 for (MCPhysReg R : *RC) {
290 if (!MRI.reg_nodbg_empty(RegNo: R)) {
291 YmmOrZmmUsed = true;
292 break;
293 }
294 }
295 }
296 }
297 if (!YmmOrZmmUsed)
298 return false;
299
300 const TargetInstrInfo *TII = ST.getInstrInfo();
301 bool IsX86INTR = MF.getFunction().getCallingConv() == CallingConv::X86_INTR;
302 bool EverMadeChange = false;
303 BlockStateMap BlockStates(MF.getNumBlockIDs());
304 DirtySuccessorsWorkList DirtySuccessors;
305
306 assert(BlockStates.size() == MF.getNumBlockIDs() && DirtySuccessors.empty() &&
307 "X86VZeroUpper state should be clear");
308
309 // Process all blocks. This will compute block exit states, record the first
310 // unguarded call in each block, and add successors of dirty blocks to the
311 // DirtySuccessors list.
312 for (MachineBasicBlock &MBB : MF)
313 EverMadeChange |=
314 processBasicBlock(MBB, BlockStates, DirtySuccessors, IsX86INTR, TII);
315
316 // If any YMM/ZMM regs are live-in to this function, add the entry block to
317 // the DirtySuccessors list
318 if (FnHasLiveInYmmOrZmm)
319 addDirtySuccessor(MBB&: MF.front(), BlockStates, DirtySuccessors);
320
321 // Re-visit all blocks that are successors of EXITS_DIRTY blocks. Add
322 // vzeroupper instructions to unguarded calls, and propagate EXITS_DIRTY
323 // through PASS_THROUGH blocks.
324 while (!DirtySuccessors.empty()) {
325 MachineBasicBlock &MBB = *DirtySuccessors.back();
326 DirtySuccessors.pop_back();
327 BlockState &BBState = BlockStates[MBB.getNumber()];
328
329 // MBB is a successor of a dirty block, so its first call needs to be
330 // guarded.
331 if (BBState.FirstUnguardedCall != MBB.end())
332 EverMadeChange |= insertVZeroUpper(I: BBState.FirstUnguardedCall, MBB, TII);
333
334 // If this successor was a pass-through block, then it is now dirty. Its
335 // successors need to be added to the worklist (if they haven't been
336 // already).
337 if (BBState.ExitState == PASS_THROUGH) {
338 LLVM_DEBUG(dbgs() << "MBB #" << MBB.getNumber()
339 << " was Pass-through, is now Dirty-out.\n");
340 for (MachineBasicBlock *Succ : MBB.successors())
341 addDirtySuccessor(MBB&: *Succ, BlockStates, DirtySuccessors);
342 }
343 }
344
345 return EverMadeChange;
346}
347
348bool X86InsertVZeroUpperLegacy::runOnMachineFunction(MachineFunction &MF) {
349 return insertVZeroUpper(MF);
350}
351
352PreservedAnalyses
353X86InsertVZeroUpperPass::run(MachineFunction &MF,
354 MachineFunctionAnalysisManager &MFAM) {
355 return insertVZeroUpper(MF) ? getMachineFunctionPassPreservedAnalyses()
356 .preserveSet<CFGAnalyses>()
357 : PreservedAnalyses::all();
358}
359