1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file defines the pass which inserts x86 AVX vzeroupper instructions
11/// before calls to SSE encoded functions. This avoids transition latency
12/// penalty when transferring control between AVX encoded instructions and old
13/// SSE encoding mode.
14///
15//===----------------------------------------------------------------------===//
16
17#include "X86.h"
18#include "X86InstrInfo.h"
19#include "X86Subtarget.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/ADT/Statistic.h"
22#include "llvm/CodeGen/MachineBasicBlock.h"
23#include "llvm/CodeGen/MachineFunction.h"
24#include "llvm/CodeGen/MachineFunctionPass.h"
25#include "llvm/CodeGen/MachineInstr.h"
26#include "llvm/CodeGen/MachineInstrBuilder.h"
27#include "llvm/CodeGen/MachineOperand.h"
28#include "llvm/CodeGen/MachinePassManager.h"
29#include "llvm/CodeGen/MachineRegisterInfo.h"
30#include "llvm/CodeGen/TargetInstrInfo.h"
31#include "llvm/CodeGen/TargetRegisterInfo.h"
32#include "llvm/IR/Analysis.h"
33#include "llvm/IR/CallingConv.h"
34#include "llvm/IR/DebugLoc.h"
35#include "llvm/IR/Function.h"
36#include "llvm/Support/Debug.h"
37#include "llvm/Support/ErrorHandling.h"
38#include "llvm/Support/raw_ostream.h"
39#include <cassert>
40
41using namespace llvm;
42
43#define DEBUG_TYPE "x86-insert-vzeroupper"
44
45static cl::opt<bool>
46 UseVZeroUpper("x86-use-vzeroupper", cl::Hidden,
47 cl::desc("Minimize AVX to SSE transition penalty"),
48 cl::init(Val: true));
49
50STATISTIC(NumVZU, "Number of vzeroupper instructions inserted");
51
52namespace {
53class X86InsertVZeroUpperLegacy : public MachineFunctionPass {
54public:
55 static char ID;
56
57 X86InsertVZeroUpperLegacy() : MachineFunctionPass(ID) {}
58
59 StringRef getPassName() const override { return "X86 vzeroupper inserter"; }
60
61 bool runOnMachineFunction(MachineFunction &MF) override;
62
63 MachineFunctionProperties getRequiredProperties() const override {
64 return MachineFunctionProperties().setNoVRegs();
65 }
66};
67
68enum BlockExitState { PASS_THROUGH, EXITS_CLEAN, EXITS_DIRTY };
69
70// Core algorithm state:
71// BlockState - Each block is either:
72// - PASS_THROUGH: There are neither YMM/ZMM dirtying instructions nor
73// vzeroupper instructions in this block.
74// - EXITS_CLEAN: There is (or will be) a vzeroupper instruction in this
75// block that will ensure that YMM/ZMM is clean on exit.
76// - EXITS_DIRTY: An instruction in the block dirties YMM/ZMM and no
77// subsequent vzeroupper in the block clears it.
78//
79// AddedToDirtySuccessors - This flag is raised when a block is added to the
80// DirtySuccessors list to ensure that it's not
81// added multiple times.
82//
83// FirstUnguardedCall - Records the location of the first unguarded call in
84// each basic block that may need to be guarded by a
85// vzeroupper. We won't know whether it actually needs
86// to be guarded until we discover a predecessor that
87// is DIRTY_OUT.
88struct BlockState {
89 BlockExitState ExitState = PASS_THROUGH;
90 bool AddedToDirtySuccessors = false;
91 MachineBasicBlock::iterator FirstUnguardedCall;
92
93 BlockState() = default;
94};
95
96using BlockStateMap = SmallVector<BlockState, 8>;
97using DirtySuccessorsWorkList = SmallVector<MachineBasicBlock *, 8>;
98} // end anonymous namespace
99
100char X86InsertVZeroUpperLegacy::ID = 0;
101
102FunctionPass *llvm::createX86InsertVZeroUpperLegacyPass() {
103 return new X86InsertVZeroUpperLegacy();
104}
105
106#ifndef NDEBUG
107static const char *getBlockExitStateName(BlockExitState ST) {
108 switch (ST) {
109 case PASS_THROUGH:
110 return "Pass-through";
111 case EXITS_DIRTY:
112 return "Exits-dirty";
113 case EXITS_CLEAN:
114 return "Exits-clean";
115 }
116 llvm_unreachable("Invalid block exit state.");
117}
118#endif
119
120/// VZEROUPPER cleans state that is related to Y/ZMM0-15 only.
121/// Thus, there is no need to check for Y/ZMM16 and above.
122static bool isYmmOrZmmReg(MCRegister Reg) {
123 return (Reg >= X86::YMM0 && Reg <= X86::YMM15) ||
124 (Reg >= X86::ZMM0 && Reg <= X86::ZMM15);
125}
126
127static bool checkFnHasLiveInYmmOrZmm(MachineRegisterInfo &MRI) {
128 for (std::pair<MCRegister, Register> LI : MRI.liveins())
129 if (isYmmOrZmmReg(Reg: LI.first))
130 return true;
131
132 return false;
133}
134
135static bool clobbersAllYmmAndZmmRegs(const MachineOperand &MO) {
136 for (unsigned reg = X86::YMM0; reg <= X86::YMM15; ++reg) {
137 if (!MO.clobbersPhysReg(PhysReg: reg))
138 return false;
139 }
140 for (unsigned reg = X86::ZMM0; reg <= X86::ZMM15; ++reg) {
141 if (!MO.clobbersPhysReg(PhysReg: reg))
142 return false;
143 }
144 return true;
145}
146
147static bool hasYmmOrZmmReg(MachineInstr &MI) {
148 for (const MachineOperand &MO : MI.operands()) {
149 if (MI.isCall() && MO.isRegMask() && !clobbersAllYmmAndZmmRegs(MO))
150 return true;
151 if (!MO.isReg())
152 continue;
153 if (MO.isDebug())
154 continue;
155 if (isYmmOrZmmReg(Reg: MO.getReg().asMCReg()))
156 return true;
157 }
158 return false;
159}
160
161/// Check if given call instruction has a RegMask operand.
162static bool callHasRegMask(MachineInstr &MI) {
163 assert(MI.isCall() && "Can only be called on call instructions.");
164 for (const MachineOperand &MO : MI.operands()) {
165 if (MO.isRegMask())
166 return true;
167 }
168 return false;
169}
170
171/// Insert a vzeroupper instruction before I.
172static bool insertVZeroUpper(MachineBasicBlock::iterator I,
173 MachineBasicBlock &MBB,
174 const TargetInstrInfo *TII) {
175 BuildMI(BB&: MBB, I, MIMD: I->getDebugLoc(), MCID: TII->get(Opcode: X86::VZEROUPPER));
176 ++NumVZU;
177 return true;
178}
179
180/// Add MBB to the DirtySuccessors list if it hasn't already been added.
181static void addDirtySuccessor(MachineBasicBlock &MBB,
182 BlockStateMap &BlockStates,
183 DirtySuccessorsWorkList &DirtySuccessors) {
184 if (!BlockStates[MBB.getNumber()].AddedToDirtySuccessors) {
185 DirtySuccessors.push_back(Elt: &MBB);
186 BlockStates[MBB.getNumber()].AddedToDirtySuccessors = true;
187 }
188}
189
190/// Loop over all of the instructions in the basic block, inserting vzeroupper
191/// instructions before function calls.
192static bool processBasicBlock(MachineBasicBlock &MBB,
193 BlockStateMap &BlockStates,
194 DirtySuccessorsWorkList &DirtySuccessors,
195 bool IsX86INTR, const TargetInstrInfo *TII) {
196 // Start by assuming that the block is PASS_THROUGH which implies no unguarded
197 // calls.
198 BlockExitState CurState = PASS_THROUGH;
199 BlockStates[MBB.getNumber()].FirstUnguardedCall = MBB.end();
200 bool MadeChange = false;
201
202 for (MachineInstr &MI : MBB) {
203 bool IsCall = MI.isCall();
204 bool IsReturn = MI.isReturn();
205 bool IsControlFlow = IsCall || IsReturn;
206
207 // No need for vzeroupper before iret in interrupt handler function,
208 // epilogue will restore YMM/ZMM registers if needed.
209 if (IsX86INTR && IsReturn)
210 continue;
211
212 // An existing VZERO* instruction resets the state.
213 if (MI.getOpcode() == X86::VZEROALL || MI.getOpcode() == X86::VZEROUPPER) {
214 CurState = EXITS_CLEAN;
215 continue;
216 }
217
218 // Shortcut: don't need to check regular instructions in dirty state.
219 if (!IsControlFlow && CurState == EXITS_DIRTY)
220 continue;
221
222 if (hasYmmOrZmmReg(MI)) {
223 // We found a ymm/zmm-using instruction; this could be an AVX/AVX512
224 // instruction, or it could be control flow.
225 CurState = EXITS_DIRTY;
226 continue;
227 }
228
229 // Check for control-flow out of the current function (which might
230 // indirectly execute SSE instructions).
231 if (!IsControlFlow)
232 continue;
233
234 // If the call has no RegMask, skip it as well. It usually happens on
235 // helper function calls (such as '_chkstk', '_ftol2') where standard
236 // calling convention is not used (RegMask is not used to mark register
237 // clobbered and register usage (def/implicit-def/use) is well-defined and
238 // explicitly specified.
239 if (IsCall && !callHasRegMask(MI))
240 continue;
241
242 // The VZEROUPPER instruction resets the upper 128 bits of YMM0-YMM15
243 // registers. In addition, the processor changes back to Clean state, after
244 // which execution of SSE instructions or AVX instructions has no transition
245 // penalty. Add the VZEROUPPER instruction before any function call/return
246 // that might execute SSE code.
247 // FIXME: In some cases, we may want to move the VZEROUPPER into a
248 // predecessor block.
249 if (CurState == EXITS_DIRTY) {
250 // After the inserted VZEROUPPER the state becomes clean again, but
251 // other YMM/ZMM may appear before other subsequent calls or even before
252 // the end of the BB.
253 MadeChange |= insertVZeroUpper(I: MI, MBB, TII);
254 CurState = EXITS_CLEAN;
255 } else if (CurState == PASS_THROUGH) {
256 // If this block is currently in pass-through state and we encounter a
257 // call then whether we need a vzeroupper or not depends on whether this
258 // block has successors that exit dirty. Record the location of the call,
259 // and set the state to EXITS_CLEAN, but do not insert the vzeroupper yet.
260 // It will be inserted later if necessary.
261 BlockStates[MBB.getNumber()].FirstUnguardedCall = MI;
262 CurState = EXITS_CLEAN;
263 }
264 }
265
266 LLVM_DEBUG(dbgs() << "MBB #" << MBB.getNumber() << " exit state: "
267 << getBlockExitStateName(CurState) << '\n');
268
269 if (CurState == EXITS_DIRTY)
270 for (MachineBasicBlock *Succ : MBB.successors())
271 addDirtySuccessor(MBB&: *Succ, BlockStates, DirtySuccessors);
272
273 BlockStates[MBB.getNumber()].ExitState = CurState;
274 return MadeChange;
275}
276
277/// Loop over all of the basic blocks, inserting vzeroupper instructions before
278/// function calls.
279static bool insertVZeroUpper(MachineFunction &MF) {
280 if (!UseVZeroUpper)
281 return false;
282
283 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
284 if (!ST.hasAVX() || !ST.insertVZEROUPPER())
285 return false;
286
287 MachineRegisterInfo &MRI = MF.getRegInfo();
288
289 bool FnHasLiveInYmmOrZmm = checkFnHasLiveInYmmOrZmm(MRI);
290
291 // Fast check: if the function doesn't use any ymm/zmm registers, we don't
292 // need to insert any VZEROUPPER instructions. This is constant-time, so it
293 // is cheap in the common case of no ymm/zmm use.
294 bool YmmOrZmmUsed = FnHasLiveInYmmOrZmm;
295 for (const auto *RC : {&X86::VR256RegClass, &X86::VR512_0_15RegClass}) {
296 if (!YmmOrZmmUsed) {
297 for (MCPhysReg R : *RC) {
298 if (!MRI.reg_nodbg_empty(RegNo: R)) {
299 YmmOrZmmUsed = true;
300 break;
301 }
302 }
303 }
304 }
305 if (!YmmOrZmmUsed)
306 return false;
307
308 const TargetInstrInfo *TII = ST.getInstrInfo();
309 bool IsX86INTR = MF.getFunction().getCallingConv() == CallingConv::X86_INTR;
310 bool EverMadeChange = false;
311 BlockStateMap BlockStates(MF.getNumBlockIDs());
312 DirtySuccessorsWorkList DirtySuccessors;
313
314 assert(BlockStates.size() == MF.getNumBlockIDs() && DirtySuccessors.empty() &&
315 "X86VZeroUpper state should be clear");
316
317 // Process all blocks. This will compute block exit states, record the first
318 // unguarded call in each block, and add successors of dirty blocks to the
319 // DirtySuccessors list.
320 for (MachineBasicBlock &MBB : MF)
321 EverMadeChange |=
322 processBasicBlock(MBB, BlockStates, DirtySuccessors, IsX86INTR, TII);
323
324 // If any YMM/ZMM regs are live-in to this function, add the entry block to
325 // the DirtySuccessors list
326 if (FnHasLiveInYmmOrZmm)
327 addDirtySuccessor(MBB&: MF.front(), BlockStates, DirtySuccessors);
328
329 // Re-visit all blocks that are successors of EXITS_DIRTY blocks. Add
330 // vzeroupper instructions to unguarded calls, and propagate EXITS_DIRTY
331 // through PASS_THROUGH blocks.
332 while (!DirtySuccessors.empty()) {
333 MachineBasicBlock &MBB = *DirtySuccessors.back();
334 DirtySuccessors.pop_back();
335 BlockState &BBState = BlockStates[MBB.getNumber()];
336
337 // MBB is a successor of a dirty block, so its first call needs to be
338 // guarded.
339 if (BBState.FirstUnguardedCall != MBB.end())
340 EverMadeChange |= insertVZeroUpper(I: BBState.FirstUnguardedCall, MBB, TII);
341
342 // If this successor was a pass-through block, then it is now dirty. Its
343 // successors need to be added to the worklist (if they haven't been
344 // already).
345 if (BBState.ExitState == PASS_THROUGH) {
346 LLVM_DEBUG(dbgs() << "MBB #" << MBB.getNumber()
347 << " was Pass-through, is now Dirty-out.\n");
348 for (MachineBasicBlock *Succ : MBB.successors())
349 addDirtySuccessor(MBB&: *Succ, BlockStates, DirtySuccessors);
350 }
351 }
352
353 return EverMadeChange;
354}
355
356bool X86InsertVZeroUpperLegacy::runOnMachineFunction(MachineFunction &MF) {
357 return insertVZeroUpper(MF);
358}
359
360PreservedAnalyses
361X86InsertVZeroUpperPass::run(MachineFunction &MF,
362 MachineFunctionAnalysisManager &MFAM) {
363 return insertVZeroUpper(MF) ? getMachineFunctionPassPreservedAnalyses()
364 .preserveSet<CFGAnalyses>()
365 : PreservedAnalyses::all();
366}
367