1//===-- X86CodeGenPassBuilder.cpp ---------------------------------*- C++ -*-=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file contains X86 CodeGen pipeline builder.
10/// TODO: Port CodeGen passes to new pass manager.
11//===----------------------------------------------------------------------===//
12
13#include "X86.h"
14#include "X86AsmPrinter.h"
15#include "X86ISelDAGToDAG.h"
16#include "X86TargetMachine.h"
17
18#include "llvm/CodeGen/AtomicExpand.h"
19#include "llvm/CodeGen/EarlyIfConversion.h"
20#include "llvm/CodeGen/IndirectBrExpand.h"
21#include "llvm/CodeGen/InterleavedAccess.h"
22#include "llvm/CodeGen/JMCInstrumenter.h"
23#include "llvm/MC/MCStreamer.h"
24#include "llvm/Passes/CodeGenPassBuilder.h"
25#include "llvm/Passes/PassBuilder.h"
26#include "llvm/Support/CodeGen.h"
27#include "llvm/Transforms/CFGuard.h"
28
29using namespace llvm;
30
31extern cl::opt<bool> X86EnableMachineCombinerPass;
32
33namespace {
34
35class X86CodeGenPassBuilder
36 : public CodeGenPassBuilder<X86CodeGenPassBuilder, X86TargetMachine> {
37 using Base = CodeGenPassBuilder<X86CodeGenPassBuilder, X86TargetMachine>;
38
39public:
40 explicit X86CodeGenPassBuilder(X86TargetMachine &TM,
41 const CGPassBuilderOption &Opts,
42 PassInstrumentationCallbacks *PIC)
43 : CodeGenPassBuilder(TM, Opts, PIC) {}
44
45 void addIRPasses(PassManagerWrapper &PMW) const;
46 void addPreISel(PassManagerWrapper &PMW) const;
47 Error addInstSelector(PassManagerWrapper &PMW) const;
48 void addILPOpts(PassManagerWrapper &PMW) const;
49 void addPreRegBankSelect(PassManagerWrapper &PMW) const;
50 void addMachineSSAOptimization(PassManagerWrapper &PMW) const;
51 void addPreRegAlloc(PassManagerWrapper &PMW) const;
52 // TODO(boomanaiden154): We need to add addPostFastRegAllocRewrite here once
53 // it is available to support AMX.
54 void addPostRegAlloc(PassManagerWrapper &PMW) const;
55 void addPreSched2(PassManagerWrapper &PMW) const;
56 void addPreEmitPass(PassManagerWrapper &PMW) const;
57 void addPreEmitPass2(PassManagerWrapper &PMW) const;
58 // TODO(boomanaiden154): We need to add addRegAssignAndRewriteOptimized here
59 // once it is available to support AMX.
60 void addAsmPrinterBegin(PassManagerWrapper &PMW,
61 CreateMCStreamer CreateStreamer) const;
62 void addAsmPrinter(PassManagerWrapper &PMW,
63 CreateMCStreamer CreateStreamer) const;
64 void addAsmPrinterEnd(PassManagerWrapper &PMW,
65 CreateMCStreamer CreateStreamer) const;
66};
67
68void X86CodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) const {
69 addFunctionPass(Pass: AtomicExpandPass(TM), PMW);
70
71 // We add both pass anyway and when these two passes run, one will be a
72 // no-op based on the optimization level/attributes.
73 addFunctionPass(Pass: X86LowerAMXIntrinsicsPass(&TM), PMW);
74 addFunctionPass(Pass: X86LowerAMXTypePass(&TM), PMW);
75
76 Base::addIRPasses(PMW);
77
78 if (getOptLevel() != CodeGenOptLevel::None) {
79 addFunctionPass(Pass: InterleavedAccessPass(TM), PMW);
80 addFunctionPass(Pass: X86PartialReductionPass(&TM), PMW);
81 }
82
83 // Add passes that handle indirect branch removal and insertion of a retpoline
84 // thunk. These will be a no-op unless a function subtarget has the retpoline
85 // feature enabled.
86 addFunctionPass(Pass: IndirectBrExpandPass(TM), PMW);
87
88 // Add Control Flow Guard checks.
89 const Triple &TT = TM.getTargetTriple();
90 if (TT.isOSWindows())
91 addFunctionPass(Pass: CFGuardPass(TT.isX86_64() ? CFGuardPass::Mechanism::Dispatch
92 : CFGuardPass::Mechanism::Check),
93 PMW);
94
95 if (TM.Options.JMCInstrument) {
96 flushFPMsToMPM(PMW);
97 addModulePass(Pass: JMCInstrumenterPass(), PMW);
98 }
99}
100
101void X86CodeGenPassBuilder::addPreISel(PassManagerWrapper &PMW) const {
102 // Only add this pass for 32-bit x86 Windows.
103 const Triple &TT = TM.getTargetTriple();
104 if (TT.isOSWindows() && TT.isX86_32()) {
105 flushFPMsToMPM(PMW);
106 addModulePass(Pass: X86WinEHStatePass(), PMW);
107 }
108}
109
110Error X86CodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) const {
111 addMachineFunctionPass(Pass: X86ISelDAGToDAGPass(TM), PMW);
112
113 // For ELF, cleanup any local-dynamic TLS accesses
114 if (TM.getTargetTriple().isOSBinFormatELF() &&
115 getOptLevel() != CodeGenOptLevel::None) {
116 addMachineFunctionPass(Pass: X86CleanupLocalDynamicTLSPass(), PMW);
117 }
118
119 addMachineFunctionPass(Pass: X86GlobalBaseRegPass(), PMW);
120 addMachineFunctionPass(Pass: X86ArgumentStackSlotPass(), PMW);
121 return Error::success();
122}
123
124void X86CodeGenPassBuilder::addILPOpts(PassManagerWrapper &PMW) const {
125 addMachineFunctionPass(Pass: EarlyIfConverterPass(), PMW);
126 if (X86EnableMachineCombinerPass) {
127 // TODO(boomanaiden154): Add the MachineCombinerPass here once it has been
128 // ported to the new pass manager.
129 }
130 addMachineFunctionPass(Pass: X86CmovConversionPass(), PMW);
131}
132
133void X86CodeGenPassBuilder::addPreRegBankSelect(PassManagerWrapper &PMW) const {
134 addMachineFunctionPass(Pass: X86PostLegalizerCombinerPass(), PMW);
135}
136
137void X86CodeGenPassBuilder::addMachineSSAOptimization(
138 PassManagerWrapper &PMW) const {
139 addMachineFunctionPass(Pass: X86DomainReassignmentPass(), PMW);
140 Base::addMachineSSAOptimization(PMW);
141}
142
143void X86CodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) const {
144 if (getOptLevel() != CodeGenOptLevel::None) {
145 addMachineFunctionPass(Pass: LiveRangeShrinkPass(), PMW);
146 addMachineFunctionPass(Pass: X86FixupSetCCPass(), PMW);
147 addMachineFunctionPass(Pass: X86CallFrameOptimizationPass(), PMW);
148 addMachineFunctionPass(Pass: X86AvoidStoreForwardingBlocksPass(), PMW);
149 }
150
151 addMachineFunctionPass(Pass: X86SuppressAPXForRelocationPass(), PMW);
152 addMachineFunctionPass(Pass: X86SpeculativeLoadHardeningPass(), PMW);
153 addMachineFunctionPass(Pass: X86FlagsCopyLoweringPass(), PMW);
154 addMachineFunctionPass(Pass: X86DynAllocaExpanderPass(), PMW);
155
156 if (getOptLevel() != CodeGenOptLevel::None)
157 addMachineFunctionPass(Pass: X86PreTileConfigPass(), PMW);
158 else
159 addMachineFunctionPass(Pass: X86FastPreTileConfigPass(), PMW);
160}
161
162void X86CodeGenPassBuilder::addPostRegAlloc(PassManagerWrapper &PMW) const {
163 addMachineFunctionPass(Pass: X86LowerTileCopyPass(), PMW);
164 addMachineFunctionPass(Pass: X86FPStackifierPass(), PMW);
165 // When -O0 is enabled, the Load Value Injection Hardening pass will fall back
166 // to using the Speculative Execution Side Effect Suppression pass for
167 // mitigation. This is to prevent slow downs due to
168 // analyses needed by the LVIHardening pass when compiling at -O0.
169 if (getOptLevel() != CodeGenOptLevel::None) {
170 addMachineFunctionPass(Pass: X86LoadValueInjectionLoadHardeningPass(), PMW);
171 }
172}
173
174void X86CodeGenPassBuilder::addPreSched2(PassManagerWrapper &PMW) const {
175 addMachineFunctionPass(Pass: X86ExpandPseudoPass(), PMW);
176 // TODO(boomanaiden154): Add KCFGPass here once it has been ported.
177}
178
179void X86CodeGenPassBuilder::addPreEmitPass(PassManagerWrapper &PMW) const {
180 if (getOptLevel() != CodeGenOptLevel::None) {
181 // TODO(boomanaiden154): Add X86ExecutionDomainFixPass here once it has
182 // been ported.
183 addMachineFunctionPass(Pass: BreakFalseDepsPass(), PMW);
184 }
185
186 addMachineFunctionPass(Pass: X86IndirectBranchTrackingPass(), PMW);
187 // TODO(boomanaiden154): Add X86IssueVZeroUpperPass here once it has been
188 // ported.
189
190 if (getOptLevel() != CodeGenOptLevel::None) {
191 addMachineFunctionPass(Pass: X86FixupBWInstsPass(), PMW);
192 // TODO(boomanaiden154): Add X86PadShortFunctionsPass here once it has been
193 // ported.
194 addMachineFunctionPass(Pass: X86FixupLEAsPass(), PMW);
195 addMachineFunctionPass(Pass: X86FixupInstTuningPass(), PMW);
196 addMachineFunctionPass(Pass: X86FixupVectorConstantsPass(), PMW);
197 }
198 addMachineFunctionPass(Pass: X86CompressEVEXPass(), PMW);
199 addMachineFunctionPass(Pass: X86InsertX87WaitPass(), PMW);
200}
201
202void X86CodeGenPassBuilder::addPreEmitPass2(PassManagerWrapper &PMW) const {
203 const Triple &TT = TM.getTargetTriple();
204 const MCAsmInfo *MAI = TM.getMCAsmInfo();
205
206 // The X86 Speculative Execution Pass must run after all control
207 // flow graph modifying passes. As a result it was listed to run right before
208 // the X86 Retpoline Thunks pass. The reason it must run after control flow
209 // graph modifications is that the model of LFENCE in LLVM has to be updated
210 // (FIXME: https://bugs.llvm.org/show_bug.cgi?id=45167). Currently the
211 // placement of this pass was hand checked to ensure that the subsequent
212 // passes don't move the code around the LFENCEs in a way that will hurt the
213 // correctness of this pass. This placement has been shown to work based on
214 // hand inspection of the codegen output.
215 addMachineFunctionPass(Pass: X86SpeculativeExecutionSideEffectSuppressionPass(),
216 PMW);
217 // TODO(boomanaiden154): Add X86IndirectThunksPass here
218 // once it has been ported.
219 addMachineFunctionPass(Pass: X86ReturnThunksPass(), PMW);
220
221 // Insert extra int3 instructions after trailing call instructions to avoid
222 // issues in the unwinder.
223 if (TT.isOSWindows() && TT.isX86_64())
224 addMachineFunctionPass(Pass: X86AvoidTrailingCallPass(), PMW);
225
226 // Verify basic block incoming and outgoing cfa offset and register values and
227 // correct CFA calculation rule where needed by inserting appropriate CFI
228 // instructions.
229 if (!TT.isOSDarwin() &&
230 (!TT.isOSWindows() ||
231 MAI->getExceptionHandlingType() == ExceptionHandling::DwarfCFI)) {
232 // TODO(boomanaiden154): Add CFInstrInserterPass here when it has been
233 // ported.
234 }
235
236 if (TT.isOSWindows()) {
237 // Identify valid longjmp targets for Windows Control Flow Guard.
238 // TODO(boomanaiden154): Add CFGuardLongjmpPass here when it has been
239 // ported.
240 // Identify valid eh continuation targets for Windows EHCont Guard.
241 // TODO(boomanaiden154): Add EHContGuardTargetsPass when it has been
242 // ported.
243 }
244
245 addMachineFunctionPass(Pass: X86LoadValueInjectionRetHardeningPass(), PMW);
246
247 // Insert pseudo probe annotation for callsite profiling
248 // TODO(boomanaiden154): Add PseudoProberInserterPass here once it has been
249 // ported.
250
251 // KCFI indirect call checks are lowered to a bundle, and on Darwin platforms,
252 // also CALL_RVMARKER.
253 // TODO(boomanaiden154): Add UnpackMachineBundlesPass here once it has been
254 // ported.
255
256 // Analyzes and emits pseudos to support Win x64 Unwind V2. This pass must run
257 // after all real instructions have been added to the epilog.
258 if (TT.isOSWindows() && TT.isX86_64()) {
259 addMachineFunctionPass(Pass: X86WinEHUnwindV2Pass(), PMW);
260 }
261}
262
263void X86CodeGenPassBuilder::addAsmPrinterBegin(
264 PassManagerWrapper &PMW, CreateMCStreamer CreateStreamer) const {
265 addModulePass(Pass: X86AsmPrinterBeginPass(TM, CreateStreamer), PMW,
266 /*Force=*/true);
267}
268
269void X86CodeGenPassBuilder::addAsmPrinter(
270 PassManagerWrapper &PMW, CreateMCStreamer CreateStreamer) const {
271 addMachineFunctionPass(Pass: X86AsmPrinterPass(TM, CreateStreamer), PMW);
272}
273
274void X86CodeGenPassBuilder::addAsmPrinterEnd(
275 PassManagerWrapper &PMW, CreateMCStreamer CreateStreamer) const {
276 addModulePass(Pass: X86AsmPrinterEndPass(TM, CreateStreamer), PMW, /*Force=*/true);
277}
278
279} // namespace
280
281void X86TargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
282#define GET_PASS_REGISTRY "X86PassRegistry.def"
283#include "llvm/Passes/TargetPassRegistry.inc"
284 // TODO(boomanaiden154): Move this into the base CodeGenPassBuilder once all
285 // targets that currently implement it have a ported asm-printer pass.
286 if (PIC) {
287 PIC->addClassToPassName(ClassName: X86AsmPrinterBeginPass::name(),
288 PassName: "x86-asm-printer-begin");
289 PIC->addClassToPassName(ClassName: X86AsmPrinterPass::name(), PassName: "x86-asm-printer");
290 PIC->addClassToPassName(ClassName: X86AsmPrinterEndPass::name(),
291 PassName: "x86-asm-printer-end");
292 }
293}
294
295Error X86TargetMachine::buildCodeGenPipeline(
296 ModulePassManager &MPM, raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut,
297 CodeGenFileType FileType, const CGPassBuilderOption &Opt, MCContext &Ctx,
298 PassInstrumentationCallbacks *PIC) {
299 auto CGPB = X86CodeGenPassBuilder(*this, Opt, PIC);
300 return CGPB.buildPipeline(MPM, Out, DwoOut, FileType, Ctx);
301}
302