1//===-- X86CodeGenPassBuilder.cpp ---------------------------------*- C++ -*-=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file contains X86 CodeGen pipeline builder.
10/// TODO: Port CodeGen passes to new pass manager.
11//===----------------------------------------------------------------------===//
12
13#include "X86.h"
14#include "X86AsmPrinter.h"
15#include "X86TargetMachine.h"
16
17#include "llvm/CodeGen/AtomicExpand.h"
18#include "llvm/CodeGen/BreakFalseDeps.h"
19#include "llvm/CodeGen/EarlyIfConversion.h"
20#include "llvm/CodeGen/IndirectBrExpand.h"
21#include "llvm/CodeGen/InterleavedAccess.h"
22#include "llvm/CodeGen/JMCInstrumenter.h"
23#include "llvm/CodeGen/KCFI.h"
24#include "llvm/MC/MCStreamer.h"
25#include "llvm/Passes/CodeGenPassBuilder.h"
26#include "llvm/Passes/PassBuilder.h"
27#include "llvm/Support/CodeGen.h"
28#include "llvm/Transforms/CFGuard.h"
29
30using namespace llvm;
31
32extern cl::opt<bool> X86EnableMachineCombinerPass;
33
34namespace {
35
36class X86CodeGenPassBuilder
37 : public CodeGenPassBuilder<X86CodeGenPassBuilder, X86TargetMachine> {
38 using Base = CodeGenPassBuilder<X86CodeGenPassBuilder, X86TargetMachine>;
39
40public:
41 explicit X86CodeGenPassBuilder(X86TargetMachine &TM,
42 const CGPassBuilderOption &Opts,
43 PassInstrumentationCallbacks *PIC)
44 : CodeGenPassBuilder(TM, Opts, PIC) {}
45
46 void addIRPasses(PassManagerWrapper &PMW) const;
47 void addPreISel(PassManagerWrapper &PMW) const;
48 Error addInstSelector(PassManagerWrapper &PMW) const;
49 void addPreLegalizeMachineIR(PassManagerWrapper &PMW) const;
50 void addILPOpts(PassManagerWrapper &PMW) const;
51 void addPreRegBankSelect(PassManagerWrapper &PMW) const;
52 void addMachineSSAOptimization(PassManagerWrapper &PMW) const;
53 void addPreRegAlloc(PassManagerWrapper &PMW) const;
54 // TODO(boomanaiden154): We need to add addPostFastRegAllocRewrite here once
55 // it is available to support AMX.
56 void addPostRegAlloc(PassManagerWrapper &PMW) const;
57 void addPreSched2(PassManagerWrapper &PMW) const;
58 void addPreEmitPass(PassManagerWrapper &PMW) const;
59 void addPreEmitPass2(PassManagerWrapper &PMW) const;
60 // TODO(boomanaiden154): We need to add addRegAssignAndRewriteOptimized here
61 // once it is available to support AMX.
62 void addAsmPrinterBegin(PassManagerWrapper &PMW) const;
63 void addAsmPrinter(PassManagerWrapper &PMW) const;
64 void addAsmPrinterEnd(PassManagerWrapper &PMW) const;
65};
66
67void X86CodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) const {
68 addFunctionPass(Pass: AtomicExpandPass(TM), PMW);
69
70 // We add both pass anyway and when these two passes run, one will be a
71 // no-op based on the optimization level/attributes.
72 addFunctionPass(Pass: X86LowerAMXIntrinsicsPass(&TM), PMW);
73 addFunctionPass(Pass: X86LowerAMXTypePass(&TM), PMW);
74
75 Base::addIRPasses(PMW);
76
77 if (getOptLevel() != CodeGenOptLevel::None) {
78 addFunctionPass(Pass: InterleavedAccessPass(TM), PMW);
79 addFunctionPass(Pass: X86PartialReductionPass(&TM), PMW);
80 }
81
82 // Add passes that handle indirect branch removal and insertion of a retpoline
83 // thunk. These will be a no-op unless a function subtarget has the retpoline
84 // feature enabled.
85 addFunctionPass(Pass: IndirectBrExpandPass(TM), PMW);
86
87 // Add Control Flow Guard checks.
88 const Triple &TT = TM.getTargetTriple();
89 if (TT.isOSWindows())
90 addFunctionPass(Pass: CFGuardPass(), PMW);
91
92 if (TM.Options.JMCInstrument) {
93 flushFPMsToMPM(PMW);
94 addModulePass(Pass: JMCInstrumenterPass(), PMW);
95 }
96}
97
98void X86CodeGenPassBuilder::addPreISel(PassManagerWrapper &PMW) const {
99 // Only add this pass for 32-bit x86 Windows.
100 const Triple &TT = TM.getTargetTriple();
101 if (TT.isOSWindows() && TT.isX86_32()) {
102 flushFPMsToMPM(PMW);
103 addModulePass(Pass: X86WinEHStatePass(), PMW);
104 }
105}
106
107Error X86CodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) const {
108 addMachineFunctionPass(Pass: X86ISelDAGToDAGPass(TM), PMW);
109
110 // For ELF, cleanup any local-dynamic TLS accesses
111 if (TM.getTargetTriple().isOSBinFormatELF() &&
112 getOptLevel() != CodeGenOptLevel::None) {
113 addMachineFunctionPass(Pass: X86CleanupLocalDynamicTLSPass(), PMW);
114 }
115
116 addMachineFunctionPass(Pass: X86GlobalBaseRegPass(), PMW);
117 addMachineFunctionPass(Pass: X86ArgumentStackSlotPass(), PMW);
118 return Error::success();
119}
120
121void X86CodeGenPassBuilder::addPreLegalizeMachineIR(
122 PassManagerWrapper &PMW) const {
123 addMachineFunctionPass(Pass: X86PreLegalizerCombinerPass(), PMW);
124}
125
126void X86CodeGenPassBuilder::addILPOpts(PassManagerWrapper &PMW) const {
127 addMachineFunctionPass(Pass: EarlyIfConverterPass(), PMW);
128 if (X86EnableMachineCombinerPass) {
129 // TODO(boomanaiden154): Add the MachineCombinerPass here once it has been
130 // ported to the new pass manager.
131 }
132 addMachineFunctionPass(Pass: X86CmovConversionPass(), PMW);
133}
134
135void X86CodeGenPassBuilder::addPreRegBankSelect(PassManagerWrapper &PMW) const {
136 addMachineFunctionPass(Pass: X86PostLegalizerCombinerPass(), PMW);
137}
138
139void X86CodeGenPassBuilder::addMachineSSAOptimization(
140 PassManagerWrapper &PMW) const {
141 addMachineFunctionPass(Pass: X86DomainReassignmentPass(), PMW);
142 Base::addMachineSSAOptimization(PMW);
143}
144
145void X86CodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) const {
146 if (getOptLevel() != CodeGenOptLevel::None) {
147 addMachineFunctionPass(Pass: LiveRangeShrinkPass(), PMW);
148 addMachineFunctionPass(Pass: X86FixupSetCCPass(), PMW);
149 addMachineFunctionPass(Pass: X86CallFrameOptimizationPass(), PMW);
150 addMachineFunctionPass(Pass: X86AvoidStoreForwardingBlocksPass(), PMW);
151 }
152
153 addMachineFunctionPass(Pass: X86SuppressAPXForRelocationPass(), PMW);
154 addMachineFunctionPass(Pass: X86SpeculativeLoadHardeningPass(), PMW);
155 addMachineFunctionPass(Pass: X86FlagsCopyLoweringPass(), PMW);
156 addMachineFunctionPass(Pass: X86DynAllocaExpanderPass(), PMW);
157
158 if (getOptLevel() != CodeGenOptLevel::None)
159 addMachineFunctionPass(Pass: X86PreTileConfigPass(), PMW);
160 else
161 addMachineFunctionPass(Pass: X86FastPreTileConfigPass(), PMW);
162}
163
164void X86CodeGenPassBuilder::addPostRegAlloc(PassManagerWrapper &PMW) const {
165 addMachineFunctionPass(Pass: X86LowerTileCopyPass(), PMW);
166 addMachineFunctionPass(Pass: X86FPStackifierPass(), PMW);
167 // When -O0 is enabled, the Load Value Injection Hardening pass will fall back
168 // to using the Speculative Execution Side Effect Suppression pass for
169 // mitigation. This is to prevent slow downs due to
170 // analyses needed by the LVIHardening pass when compiling at -O0.
171 if (getOptLevel() != CodeGenOptLevel::None) {
172 addMachineFunctionPass(Pass: X86LoadValueInjectionLoadHardeningPass(), PMW);
173 }
174}
175
176void X86CodeGenPassBuilder::addPreSched2(PassManagerWrapper &PMW) const {
177 addMachineFunctionPass(Pass: X86ExpandPseudoPass(), PMW);
178 addMachineFunctionPass(Pass: MachineKCFIPass(), PMW);
179}
180
181void X86CodeGenPassBuilder::addPreEmitPass(PassManagerWrapper &PMW) const {
182 if (getOptLevel() != CodeGenOptLevel::None) {
183 // TODO(boomanaiden154): Add X86ExecutionDomainFixPass here once it has
184 // been ported.
185 addMachineFunctionPass(Pass: BreakFalseDepsPass(), PMW);
186 }
187
188 addMachineFunctionPass(Pass: X86IndirectBranchTrackingPass(), PMW);
189 addMachineFunctionPass(Pass: X86InsertVZeroUpperPass(), PMW);
190
191 if (getOptLevel() != CodeGenOptLevel::None) {
192 addMachineFunctionPass(Pass: X86FixupBWInstsPass(), PMW);
193 // TODO(boomanaiden154): Add X86PadShortFunctionsPass here once it has been
194 // ported.
195 addMachineFunctionPass(Pass: X86FixupLEAsPass(), PMW);
196 addMachineFunctionPass(Pass: X86FixupInstTuningPass(), PMW);
197 addMachineFunctionPass(Pass: X86FixupVectorConstantsPass(), PMW);
198 }
199 addMachineFunctionPass(Pass: X86CompressEVEXPass(), PMW);
200 addMachineFunctionPass(Pass: X86InsertX87WaitPass(), PMW);
201}
202
203void X86CodeGenPassBuilder::addPreEmitPass2(PassManagerWrapper &PMW) const {
204 const Triple &TT = TM.getTargetTriple();
205 const MCAsmInfo &MAI = TM.getMCAsmInfo();
206
207 // The X86 Speculative Execution Pass must run after all control
208 // flow graph modifying passes. As a result it was listed to run right before
209 // the X86 Retpoline Thunks pass. The reason it must run after control flow
210 // graph modifications is that the model of LFENCE in LLVM has to be updated
211 // (FIXME: https://bugs.llvm.org/show_bug.cgi?id=45167). Currently the
212 // placement of this pass was hand checked to ensure that the subsequent
213 // passes don't move the code around the LFENCEs in a way that will hurt the
214 // correctness of this pass. This placement has been shown to work based on
215 // hand inspection of the codegen output.
216 addMachineFunctionPass(Pass: X86SpeculativeExecutionSideEffectSuppressionPass(),
217 PMW);
218 // TODO(boomanaiden154): Add X86IndirectThunksPass here
219 // once it has been ported.
220 addMachineFunctionPass(Pass: X86ReturnThunksPass(), PMW);
221
222 // Insert extra int3 instructions after trailing call instructions to avoid
223 // issues in the unwinder.
224 if (TT.isOSWindows() && TT.isX86_64())
225 addMachineFunctionPass(Pass: X86AvoidTrailingCallPass(), PMW);
226
227 // Verify basic block incoming and outgoing cfa offset and register values and
228 // correct CFA calculation rule where needed by inserting appropriate CFI
229 // instructions.
230 if (!TT.isOSDarwin() &&
231 (!TT.isOSWindows() ||
232 MAI.getExceptionHandlingType() == ExceptionHandling::DwarfCFI)) {
233 // TODO(boomanaiden154): Add CFInstrInserterPass here when it has been
234 // ported.
235 }
236
237 if (TT.isOSWindows()) {
238 // Identify valid longjmp targets for Windows Control Flow Guard.
239 // TODO(boomanaiden154): Add CFGuardLongjmpPass here when it has been
240 // ported.
241 // Identify valid eh continuation targets for Windows EHCont Guard.
242 // TODO(boomanaiden154): Add EHContGuardTargetsPass when it has been
243 // ported.
244 }
245
246 addMachineFunctionPass(Pass: X86LoadValueInjectionRetHardeningPass(), PMW);
247
248 // Insert pseudo probe annotation for callsite profiling
249 // TODO(boomanaiden154): Add PseudoProberInserterPass here once it has been
250 // ported.
251
252 // KCFI indirect call checks are lowered to a bundle, and on Darwin platforms,
253 // also CALL_RVMARKER.
254 // TODO(boomanaiden154): Add UnpackMachineBundlesPass here once it has been
255 // ported.
256
257 // Analyzes and emits pseudos to support Win x64 Unwind V2. This pass must run
258 // after all real instructions have been added to the epilog.
259 if (TT.isOSWindows() && TT.isX86_64()) {
260 addMachineFunctionPass(Pass: X86WinEHUnwindV2Pass(), PMW);
261 }
262}
263
264void X86CodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) const {
265 addModulePass(Pass: X86AsmPrinterBeginPass(), PMW, /*Force=*/true);
266}
267
268void X86CodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) const {
269 addMachineFunctionPass(Pass: X86AsmPrinterPass(), PMW);
270}
271
272void X86CodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) const {
273 addModulePass(Pass: X86AsmPrinterEndPass(), PMW, /*Force=*/true);
274}
275
276} // namespace
277
278void X86TargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
279#define GET_PASS_REGISTRY "X86PassRegistry.def"
280#include "llvm/Passes/TargetPassRegistry.inc"
281 // TODO(boomanaiden154): Move this into the base CodeGenPassBuilder once all
282 // targets that currently implement it have a ported asm-printer pass.
283 if (PIC) {
284 PIC->addClassToPassName(ClassName: X86AsmPrinterBeginPass::name(),
285 PassName: "x86-asm-printer-begin");
286 PIC->addClassToPassName(ClassName: X86AsmPrinterPass::name(), PassName: "x86-asm-printer");
287 PIC->addClassToPassName(ClassName: X86AsmPrinterEndPass::name(),
288 PassName: "x86-asm-printer-end");
289 }
290}
291
292Error X86TargetMachine::buildCodeGenPipeline(
293 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
294 raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
295 const CGPassBuilderOption &Opt, MCContext &Ctx,
296 PassInstrumentationCallbacks *PIC) {
297 auto CGPB = X86CodeGenPassBuilder(*this, Opt, PIC);
298 return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
299}
300