1//===-- X86CodeGenPassBuilder.cpp ---------------------------------*- C++ -*-=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file contains X86 CodeGen pipeline builder.
10/// TODO: Port CodeGen passes to new pass manager.
11//===----------------------------------------------------------------------===//
12
13#include "X86.h"
14#include "X86AsmPrinter.h"
15#include "X86TargetMachine.h"
16
17#include "llvm/CodeGen/AtomicExpand.h"
18#include "llvm/CodeGen/BreakFalseDeps.h"
19#include "llvm/CodeGen/CFIInstrInserter.h"
20#include "llvm/CodeGen/EHContGuardTargets.h"
21#include "llvm/CodeGen/EarlyIfConversion.h"
22#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
23#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
24#include "llvm/CodeGen/GlobalISel/Legalizer.h"
25#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
26#include "llvm/CodeGen/IndirectBrExpand.h"
27#include "llvm/CodeGen/InterleavedAccess.h"
28#include "llvm/CodeGen/JMCInstrumenter.h"
29#include "llvm/CodeGen/KCFI.h"
30#include "llvm/CodeGen/MachineCombiner.h"
31#include "llvm/MC/MCAsmInfo.h"
32#include "llvm/MC/MCStreamer.h"
33#include "llvm/Passes/CodeGenPassBuilder.h"
34#include "llvm/Passes/PassBuilder.h"
35#include "llvm/Support/CodeGen.h"
36#include "llvm/Transforms/CFGuard.h"
37
38using namespace llvm;
39
40extern cl::opt<bool> X86EnableMachineCombinerPass;
41
42namespace {
43
44class X86CodeGenPassBuilder : public CodeGenPassBuilder {
45 using Base = CodeGenPassBuilder;
46
47 X86TargetMachine &getTM() const {
48 return static_cast<X86TargetMachine &>(TM);
49 }
50
51public:
52 explicit X86CodeGenPassBuilder(X86TargetMachine &TM,
53 const CGPassBuilderOption &Opts,
54 PassInstrumentationCallbacks *PIC)
55 : CodeGenPassBuilder(TM, Opts, PIC) {}
56
57 void addIRPasses(PassManagerWrapper &PMW) override;
58 void addPreISel(PassManagerWrapper &PMW) override;
59 Error addInstSelector(PassManagerWrapper &PMW) override;
60
61 Error addIRTranslator(PassManagerWrapper &PMW) override;
62 void addPreLegalizeMachineIR(PassManagerWrapper &PMW) override;
63 Error addLegalizeMachineIR(PassManagerWrapper &PMW) override;
64 void addPreRegBankSelect(PassManagerWrapper &PMW) override;
65 Error addRegBankSelect(PassManagerWrapper &PMW) override;
66 Error addGlobalInstructionSelect(PassManagerWrapper &PMW) override;
67
68 void addILPOpts(PassManagerWrapper &PMW) override;
69 void addMachineSSAOptimization(PassManagerWrapper &PMW) override;
70 void addPreRegAlloc(PassManagerWrapper &PMW) override;
71 // TODO(boomanaiden154): We need to add addPostFastRegAllocRewrite here once
72 // it is available to support AMX.
73 void addPostRegAlloc(PassManagerWrapper &PMW) override;
74 void addPreSched2(PassManagerWrapper &PMW) override;
75 void addPreEmitPass(PassManagerWrapper &PMW) override;
76 void addPreEmitPass2(PassManagerWrapper &PMW) override;
77 // TODO(boomanaiden154): We need to add addRegAssignAndRewriteOptimized here
78 // once it is available to support AMX.
79 void addAsmPrinterBegin(PassManagerWrapper &PMW) override;
80 void addAsmPrinter(PassManagerWrapper &PMW) override;
81 void addAsmPrinterEnd(PassManagerWrapper &PMW) override;
82};
83
84void X86CodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
85 addFunctionPass(Pass: AtomicExpandPass(TM), PMW);
86
87 // We add both pass anyway and when these two passes run, one will be a
88 // no-op based on the optimization level/attributes.
89 addFunctionPass(Pass: X86LowerAMXIntrinsicsPass(&TM), PMW);
90 addFunctionPass(Pass: X86LowerAMXTypePass(&TM), PMW);
91
92 Base::addIRPasses(PMW);
93
94 if (getOptLevel() != CodeGenOptLevel::None) {
95 addFunctionPass(Pass: InterleavedAccessPass(TM), PMW);
96 addFunctionPass(Pass: X86PartialReductionPass(&getTM()), PMW);
97 }
98
99 // Add passes that handle indirect branch removal and insertion of a retpoline
100 // thunk. These will be a no-op unless a function subtarget has the retpoline
101 // feature enabled.
102 addFunctionPass(Pass: IndirectBrExpandPass(TM), PMW);
103
104 // Add Control Flow Guard checks.
105 const Triple &TT = TM.getTargetTriple();
106 if (TT.isOSWindows())
107 addFunctionPass(Pass: CFGuardPass(), PMW);
108
109 if (TM.Options.JMCInstrument) {
110 flushFPMsToMPM(PMW);
111 addModulePass(Pass: JMCInstrumenterPass(), PMW);
112 }
113}
114
115void X86CodeGenPassBuilder::addPreISel(PassManagerWrapper &PMW) {
116 // Only add this pass for 32-bit x86 Windows.
117 const Triple &TT = TM.getTargetTriple();
118 if (TT.isOSWindows() && TT.isX86_32()) {
119 flushFPMsToMPM(PMW);
120 addModulePass(Pass: X86WinEHStatePass(), PMW);
121 }
122}
123
124Error X86CodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) {
125 addMachineFunctionPass(Pass: X86ISelDAGToDAGPass(getTM()), PMW);
126
127 // For ELF, cleanup any local-dynamic TLS accesses
128 if (TM.getTargetTriple().isOSBinFormatELF() &&
129 getOptLevel() != CodeGenOptLevel::None) {
130 addMachineFunctionPass(Pass: X86CleanupLocalDynamicTLSPass(), PMW);
131 }
132
133 addMachineFunctionPass(Pass: X86GlobalBaseRegPass(), PMW);
134 addMachineFunctionPass(Pass: X86ArgumentStackSlotPass(), PMW);
135 return Error::success();
136}
137
138Error X86CodeGenPassBuilder::addIRTranslator(PassManagerWrapper &PMW) {
139 addMachineFunctionPass(Pass: IRTranslatorPass(getOptLevel()), PMW);
140 return Error::success();
141}
142
143void X86CodeGenPassBuilder::addPreLegalizeMachineIR(PassManagerWrapper &PMW) {
144 if (getOptLevel() != CodeGenOptLevel::None)
145 addMachineFunctionPass(Pass: X86PreLegalizerCombinerPass(), PMW);
146}
147
148Error X86CodeGenPassBuilder::addLegalizeMachineIR(PassManagerWrapper &PMW) {
149 addMachineFunctionPass(Pass: LegalizerPass(), PMW);
150 return Error::success();
151}
152
153void X86CodeGenPassBuilder::addPreRegBankSelect(PassManagerWrapper &PMW) {
154 if (getOptLevel() != CodeGenOptLevel::None)
155 addMachineFunctionPass(Pass: X86PostLegalizerCombinerPass(), PMW);
156}
157
158Error X86CodeGenPassBuilder::addRegBankSelect(PassManagerWrapper &PMW) {
159 addMachineFunctionPass(Pass: RegBankSelectPass(), PMW);
160 return Error::success();
161}
162
163Error X86CodeGenPassBuilder::addGlobalInstructionSelect(
164 PassManagerWrapper &PMW) {
165 addMachineFunctionPass(Pass: InstructionSelectPass(getOptLevel()), PMW);
166 // Add GlobalBaseReg in case there is no SelectionDAG passes afterwards
167 if (isGlobalISelAbortEnabled())
168 addMachineFunctionPass(Pass: X86GlobalBaseRegPass(), PMW);
169 return Error::success();
170}
171
172void X86CodeGenPassBuilder::addILPOpts(PassManagerWrapper &PMW) {
173 addMachineFunctionPass(Pass: EarlyIfConverterPass(), PMW);
174 if (X86EnableMachineCombinerPass)
175 addMachineFunctionPass(Pass: MachineCombinerPass(), PMW);
176 addMachineFunctionPass(Pass: X86CmovConversionPass(), PMW);
177}
178
179void X86CodeGenPassBuilder::addMachineSSAOptimization(PassManagerWrapper &PMW) {
180 addMachineFunctionPass(Pass: X86DomainReassignmentPass(), PMW);
181 Base::addMachineSSAOptimization(PMW);
182}
183
184void X86CodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) {
185 if (getOptLevel() != CodeGenOptLevel::None) {
186 addMachineFunctionPass(Pass: LiveRangeShrinkPass(), PMW);
187 addMachineFunctionPass(Pass: X86FixupSetCCPass(), PMW);
188 addMachineFunctionPass(Pass: X86CallFrameOptimizationPass(), PMW);
189 addMachineFunctionPass(Pass: X86AvoidStoreForwardingBlocksPass(), PMW);
190 }
191
192 addMachineFunctionPass(Pass: X86SuppressAPXForRelocationPass(), PMW);
193 addMachineFunctionPass(Pass: X86SpeculativeLoadHardeningPass(), PMW);
194 addMachineFunctionPass(Pass: X86FlagsCopyLoweringPass(), PMW);
195 addMachineFunctionPass(Pass: X86DynAllocaExpanderPass(), PMW);
196
197 if (getOptLevel() != CodeGenOptLevel::None)
198 addMachineFunctionPass(Pass: X86PreTileConfigPass(), PMW);
199 else
200 addMachineFunctionPass(Pass: X86FastPreTileConfigPass(), PMW);
201}
202
203void X86CodeGenPassBuilder::addPostRegAlloc(PassManagerWrapper &PMW) {
204 addMachineFunctionPass(Pass: X86LowerTileCopyPass(), PMW);
205 addMachineFunctionPass(Pass: X86FPStackifierPass(), PMW);
206 // When -O0 is enabled, the Load Value Injection Hardening pass will fall back
207 // to using the Speculative Execution Side Effect Suppression pass for
208 // mitigation. This is to prevent slow downs due to
209 // analyses needed by the LVIHardening pass when compiling at -O0.
210 if (getOptLevel() != CodeGenOptLevel::None) {
211 addMachineFunctionPass(Pass: X86LoadValueInjectionLoadHardeningPass(), PMW);
212 }
213}
214
215void X86CodeGenPassBuilder::addPreSched2(PassManagerWrapper &PMW) {
216 addMachineFunctionPass(Pass: X86ExpandPseudoPass(), PMW);
217 addMachineFunctionPass(Pass: MachineKCFIPass(), PMW);
218}
219
220void X86CodeGenPassBuilder::addPreEmitPass(PassManagerWrapper &PMW) {
221 if (getOptLevel() != CodeGenOptLevel::None) {
222 // TODO(boomanaiden154): Add X86ExecutionDomainFixPass here once it has
223 // been ported.
224 addMachineFunctionPass(Pass: BreakFalseDepsPass(), PMW);
225 }
226
227 addMachineFunctionPass(Pass: X86IndirectBranchTrackingPass(), PMW);
228 addMachineFunctionPass(Pass: X86InsertVZeroUpperPass(), PMW);
229
230 if (getOptLevel() != CodeGenOptLevel::None) {
231 addMachineFunctionPass(Pass: X86FixupBWInstsPass(), PMW);
232 // TODO(boomanaiden154): Add X86PadShortFunctionsPass here once it has been
233 // ported.
234 addMachineFunctionPass(Pass: X86FixupLEAsPass(), PMW);
235 addMachineFunctionPass(Pass: X86FixupInstTuningPass(), PMW);
236 addMachineFunctionPass(Pass: X86FixupVectorConstantsPass(), PMW);
237 }
238 addMachineFunctionPass(Pass: X86CompressEVEXPass(), PMW);
239 addMachineFunctionPass(Pass: X86InsertX87WaitPass(), PMW);
240}
241
242void X86CodeGenPassBuilder::addPreEmitPass2(PassManagerWrapper &PMW) {
243 const Triple &TT = TM.getTargetTriple();
244 const MCAsmInfo &MAI = TM.getMCAsmInfo();
245
246 // The X86 Speculative Execution Pass must run after all control
247 // flow graph modifying passes. As a result it was listed to run right before
248 // the X86 Retpoline Thunks pass. The reason it must run after control flow
249 // graph modifications is that the model of LFENCE in LLVM has to be updated
250 // (FIXME: https://bugs.llvm.org/show_bug.cgi?id=45167). Currently the
251 // placement of this pass was hand checked to ensure that the subsequent
252 // passes don't move the code around the LFENCEs in a way that will hurt the
253 // correctness of this pass. This placement has been shown to work based on
254 // hand inspection of the codegen output.
255 addMachineFunctionPass(Pass: X86SpeculativeExecutionSideEffectSuppressionPass(),
256 PMW);
257 // TODO(boomanaiden154): Add X86IndirectThunksPass here
258 // once it has been ported.
259 addMachineFunctionPass(Pass: X86ReturnThunksPass(), PMW);
260
261 // Insert extra int3 instructions after trailing call instructions to avoid
262 // issues in the unwinder.
263 if (TT.isOSWindows() && TT.isX86_64())
264 addMachineFunctionPass(Pass: X86AvoidTrailingCallPass(), PMW);
265
266 // Verify basic block incoming and outgoing cfa offset and register values and
267 // correct CFA calculation rule where needed by inserting appropriate CFI
268 // instructions.
269 if (!TT.isOSDarwin() &&
270 (!TT.isOSWindows() ||
271 MAI.getExceptionHandlingType() == ExceptionHandling::DwarfCFI))
272 addMachineFunctionPass(Pass: CFIInstrInserterPass(), PMW);
273
274 if (TT.isOSWindows()) {
275 // Identify valid longjmp targets for Windows Control Flow Guard.
276 // TODO(boomanaiden154): Add CFGuardLongjmpPass here when it has been
277 // ported.
278 // Identify valid eh continuation targets for Windows EHCont Guard.
279 addMachineFunctionPass(Pass: EHContGuardTargetsPass(), PMW);
280 }
281
282 addMachineFunctionPass(Pass: X86LoadValueInjectionRetHardeningPass(), PMW);
283
284 // Insert pseudo probe annotation for callsite profiling
285 // TODO(boomanaiden154): Add PseudoProberInserterPass here once it has been
286 // ported.
287
288 // KCFI indirect call checks are lowered to a bundle, and on Darwin platforms,
289 // also CALL_RVMARKER.
290 // TODO(boomanaiden154): Add UnpackMachineBundlesPass here once it has been
291 // ported.
292
293 // Analyzes and emits pseudos to support Win x64 Unwind V2. This pass must run
294 // after all real instructions have been added to the epilog.
295 if (TT.isOSWindows() && TT.isX86_64()) {
296 addMachineFunctionPass(Pass: X86WinEHUnwindV2Pass(), PMW);
297 }
298}
299
300void X86CodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) {
301 addModulePass(Pass: X86AsmPrinterBeginPass(), PMW, /*Force=*/true);
302}
303
304void X86CodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
305 addMachineFunctionPass(Pass: X86AsmPrinterPass(), PMW);
306}
307
308void X86CodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
309 addModulePass(Pass: X86AsmPrinterEndPass(), PMW, /*Force=*/true);
310}
311
312} // namespace
313
314void X86TargetMachine::registerPassBuilderCallbacks(PassBuilder &PB){
315#define GET_PASS_REGISTRY "X86PassRegistry.def"
316#include "llvm/Passes/TargetPassRegistry.inc"
317}
318
319Error X86TargetMachine::buildCodeGenPipeline(
320 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
321 raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
322 const CGPassBuilderOption &Opt, MCContext &Ctx,
323 PassInstrumentationCallbacks *PIC) {
324 auto CGPB = X86CodeGenPassBuilder(*this, Opt, PIC);
325 return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
326}
327