1//===-- X86CodeGenPassBuilder.cpp ---------------------------------*- C++ -*-=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file contains X86 CodeGen pipeline builder.
10/// TODO: Port CodeGen passes to new pass manager.
11//===----------------------------------------------------------------------===//
12
13#include "X86.h"
14#include "X86AsmPrinter.h"
15#include "X86TargetMachine.h"
16
17#include "llvm/CodeGen/AtomicExpand.h"
18#include "llvm/CodeGen/BreakFalseDeps.h"
19#include "llvm/CodeGen/CFIInstrInserter.h"
20#include "llvm/CodeGen/EHContGuardTargets.h"
21#include "llvm/CodeGen/EarlyIfConversion.h"
22#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
23#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
24#include "llvm/CodeGen/GlobalISel/Legalizer.h"
25#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
26#include "llvm/CodeGen/IndirectBrExpand.h"
27#include "llvm/CodeGen/InterleavedAccess.h"
28#include "llvm/CodeGen/JMCInstrumenter.h"
29#include "llvm/CodeGen/KCFI.h"
30#include "llvm/CodeGen/MachineCombiner.h"
31#include "llvm/MC/MCAsmInfo.h"
32#include "llvm/MC/MCStreamer.h"
33#include "llvm/Passes/CodeGenPassBuilder.h"
34#include "llvm/Passes/PassBuilder.h"
35#include "llvm/Support/CodeGen.h"
36#include "llvm/Transforms/CFGuard.h"
37
38using namespace llvm;
39
40namespace {
41
42class X86CodeGenPassBuilder : public CodeGenPassBuilder {
43 using Base = CodeGenPassBuilder;
44
45 X86TargetMachine &getTM() const {
46 return static_cast<X86TargetMachine &>(TM);
47 }
48
49public:
50 explicit X86CodeGenPassBuilder(X86TargetMachine &TM,
51 const CGPassBuilderOption &Opts,
52 PassInstrumentationCallbacks *PIC)
53 : CodeGenPassBuilder(TM, Opts, PIC) {}
54
55 void addIRPasses(PassManagerWrapper &PMW) override;
56 void addPreISel(PassManagerWrapper &PMW) override;
57 Error addInstSelector(PassManagerWrapper &PMW) override;
58
59 Error addIRTranslator(PassManagerWrapper &PMW) override;
60 void addPreLegalizeMachineIR(PassManagerWrapper &PMW) override;
61 Error addLegalizeMachineIR(PassManagerWrapper &PMW) override;
62 void addPreRegBankSelect(PassManagerWrapper &PMW) override;
63 Error addRegBankSelect(PassManagerWrapper &PMW) override;
64 Error addGlobalInstructionSelect(PassManagerWrapper &PMW) override;
65
66 void addILPOpts(PassManagerWrapper &PMW) override;
67 void addMachineSSAOptimization(PassManagerWrapper &PMW) override;
68 void addPreRegAlloc(PassManagerWrapper &PMW) override;
69 // TODO(boomanaiden154): We need to add addPostFastRegAllocRewrite here once
70 // it is available to support AMX.
71 void addPostRegAlloc(PassManagerWrapper &PMW) override;
72 void addPreSched2(PassManagerWrapper &PMW) override;
73 void addPreEmitPass(PassManagerWrapper &PMW) override;
74 void addPreEmitPass2(PassManagerWrapper &PMW) override;
75 // TODO(boomanaiden154): We need to add addRegAssignAndRewriteOptimized here
76 // once it is available to support AMX.
77 void addAsmPrinterBegin(PassManagerWrapper &PMW) override;
78 void addAsmPrinter(PassManagerWrapper &PMW) override;
79 void addAsmPrinterEnd(PassManagerWrapper &PMW) override;
80};
81
82void X86CodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
83 addFunctionPass(Pass: AtomicExpandPass(TM), PMW);
84
85 // We add both pass anyway and when these two passes run, one will be a
86 // no-op based on the optimization level/attributes.
87 addFunctionPass(Pass: X86LowerAMXIntrinsicsPass(&TM), PMW);
88 addFunctionPass(Pass: X86LowerAMXTypePass(&TM), PMW);
89
90 Base::addIRPasses(PMW);
91
92 if (getOptLevel() != CodeGenOptLevel::None) {
93 addFunctionPass(Pass: InterleavedAccessPass(TM), PMW);
94 addFunctionPass(Pass: X86PartialReductionPass(&getTM()), PMW);
95 }
96
97 // Add passes that handle indirect branch removal and insertion of a retpoline
98 // thunk. These will be a no-op unless a function subtarget has the retpoline
99 // feature enabled.
100 addFunctionPass(Pass: IndirectBrExpandPass(TM), PMW);
101
102 // Add Control Flow Guard checks.
103 const Triple &TT = TM.getTargetTriple();
104 if (TT.isOSWindows())
105 addFunctionPass(Pass: CFGuardPass(), PMW);
106
107 if (TM.Options.JMCInstrument) {
108 flushFPMsToMPM(PMW);
109 addModulePass(Pass: JMCInstrumenterPass(), PMW);
110 }
111}
112
113void X86CodeGenPassBuilder::addPreISel(PassManagerWrapper &PMW) {
114 // Only add this pass for 32-bit x86 Windows.
115 const Triple &TT = TM.getTargetTriple();
116 if (TT.isOSWindows() && TT.isX86_32()) {
117 flushFPMsToMPM(PMW);
118 addModulePass(Pass: X86WinEHStatePass(), PMW);
119 }
120}
121
122Error X86CodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) {
123 addMachineFunctionPass(Pass: X86ISelDAGToDAGPass(getTM()), PMW);
124
125 // For ELF, cleanup any local-dynamic TLS accesses
126 if (TM.getTargetTriple().isOSBinFormatELF() &&
127 getOptLevel() != CodeGenOptLevel::None) {
128 addMachineFunctionPass(Pass: X86CleanupLocalDynamicTLSPass(), PMW);
129 }
130
131 addMachineFunctionPass(Pass: X86GlobalBaseRegPass(), PMW);
132 addMachineFunctionPass(Pass: X86ArgumentStackSlotPass(), PMW);
133 return Error::success();
134}
135
136Error X86CodeGenPassBuilder::addIRTranslator(PassManagerWrapper &PMW) {
137 addMachineFunctionPass(Pass: IRTranslatorPass(getOptLevel()), PMW);
138 return Error::success();
139}
140
141void X86CodeGenPassBuilder::addPreLegalizeMachineIR(PassManagerWrapper &PMW) {
142 if (getOptLevel() != CodeGenOptLevel::None)
143 addMachineFunctionPass(Pass: X86PreLegalizerCombinerPass(), PMW);
144}
145
146Error X86CodeGenPassBuilder::addLegalizeMachineIR(PassManagerWrapper &PMW) {
147 addMachineFunctionPass(Pass: LegalizerPass(), PMW);
148 return Error::success();
149}
150
151void X86CodeGenPassBuilder::addPreRegBankSelect(PassManagerWrapper &PMW) {
152 if (getOptLevel() != CodeGenOptLevel::None)
153 addMachineFunctionPass(Pass: X86PostLegalizerCombinerPass(), PMW);
154}
155
156Error X86CodeGenPassBuilder::addRegBankSelect(PassManagerWrapper &PMW) {
157 addMachineFunctionPass(Pass: RegBankSelectPass(), PMW);
158 return Error::success();
159}
160
161Error X86CodeGenPassBuilder::addGlobalInstructionSelect(
162 PassManagerWrapper &PMW) {
163 addMachineFunctionPass(Pass: InstructionSelectPass(getOptLevel()), PMW);
164 // Add GlobalBaseReg in case there is no SelectionDAG passes afterwards
165 if (isGlobalISelAbortEnabled())
166 addMachineFunctionPass(Pass: X86GlobalBaseRegPass(), PMW);
167 return Error::success();
168}
169
170void X86CodeGenPassBuilder::addILPOpts(PassManagerWrapper &PMW) {
171 addMachineFunctionPass(Pass: EarlyIfConverterPass(), PMW);
172 if (getTM().getCLOpts().machine_combiner)
173 addMachineFunctionPass(Pass: MachineCombinerPass(), PMW);
174 addMachineFunctionPass(Pass: X86CmovConversionPass(), PMW);
175}
176
177void X86CodeGenPassBuilder::addMachineSSAOptimization(PassManagerWrapper &PMW) {
178 addMachineFunctionPass(Pass: X86DomainReassignmentPass(), PMW);
179 Base::addMachineSSAOptimization(PMW);
180}
181
182void X86CodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) {
183 if (getOptLevel() != CodeGenOptLevel::None) {
184 addMachineFunctionPass(Pass: LiveRangeShrinkPass(), PMW);
185 addMachineFunctionPass(Pass: X86FixupSetCCPass(), PMW);
186 addMachineFunctionPass(Pass: X86OptimizeLEAsPass(), PMW);
187 addMachineFunctionPass(Pass: X86CallFrameOptimizationPass(), PMW);
188 addMachineFunctionPass(Pass: X86AvoidStoreForwardingBlocksPass(), PMW);
189 }
190
191 addMachineFunctionPass(Pass: X86SuppressAPXForRelocationPass(), PMW);
192 addMachineFunctionPass(Pass: X86SpeculativeLoadHardeningPass(), PMW);
193 addMachineFunctionPass(Pass: X86FlagsCopyLoweringPass(), PMW);
194 addMachineFunctionPass(Pass: X86DynAllocaExpanderPass(), PMW);
195
196 if (getOptLevel() != CodeGenOptLevel::None)
197 addMachineFunctionPass(Pass: X86PreTileConfigPass(), PMW);
198 else
199 addMachineFunctionPass(Pass: X86FastPreTileConfigPass(), PMW);
200}
201
202void X86CodeGenPassBuilder::addPostRegAlloc(PassManagerWrapper &PMW) {
203 addMachineFunctionPass(Pass: X86LowerTileCopyPass(), PMW);
204 addMachineFunctionPass(Pass: X86FPStackifierPass(), PMW);
205 // When -O0 is enabled, the Load Value Injection Hardening pass will fall back
206 // to using the Speculative Execution Side Effect Suppression pass for
207 // mitigation. This is to prevent slow downs due to
208 // analyses needed by the LVIHardening pass when compiling at -O0.
209 if (getOptLevel() != CodeGenOptLevel::None) {
210 addMachineFunctionPass(Pass: X86LoadValueInjectionLoadHardeningPass(), PMW);
211 }
212}
213
214void X86CodeGenPassBuilder::addPreSched2(PassManagerWrapper &PMW) {
215 addMachineFunctionPass(Pass: X86ExpandPseudoPass(), PMW);
216 addMachineFunctionPass(Pass: MachineKCFIPass(), PMW);
217}
218
219void X86CodeGenPassBuilder::addPreEmitPass(PassManagerWrapper &PMW) {
220 if (getOptLevel() != CodeGenOptLevel::None) {
221 // TODO(boomanaiden154): Add X86ExecutionDomainFixPass here once it has
222 // been ported.
223 addMachineFunctionPass(Pass: BreakFalseDepsPass(), PMW);
224 }
225
226 addMachineFunctionPass(Pass: X86IndirectBranchTrackingPass(), PMW);
227 addMachineFunctionPass(Pass: X86InsertVZeroUpperPass(), PMW);
228
229 if (getOptLevel() != CodeGenOptLevel::None) {
230 addMachineFunctionPass(Pass: X86FixupBWInstsPass(), PMW);
231 // TODO(boomanaiden154): Add X86PadShortFunctionsPass here once it has been
232 // ported.
233 addMachineFunctionPass(Pass: X86FixupLEAsPass(), PMW);
234 addMachineFunctionPass(Pass: X86FixupInstTuningPass(), PMW);
235 addMachineFunctionPass(Pass: X86FixupVectorConstantsPass(), PMW);
236 }
237 addMachineFunctionPass(Pass: X86CompressEVEXPass(), PMW);
238 addMachineFunctionPass(Pass: X86InsertX87WaitPass(), PMW);
239
240 if (TM.getTargetTriple().isLFI())
241 addMachineFunctionPass(Pass: X86LFIRewritePass(), PMW);
242}
243
244void X86CodeGenPassBuilder::addPreEmitPass2(PassManagerWrapper &PMW) {
245 const Triple &TT = TM.getTargetTriple();
246 const MCAsmInfo &MAI = TM.getMCAsmInfo();
247
248 // The X86 Speculative Execution Pass must run after all control
249 // flow graph modifying passes. As a result it was listed to run right before
250 // the X86 Retpoline Thunks pass. The reason it must run after control flow
251 // graph modifications is that the model of LFENCE in LLVM has to be updated
252 // (FIXME: https://bugs.llvm.org/show_bug.cgi?id=45167). Currently the
253 // placement of this pass was hand checked to ensure that the subsequent
254 // passes don't move the code around the LFENCEs in a way that will hurt the
255 // correctness of this pass. This placement has been shown to work based on
256 // hand inspection of the codegen output.
257 addMachineFunctionPass(Pass: X86SpeculativeExecutionSideEffectSuppressionPass(),
258 PMW);
259 // TODO(boomanaiden154): Add X86IndirectThunksPass here
260 // once it has been ported.
261 addMachineFunctionPass(Pass: X86ReturnThunksPass(), PMW);
262
263 // Insert extra int3 instructions after trailing call instructions to avoid
264 // issues in the unwinder.
265 if (TT.isOSWindows() && TT.isX86_64())
266 addMachineFunctionPass(Pass: X86AvoidTrailingCallPass(), PMW);
267
268 // Verify basic block incoming and outgoing cfa offset and register values and
269 // correct CFA calculation rule where needed by inserting appropriate CFI
270 // instructions.
271 if (!TT.isOSDarwin() &&
272 (!TT.isOSWindows() ||
273 MAI.getExceptionHandlingType() == ExceptionHandling::DwarfCFI))
274 addMachineFunctionPass(Pass: CFIInstrInserterPass(), PMW);
275
276 if (TT.isOSWindows()) {
277 // Identify valid longjmp targets for Windows Control Flow Guard.
278 // TODO(boomanaiden154): Add CFGuardLongjmpPass here when it has been
279 // ported.
280 // Identify valid eh continuation targets for Windows EHCont Guard.
281 addMachineFunctionPass(Pass: EHContGuardTargetsPass(), PMW);
282 }
283
284 addMachineFunctionPass(Pass: X86LoadValueInjectionRetHardeningPass(), PMW);
285
286 // Insert pseudo probe annotation for callsite profiling
287 // TODO(boomanaiden154): Add PseudoProberInserterPass here once it has been
288 // ported.
289
290 // KCFI indirect call checks are lowered to a bundle, and on Darwin platforms,
291 // also CALL_RVMARKER.
292 addMachineFunctionPass(
293 Pass: UnpackMachineBundlesPass([&TT](const MachineFunction &MF) {
294 // Only run bundle expansion if the module uses kcfi, or there are
295 // relevant ObjC runtime functions present in the module.
296 const Function &F = MF.getFunction();
297 const Module *M = F.getParent();
298 return M->getModuleFlag(Key: "kcfi") ||
299 (TT.isOSDarwin() &&
300 (M->getFunction(Name: "objc_retainAutoreleasedReturnValue") ||
301 M->getFunction(Name: "objc_unsafeClaimAutoreleasedReturnValue")));
302 }),
303 PMW);
304
305 // Analyzes and emits pseudos to support Win x64 Unwind V2. This pass must run
306 // after all real instructions have been added to the epilog.
307 if (TT.isOSWindows() && TT.isX86_64()) {
308 addMachineFunctionPass(Pass: X86WinEHUnwindV2Pass(), PMW);
309 }
310}
311
312void X86CodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) {
313 addModulePass(Pass: X86AsmPrinterBeginPass(), PMW, /*Force=*/true);
314}
315
316void X86CodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
317 addMachineFunctionPass(Pass: X86AsmPrinterPass(), PMW);
318}
319
320void X86CodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
321 addModulePass(Pass: X86AsmPrinterEndPass(), PMW, /*Force=*/true);
322}
323
324} // namespace
325
326void X86TargetMachine::registerPassBuilderCallbacks(PassBuilder &PB){
327#define GET_PASS_REGISTRY "X86PassRegistry.def"
328#include "llvm/Passes/TargetPassRegistry.inc"
329}
330
331Error X86TargetMachine::buildCodeGenPipeline(
332 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
333 raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
334 const CGPassBuilderOption &Opt, MCContext &Ctx,
335 PassInstrumentationCallbacks *PIC) {
336 auto CGPB = X86CodeGenPassBuilder(*this, Opt, PIC);
337 return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
338}
339