1//===-- X86CodeGenPassBuilder.cpp ---------------------------------*- C++ -*-=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file contains X86 CodeGen pipeline builder.
10///
11//===----------------------------------------------------------------------===//
12
13#include "X86.h"
14#include "X86AsmPrinter.h"
15#include "X86TargetMachine.h"
16
17#include "llvm/CodeGen/AtomicExpand.h"
18#include "llvm/CodeGen/BreakFalseDeps.h"
19#include "llvm/CodeGen/CFGuardLongjmp.h"
20#include "llvm/CodeGen/CFIInstrInserter.h"
21#include "llvm/CodeGen/EHContGuardTargets.h"
22#include "llvm/CodeGen/EarlyIfConversion.h"
23#include "llvm/CodeGen/ExecutionDomainFix.h"
24#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
25#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
26#include "llvm/CodeGen/GlobalISel/Legalizer.h"
27#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
28#include "llvm/CodeGen/IndirectBrExpand.h"
29#include "llvm/CodeGen/InterleavedAccess.h"
30#include "llvm/CodeGen/JMCInstrumenter.h"
31#include "llvm/CodeGen/KCFI.h"
32#include "llvm/CodeGen/MachineCombiner.h"
33#include "llvm/CodeGen/PseudoProbeInserter.h"
34#include "llvm/CodeGen/RegAllocGreedyPass.h"
35#include "llvm/MC/MCAsmInfo.h"
36#include "llvm/MC/MCStreamer.h"
37#include "llvm/Pass.h"
38#include "llvm/Passes/CodeGenPassBuilder.h"
39#include "llvm/Passes/PassBuilder.h"
40#include "llvm/Support/CodeGen.h"
41#include "llvm/Target/CGPassBuilderOption.h"
42#include "llvm/Transforms/CFGuard.h"
43
44using namespace llvm;
45
46namespace {
47
48class X86CodeGenPassBuilder : public CodeGenPassBuilder {
49 using Base = CodeGenPassBuilder;
50
51 X86TargetMachine &getTM() const {
52 return static_cast<X86TargetMachine &>(TM);
53 }
54
55public:
56 explicit X86CodeGenPassBuilder(X86TargetMachine &TM,
57 const CGPassBuilderOption &Opts,
58 PassInstrumentationCallbacks *PIC)
59 : CodeGenPassBuilder(TM, Opts, PIC) {}
60
61 void addIRPasses(PassManagerWrapper &PMW) override;
62 void addPreISel(PassManagerWrapper &PMW) override;
63 Error addInstSelector(PassManagerWrapper &PMW) override;
64
65 Error addIRTranslator(PassManagerWrapper &PMW) override;
66 void addPreLegalizeMachineIR(PassManagerWrapper &PMW) override;
67 Error addLegalizeMachineIR(PassManagerWrapper &PMW) override;
68 void addPreRegBankSelect(PassManagerWrapper &PMW) override;
69 Error addRegBankSelect(PassManagerWrapper &PMW) override;
70 Error addGlobalInstructionSelect(PassManagerWrapper &PMW) override;
71
72 void addMachineSSAOptimization(PassManagerWrapper &PMW) override;
73 void addILPOpts(PassManagerWrapper &PMW) override;
74 void addPreRegAlloc(PassManagerWrapper &PMW) override;
75 Error addRegAssignAndRewriteFast(PassManagerWrapper &PMW) override;
76 Expected<bool>
77 addRegAssignAndRewriteOptimized(PassManagerWrapper &PMW) override;
78 void addPostRegAlloc(PassManagerWrapper &PMW) override;
79 void addPreSched2(PassManagerWrapper &PMW) override;
80 void addPreEmitPass(PassManagerWrapper &PMW) override;
81 void addPreEmitPass2(PassManagerWrapper &PMW) override;
82 void addAsmPrinterBegin(PassManagerWrapper &PMW) override;
83 void addAsmPrinter(PassManagerWrapper &PMW) override;
84 void addAsmPrinterEnd(PassManagerWrapper &PMW) override;
85};
86
87void X86CodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
88 addFunctionPass(Pass: AtomicExpandPass(TM), PMW);
89
90 // We add both pass anyway and when these two passes run, one will be a
91 // no-op based on the optimization level/attributes.
92 addFunctionPass(Pass: X86LowerAMXIntrinsicsPass(&TM), PMW);
93 addFunctionPass(Pass: X86LowerAMXTypePass(&TM), PMW);
94
95 Base::addIRPasses(PMW);
96
97 if (getOptLevel() != CodeGenOptLevel::None) {
98 addFunctionPass(Pass: InterleavedAccessPass(TM), PMW);
99 addFunctionPass(Pass: X86PartialReductionPass(&getTM()), PMW);
100 }
101
102 // Add passes that handle indirect branch removal and insertion of a retpoline
103 // thunk. These will be a no-op unless a function subtarget has the retpoline
104 // feature enabled.
105 addFunctionPass(Pass: IndirectBrExpandPass(TM), PMW);
106
107 // Add Control Flow Guard checks.
108 const Triple &TT = TM.getTargetTriple();
109 if (TT.isOSWindows())
110 addFunctionPass(Pass: CFGuardPass(), PMW);
111
112 if (TM.Options.JMCInstrument) {
113 flushFPMsToMPM(PMW);
114 addModulePass(Pass: JMCInstrumenterPass(), PMW);
115 }
116
117 addFunctionPass(Pass: X86FenceNonTemporalStoresPass(&getTM()), PMW);
118}
119
120void X86CodeGenPassBuilder::addPreISel(PassManagerWrapper &PMW) {
121 // Only add this pass for 32-bit x86 Windows.
122 const Triple &TT = TM.getTargetTriple();
123 if (TT.isOSWindows() && TT.isX86_32()) {
124 flushFPMsToMPM(PMW);
125 addModulePass(Pass: X86WinEHStatePass(), PMW);
126 }
127}
128
129Error X86CodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) {
130 addMachineFunctionPass(Pass: X86ISelDAGToDAGPass(getTM()), PMW);
131
132 // For ELF, cleanup any local-dynamic TLS accesses
133 if (TM.getTargetTriple().isOSBinFormatELF() &&
134 getOptLevel() != CodeGenOptLevel::None) {
135 addMachineFunctionPass(Pass: X86CleanupLocalDynamicTLSPass(), PMW);
136 }
137
138 addMachineFunctionPass(Pass: X86GlobalBaseRegPass(), PMW);
139 addMachineFunctionPass(Pass: X86ArgumentStackSlotPass(), PMW);
140 return Error::success();
141}
142
143Error X86CodeGenPassBuilder::addIRTranslator(PassManagerWrapper &PMW) {
144 addMachineFunctionPass(Pass: IRTranslatorPass(getOptLevel()), PMW);
145 return Error::success();
146}
147
148void X86CodeGenPassBuilder::addPreLegalizeMachineIR(PassManagerWrapper &PMW) {
149 if (getOptLevel() != CodeGenOptLevel::None)
150 addMachineFunctionPass(Pass: X86PreLegalizerCombinerPass(), PMW);
151}
152
153Error X86CodeGenPassBuilder::addLegalizeMachineIR(PassManagerWrapper &PMW) {
154 addMachineFunctionPass(Pass: LegalizerPass(), PMW);
155 return Error::success();
156}
157
158void X86CodeGenPassBuilder::addPreRegBankSelect(PassManagerWrapper &PMW) {
159 if (getOptLevel() != CodeGenOptLevel::None)
160 addMachineFunctionPass(Pass: X86PostLegalizerCombinerPass(), PMW);
161}
162
163Error X86CodeGenPassBuilder::addRegBankSelect(PassManagerWrapper &PMW) {
164 addMachineFunctionPass(Pass: RegBankSelectPass(), PMW);
165 return Error::success();
166}
167
168Error X86CodeGenPassBuilder::addGlobalInstructionSelect(
169 PassManagerWrapper &PMW) {
170 addMachineFunctionPass(Pass: InstructionSelectPass(getOptLevel()), PMW);
171 // Add GlobalBaseReg in case there is no SelectionDAG passes afterwards
172 if (isGlobalISelAbortEnabled())
173 addMachineFunctionPass(Pass: X86GlobalBaseRegPass(), PMW);
174 return Error::success();
175}
176
177void X86CodeGenPassBuilder::addMachineSSAOptimization(PassManagerWrapper &PMW) {
178 addMachineFunctionPass(Pass: X86DomainReassignmentPass(), PMW);
179 Base::addMachineSSAOptimization(PMW);
180}
181
182void X86CodeGenPassBuilder::addILPOpts(PassManagerWrapper &PMW) {
183 addMachineFunctionPass(Pass: EarlyIfConverterPass(), PMW);
184 if (getTM().getCLOpts().machine_combiner)
185 addMachineFunctionPass(Pass: MachineCombinerPass(), PMW);
186 addMachineFunctionPass(Pass: X86CmovConversionPass(), PMW);
187}
188
189void X86CodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) {
190 if (getOptLevel() != CodeGenOptLevel::None) {
191 addMachineFunctionPass(Pass: LiveRangeShrinkPass(), PMW);
192 addMachineFunctionPass(Pass: X86FixupSetCCPass(), PMW);
193 addMachineFunctionPass(Pass: X86OptimizeLEAsPass(), PMW);
194 addMachineFunctionPass(Pass: X86CallFrameOptimizationPass(), PMW);
195 addMachineFunctionPass(Pass: X86AvoidStoreForwardingBlocksPass(), PMW);
196 }
197
198 addMachineFunctionPass(Pass: X86SuppressAPXForRelocationPass(), PMW);
199 addMachineFunctionPass(Pass: X86SpeculativeLoadHardeningPass(), PMW);
200 addMachineFunctionPass(Pass: X86FlagsCopyLoweringPass(), PMW);
201 addMachineFunctionPass(Pass: X86DynAllocaExpanderPass(), PMW);
202
203 if (getOptLevel() != CodeGenOptLevel::None)
204 addMachineFunctionPass(Pass: X86PreTileConfigPass(), PMW);
205 else
206 addMachineFunctionPass(Pass: X86FastPreTileConfigPass(), PMW);
207}
208
209static bool onlyAllocateTileRegisters(const TargetRegisterInfo &TRI,
210 const MachineRegisterInfo &MRI,
211 const Register Reg) {
212 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
213 return static_cast<const X86RegisterInfo &>(TRI).isTileRegisterClass(RC);
214}
215
216Error X86CodeGenPassBuilder::addRegAssignAndRewriteFast(
217 PassManagerWrapper &PMW) {
218 Error PossibleError = Base::addRegAssignAndRewriteFast(PMW);
219 addMachineFunctionPass(Pass: X86FastTileConfigPass(), PMW);
220 return PossibleError;
221}
222
223Expected<bool> X86CodeGenPassBuilder::addRegAssignAndRewriteOptimized(
224 PassManagerWrapper &PMW) {
225 // Don't support tile RA when RA is specified by command line "-regalloc".
226 if (Opt.RegAlloc == RegAllocType::Default && getTM().getCLOpts().tile_ra) {
227 // Allocate tile register first.
228 addMachineFunctionPass(
229 Pass: RAGreedyPass({RAGreedyPass::Options(onlyAllocateTileRegisters)}), PMW);
230 addMachineFunctionPass(Pass: X86TileConfigPass(), PMW);
231 }
232 return Base::addRegAssignAndRewriteOptimized(PMW);
233}
234
235void X86CodeGenPassBuilder::addPostRegAlloc(PassManagerWrapper &PMW) {
236 addMachineFunctionPass(Pass: X86LowerTileCopyPass(), PMW);
237 addMachineFunctionPass(Pass: X86FPStackifierPass(), PMW);
238 // When -O0 is enabled, the Load Value Injection Hardening pass will fall back
239 // to using the Speculative Execution Side Effect Suppression pass for
240 // mitigation. This is to prevent slow downs due to
241 // analyses needed by the LVIHardening pass when compiling at -O0.
242 if (getOptLevel() != CodeGenOptLevel::None) {
243 addMachineFunctionPass(Pass: X86LoadValueInjectionLoadHardeningPass(), PMW);
244 }
245}
246
247void X86CodeGenPassBuilder::addPreSched2(PassManagerWrapper &PMW) {
248 addMachineFunctionPass(Pass: X86ExpandPseudoPass(), PMW);
249 addMachineFunctionPass(Pass: MachineKCFIPass(), PMW);
250}
251
252void X86CodeGenPassBuilder::addPreEmitPass(PassManagerWrapper &PMW) {
253 if (getOptLevel() != CodeGenOptLevel::None) {
254 addMachineFunctionPass(Pass: ExecutionDomainFixPass(), PMW);
255 addMachineFunctionPass(Pass: BreakFalseDepsPass(), PMW);
256 }
257
258 addMachineFunctionPass(Pass: X86IndirectBranchTrackingPass(), PMW);
259 addMachineFunctionPass(Pass: X86InsertVZeroUpperPass(), PMW);
260
261 if (getOptLevel() != CodeGenOptLevel::None) {
262 addMachineFunctionPass(Pass: X86FixupBWInstsPass(), PMW);
263 addMachineFunctionPass(Pass: X86PadShortFunctionsPass(), PMW);
264 addMachineFunctionPass(Pass: X86FixupLEAsPass(), PMW);
265 addMachineFunctionPass(Pass: X86FixupInstTuningPass(), PMW);
266 addMachineFunctionPass(Pass: X86FixupVectorConstantsPass(), PMW);
267 }
268 addMachineFunctionPass(Pass: X86CompressEVEXPass(), PMW);
269 addMachineFunctionPass(Pass: X86InsertX87WaitPass(), PMW);
270
271 if (TM.getTargetTriple().isLFI())
272 addMachineFunctionPass(Pass: X86LFIRewritePass(), PMW);
273}
274
275void X86CodeGenPassBuilder::addPreEmitPass2(PassManagerWrapper &PMW) {
276 const Triple &TT = TM.getTargetTriple();
277
278 // The X86 Speculative Execution Pass must run after all control
279 // flow graph modifying passes. As a result it was listed to run right before
280 // the X86 Retpoline Thunks pass. The reason it must run after control flow
281 // graph modifications is that the model of LFENCE in LLVM has to be updated
282 // (FIXME: https://bugs.llvm.org/show_bug.cgi?id=45167). Currently the
283 // placement of this pass was hand checked to ensure that the subsequent
284 // passes don't move the code around the LFENCEs in a way that will hurt the
285 // correctness of this pass. This placement has been shown to work based on
286 // hand inspection of the codegen output.
287 addMachineFunctionPass(Pass: X86SpeculativeExecutionSideEffectSuppressionPass(),
288 PMW);
289 addMachineFunctionPass(Pass: X86IndirectThunksPass(), PMW);
290 addMachineFunctionPass(Pass: X86ReturnThunksPass(), PMW);
291
292 // Insert extra int3 instructions after trailing call instructions to avoid
293 // issues in the unwinder.
294 if (TT.isOSWindows() && TT.isX86_64())
295 addMachineFunctionPass(Pass: X86AvoidTrailingCallPass(), PMW);
296
297 // Verify basic block incoming and outgoing cfa offset and register values and
298 // correct CFA calculation rule where needed by inserting appropriate CFI
299 // instructions. CFIInstrInserter self-skips targets that do not use DWARF
300 // CFI.
301 if (!TT.isOSDarwin())
302 addMachineFunctionPass(Pass: CFIInstrInserterPass(), PMW);
303
304 if (TT.isOSWindows()) {
305 // Identify valid longjmp targets for Windows Control Flow Guard.
306 addMachineFunctionPass(Pass: CFGuardLongjmpPass(), PMW);
307 // Identify valid eh continuation targets for Windows EHCont Guard.
308 addMachineFunctionPass(Pass: EHContGuardTargetsPass(), PMW);
309 }
310
311 addMachineFunctionPass(Pass: X86LoadValueInjectionRetHardeningPass(), PMW);
312
313 // Insert pseudo probe annotation for callsite profiling
314 addMachineFunctionPass(Pass: PseudoProbeInserterPass(), PMW);
315
316 // KCFI indirect call checks are lowered to a bundle, and on Darwin platforms,
317 // also CALL_RVMARKER.
318 addMachineFunctionPass(
319 Pass: UnpackMachineBundlesPass([&TT](const MachineFunction &MF) {
320 // Only run bundle expansion if the module uses kcfi, or there are
321 // relevant ObjC runtime functions present in the module.
322 const Function &F = MF.getFunction();
323 const Module *M = F.getParent();
324 return M->getModuleFlag(Key: "kcfi") ||
325 (TT.isOSDarwin() &&
326 (M->getFunction(Name: "objc_retainAutoreleasedReturnValue") ||
327 M->getFunction(Name: "objc_unsafeClaimAutoreleasedReturnValue")));
328 }),
329 PMW);
330
331 // Analyzes and emits pseudos to support Win x64 Unwind V2. This pass must run
332 // after all real instructions have been added to the epilog.
333 if (TT.isOSWindows() && TT.isX86_64()) {
334 addMachineFunctionPass(Pass: X86WinEHUnwindV2Pass(), PMW);
335 }
336}
337
338void X86CodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) {
339 addModulePass(Pass: X86AsmPrinterBeginPass(), PMW, /*Force=*/true);
340}
341
342void X86CodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
343 addMachineFunctionPass(Pass: X86AsmPrinterPass(), PMW);
344}
345
346void X86CodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
347 addModulePass(Pass: X86AsmPrinterEndPass(), PMW, /*Force=*/true);
348}
349
350} // namespace
351
352void X86TargetMachine::registerPassBuilderCallbacks(PassBuilder &PB){
353#define GET_PASS_REGISTRY "X86PassRegistry.def"
354#include "llvm/Passes/TargetPassRegistry.inc"
355}
356
357Error X86TargetMachine::buildCodeGenPipeline(
358 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
359 raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
360 const CGPassBuilderOption &Opt, MCContext &Ctx,
361 PassInstrumentationCallbacks *PIC) {
362 auto CGPB = X86CodeGenPassBuilder(*this, Opt, PIC);
363 return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
364}
365