1//===--- CodeGenPassBuilder.cpp --------------------------------------- ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines interfaces to access the target independent code
10// generation passes provided by the LLVM backend.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Passes/CodeGenPassBuilder.h"
15#include "llvm/ADT/SmallVector.h"
16#include "llvm/ADT/StringRef.h"
17#include "llvm/Analysis/CGSCCPassManager.h"
18#include "llvm/Analysis/ProfileSummaryInfo.h"
19#include "llvm/Analysis/RuntimeLibcallInfo.h"
20#include "llvm/CodeGen/AsmPrinter.h"
21#include "llvm/CodeGen/AsmPrinterAnalysis.h"
22#include "llvm/CodeGen/BranchFoldingPass.h"
23#include "llvm/CodeGen/CodeGenPrepare.h"
24#include "llvm/CodeGen/DeadMachineInstructionElim.h"
25#include "llvm/CodeGen/DetectDeadLanes.h"
26#include "llvm/CodeGen/DwarfEHPrepare.h"
27#include "llvm/CodeGen/ExpandIRInsts.h"
28#include "llvm/CodeGen/ExpandPostRAPseudos.h"
29#include "llvm/CodeGen/ExpandReductions.h"
30#include "llvm/CodeGen/FEntryInserter.h"
31#include "llvm/CodeGen/FinalizeISel.h"
32#include "llvm/CodeGen/FixupStatepointCallerSaved.h"
33#include "llvm/CodeGen/FuncletLayout.h"
34#include "llvm/CodeGen/GCEmptyBasicBlocks.h"
35#include "llvm/CodeGen/GCMetadata.h"
36#include "llvm/CodeGen/GlobalMerge.h"
37#include "llvm/CodeGen/GlobalMergeFunctions.h"
38#include "llvm/CodeGen/ImplicitNullChecks.h"
39#include "llvm/CodeGen/IndirectBrExpand.h"
40#include "llvm/CodeGen/InitUndef.h"
41#include "llvm/CodeGen/InlineAsmPrepare.h"
42#include "llvm/CodeGen/InterleavedAccess.h"
43#include "llvm/CodeGen/InterleavedLoadCombine.h"
44#include "llvm/CodeGen/LiveDebugValuesPass.h"
45#include "llvm/CodeGen/LiveIntervals.h"
46#include "llvm/CodeGen/LiveVariables.h"
47#include "llvm/CodeGen/LocalStackSlotAllocation.h"
48#include "llvm/CodeGen/LowerEmuTLS.h"
49#include "llvm/CodeGen/MIRPrinter.h"
50#include "llvm/CodeGen/MachineBlockPlacement.h"
51#include "llvm/CodeGen/MachineCSE.h"
52#include "llvm/CodeGen/MachineCopyPropagation.h"
53#include "llvm/CodeGen/MachineFunctionAnalysis.h"
54#include "llvm/CodeGen/MachineLICM.h"
55#include "llvm/CodeGen/MachineLateInstrsCleanup.h"
56#include "llvm/CodeGen/MachineModuleInfo.h"
57#include "llvm/CodeGen/MachinePassManager.h"
58#include "llvm/CodeGen/MachineScheduler.h"
59#include "llvm/CodeGen/MachineSink.h"
60#include "llvm/CodeGen/MachineVerifier.h"
61#include "llvm/CodeGen/OptimizePHIs.h"
62#include "llvm/CodeGen/PEI.h"
63#include "llvm/CodeGen/PHIElimination.h"
64#include "llvm/CodeGen/PatchableFunction.h"
65#include "llvm/CodeGen/PeepholeOptimizer.h"
66#include "llvm/CodeGen/PostRAMachineSink.h"
67#include "llvm/CodeGen/PostRASchedulerList.h"
68#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
69#include "llvm/CodeGen/ProcessImplicitDefs.h"
70#include "llvm/CodeGen/RegAllocEvictionAdvisor.h"
71#include "llvm/CodeGen/RegAllocFast.h"
72#include "llvm/CodeGen/RegAllocGreedyPass.h"
73#include "llvm/CodeGen/RegUsageInfoCollector.h"
74#include "llvm/CodeGen/RegUsageInfoPropagate.h"
75#include "llvm/CodeGen/RegisterCoalescerPass.h"
76#include "llvm/CodeGen/RegisterUsageInfo.h"
77#include "llvm/CodeGen/RemoveLoadsIntoFakeUses.h"
78#include "llvm/CodeGen/RemoveRedundantDebugValues.h"
79#include "llvm/CodeGen/RenameIndependentSubregs.h"
80#include "llvm/CodeGen/ReplaceWithVeclib.h"
81#include "llvm/CodeGen/ResetMachineFunctionPass.h"
82#include "llvm/CodeGen/SafeStack.h"
83#include "llvm/CodeGen/SanitizerBinaryMetadata.h"
84#include "llvm/CodeGen/SelectOptimize.h"
85#include "llvm/CodeGen/ShadowStackGCLowering.h"
86#include "llvm/CodeGen/ShrinkWrap.h"
87#include "llvm/CodeGen/SjLjEHPrepare.h"
88#include "llvm/CodeGen/StackColoring.h"
89#include "llvm/CodeGen/StackFrameLayoutAnalysisPass.h"
90#include "llvm/CodeGen/StackProtector.h"
91#include "llvm/CodeGen/StackSlotColoring.h"
92#include "llvm/CodeGen/TailDuplication.h"
93#include "llvm/CodeGen/TargetPassConfig.h"
94#include "llvm/CodeGen/TwoAddressInstructionPass.h"
95#include "llvm/CodeGen/UnreachableBlockElim.h"
96#include "llvm/CodeGen/VirtRegMap.h"
97#include "llvm/CodeGen/WinEHPrepare.h"
98#include "llvm/CodeGen/XRayInstrumentation.h"
99#include "llvm/IR/PassManager.h"
100#include "llvm/IR/Verifier.h"
101#include "llvm/IRPrinter/IRPrintingPasses.h"
102#include "llvm/MC/MCAsmInfo.h"
103#include "llvm/MC/MCStreamer.h"
104#include "llvm/MC/MCTargetOptions.h"
105#include "llvm/MC/TargetRegistry.h"
106#include "llvm/Passes/TriggerCrashPasses.h"
107#include "llvm/Support/CodeGen.h"
108#include "llvm/Support/Debug.h"
109#include "llvm/Support/Error.h"
110#include "llvm/Support/ErrorHandling.h"
111#include "llvm/Target/CGPassBuilderOption.h"
112#include "llvm/Target/TargetMachine.h"
113#include "llvm/Transforms/ObjCARC.h"
114#include "llvm/Transforms/Scalar/ConstantHoisting.h"
115#include "llvm/Transforms/Scalar/LoopPassManager.h"
116#include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
117#include "llvm/Transforms/Scalar/LoopTermFold.h"
118#include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
119#include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
120#include "llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h"
121#include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
122#include "llvm/Transforms/Utils/LowerInvoke.h"
123#include <cassert>
124#include <utility>
125
126using namespace llvm;
127
128namespace llvm {
129#define DUMMY_MACHINE_FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
130 AnalysisKey PASS_NAME::Key;
131#include "llvm/Passes/MachinePassRegistry.def"
132} // namespace llvm
133
134CodeGenPassBuilder::CodeGenPassBuilder(TargetMachine &TM,
135 const CGPassBuilderOption &Opts,
136 PassInstrumentationCallbacks *PIC)
137 : TM(TM), Opt(Opts), PIC(PIC) {
138 // Target could set CGPassBuilderOption::MISchedPostRA to true to achieve
139 // substitutePass(&PostRASchedulerID, &PostMachineSchedulerID)
140
141 // Target should override TM.Options.EnableIPRA in their target-specific
142 // LLVMTM ctor. See TargetMachine::setGlobalISel for example.
143 if (Opt.EnableIPRA) {
144 TM.Options.EnableIPRA = *Opt.EnableIPRA;
145 } else {
146 // If not explicitly specified, use target default.
147 TM.Options.EnableIPRA |= TM.useIPRA();
148 }
149
150 if (Opt.EnableGlobalISelAbort)
151 TM.Options.GlobalISelAbort = *Opt.EnableGlobalISelAbort;
152
153 if (Opt.EnableRegAllocFastTied != cl::boolOrDefault::BOU_UNSET)
154 TM.setEnableTiedFastRegAlloc(Opt.EnableRegAllocFastTied ==
155 cl::boolOrDefault::BOU_TRUE);
156
157 // An explicit RegAlloc choice implies its pipeline: only the fast
158 // allocator uses the unoptimized one.
159 if (Opt.OptimizeRegAlloc == cl::boolOrDefault::BOU_UNSET) {
160 bool Optimized = Opt.RegAlloc > RegAllocType::Default
161 ? Opt.RegAlloc != RegAllocType::Fast
162 : getOptLevel() != CodeGenOptLevel::None;
163 Opt.OptimizeRegAlloc =
164 Optimized ? cl::boolOrDefault::BOU_TRUE : cl::boolOrDefault::BOU_FALSE;
165 }
166}
167
168// Out-of-line to anchor the vtable in this translation unit.
169CodeGenPassBuilder::~CodeGenPassBuilder() = default;
170
171Error CodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) {
172 return make_error<StringError>(Args: "addInstSelector is not overridden",
173 Args: inconvertibleErrorCode());
174}
175
176Error CodeGenPassBuilder::addIRTranslator(PassManagerWrapper &PMW) {
177 return make_error<StringError>(Args: "addIRTranslator is not overridden",
178 Args: inconvertibleErrorCode());
179}
180
181Error CodeGenPassBuilder::addLegalizeMachineIR(PassManagerWrapper &PMW) {
182 return make_error<StringError>(Args: "addLegalizeMachineIR is not overridden",
183 Args: inconvertibleErrorCode());
184}
185
186Error CodeGenPassBuilder::addRegBankSelect(PassManagerWrapper &PMW) {
187 return make_error<StringError>(Args: "addRegBankSelect is not overridden",
188 Args: inconvertibleErrorCode());
189}
190
191Error CodeGenPassBuilder::addGlobalInstructionSelect(PassManagerWrapper &PMW) {
192 return make_error<StringError>(Args: "addGlobalInstructionSelect is not overridden",
193 Args: inconvertibleErrorCode());
194}
195
196void CodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) {
197 llvm_unreachable("addAsmPrinterBegin is not overriden");
198}
199
200void CodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
201 llvm_unreachable("addAsmPrinter is not overridden");
202}
203
204void CodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
205 llvm_unreachable("addAsmPrinterEnd is not overriden");
206}
207
208void CodeGenPassBuilder::flushFPMsToMPM(PassManagerWrapper &PMW,
209 bool FreeMachineFunctions) {
210 if (PMW.FPM.isEmpty() && PMW.MFPM.isEmpty())
211 return;
212 if (!PMW.MFPM.isEmpty()) {
213 PMW.FPM.addPass(
214 Pass: createFunctionToMachineFunctionPassAdaptor(Pass: std::move(PMW.MFPM)));
215 PMW.MFPM = MachineFunctionPassManager();
216 }
217 if (FreeMachineFunctions)
218 PMW.FPM.addPass(Pass: FreeMachineFunctionPass());
219 if (AddInCGSCCOrder) {
220 PMW.MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(
221 Pass: createCGSCCToFunctionPassAdaptor(Pass: std::move(PMW.FPM))));
222 } else {
223 PMW.MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(PMW.FPM)));
224 }
225 PMW.FPM = FunctionPassManager();
226}
227
228Error CodeGenPassBuilder::buildPipeline(
229 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
230 raw_pwrite_stream *DwoOut, CodeGenFileType FileType, MCContext &Ctx) {
231 auto StartStopInfo = TargetPassConfig::getStartStopInfo(PIC&: *PIC);
232 if (!StartStopInfo)
233 return StartStopInfo.takeError();
234 setStartStopPasses(*StartStopInfo);
235
236 bool PrintAsm = TargetPassConfig::willCompleteCodeGenPipeline();
237 bool PrintMIR = !PrintAsm && FileType != CodeGenFileType::Null;
238
239 PassManagerWrapper PMW(MPM);
240
241 addModulePass(Pass: RequireAnalysisPass<MachineModuleAnalysis, Module>(), PMW,
242 /*Force=*/true);
243 addModulePass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>(), PMW,
244 /*Force=*/true);
245 addModulePass(Pass: RequireAnalysisPass<CollectorMetadataAnalysis, Module>(), PMW,
246 /*Force=*/true);
247 addModulePass(Pass: RequireAnalysisPass<RuntimeLibraryAnalysis, Module>(), PMW,
248 /*Force=*/true);
249 addModulePass(Pass: RequireAnalysisPass<LibcallLoweringModuleAnalysis, Module>(),
250 PMW,
251 /*Force=*/true);
252 addISelPasses(PMW);
253 flushFPMsToMPM(PMW);
254
255 if (PrintAsm) {
256 Expected<std::unique_ptr<MCStreamer>> MCStreamerOrErr =
257 TM.createMCStreamer(Out, DwoOut, FileType, Ctx);
258 if (!MCStreamerOrErr)
259 return MCStreamerOrErr.takeError();
260 std::unique_ptr<AsmPrinter> Printer(
261 TM.getTarget().createAsmPrinter(TM, Streamer: std::move(*MCStreamerOrErr)));
262 if (!Printer)
263 return createStringError(Fmt: "failed to create AsmPrinter");
264 MAM.registerPass(PassBuilder: [&] { return AsmPrinterAnalysis(std::move(Printer)); });
265 addAsmPrinterBegin(PMW);
266 }
267
268 if (PrintMIR)
269 addModulePass(Pass: PrintMIRPreparePass(Out), PMW, /*Force=*/true);
270
271 if (auto Err = addCoreISelPasses(PMW))
272 return Err;
273
274 if (auto Err = addMachinePasses(PMW))
275 return Err;
276
277 if (!Opt.DisableVerify && TM.Options.EnableDefaultMachineVerifier)
278 addMachineFunctionPass(Pass: MachineVerifierPass(), PMW);
279
280 // We add AsmPrinter regardless if we are emitting MIR or Assembly as the
281 // final output so that -stop-before=<target>-asm-printer works. When printing
282 // MIR as the final output, we never end up running AsmPrinter.
283 addAsmPrinter(PMW);
284
285 if (PrintAsm) {
286 flushFPMsToMPM(PMW, /*FreeMachineFunctions=*/true);
287 addAsmPrinterEnd(PMW);
288 } else {
289 if (PrintMIR)
290 addMachineFunctionPass(Pass: PrintMIRPass(Out), PMW, /*Force=*/true);
291 flushFPMsToMPM(PMW, /*FreeMachineFunctions=*/true);
292 }
293
294 return verifyStartStop(Info: *StartStopInfo);
295}
296
297void CodeGenPassBuilder::setStartStopPasses(
298 const TargetPassConfig::StartStopInfo &Info) {
299 if (!Info.StartPass.empty()) {
300 Started = false;
301 BeforeCallbacks.emplace_back(Args: [this, &Info, AfterFlag = Info.StartAfter,
302 Count = 0u](StringRef ClassName) mutable {
303 if (Count == Info.StartInstanceNum) {
304 if (AfterFlag) {
305 AfterFlag = false;
306 Started = true;
307 }
308 return Started;
309 }
310
311 auto PassName = PIC->getPassNameForClassName(ClassName);
312 if (Info.StartPass == PassName && ++Count == Info.StartInstanceNum)
313 Started = !Info.StartAfter;
314
315 return Started;
316 });
317 }
318
319 if (!Info.StopPass.empty()) {
320 Stopped = false;
321 BeforeCallbacks.emplace_back(Args: [this, &Info, AfterFlag = Info.StopAfter,
322 Count = 0u](StringRef ClassName) mutable {
323 if (Count == Info.StopInstanceNum) {
324 if (AfterFlag) {
325 AfterFlag = false;
326 Stopped = true;
327 }
328 return !Stopped;
329 }
330
331 auto PassName = PIC->getPassNameForClassName(ClassName);
332 if (Info.StopPass == PassName && ++Count == Info.StopInstanceNum)
333 Stopped = !Info.StopAfter;
334 return !Stopped;
335 });
336 }
337}
338
339Error CodeGenPassBuilder::verifyStartStop(
340 const TargetPassConfig::StartStopInfo &Info) const {
341 if (Started && Stopped)
342 return Error::success();
343
344 if (!Started)
345 return make_error<StringError>(
346 Args: "Can't find start pass \"" + Info.StartPass + "\".",
347 Args: std::make_error_code(e: std::errc::invalid_argument));
348 if (!Stopped)
349 return make_error<StringError>(
350 Args: "Can't find stop pass \"" + Info.StopPass + "\".",
351 Args: std::make_error_code(e: std::errc::invalid_argument));
352 return Error::success();
353}
354
355void CodeGenPassBuilder::addISelPasses(PassManagerWrapper &PMW) {
356 addGlobalMergePass(PMW);
357 if (TM.useEmulatedTLS())
358 addModulePass(Pass: LowerEmuTLSPass(), PMW);
359
360 // ObjCARCContract operates on ObjC intrinsics and must run before
361 // PreISelIntrinsicLowering.
362 if (getOptLevel() != CodeGenOptLevel::None) {
363 addFunctionPass(Pass: ObjCARCContractPass(), PMW);
364 flushFPMsToMPM(PMW);
365 }
366 addModulePass(Pass: PreISelIntrinsicLoweringPass(&TM), PMW);
367 addFunctionPass(Pass: ExpandIRInstsPass(TM, getOptLevel()), PMW);
368
369 addIRPasses(PMW);
370 addCodeGenPrepare(PMW);
371 addPassesToHandleExceptions(PMW);
372 addISelPrepare(PMW);
373}
374
375/// Add common target configurable passes that perform LLVM IR to IR transforms
376/// following machine independent optimization.
377void CodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
378 // Before running any passes, run the verifier to determine if the input
379 // coming from the front-end and/or optimizer is valid.
380 if (!Opt.DisableVerify)
381 addFunctionPass(Pass: VerifierPass(), PMW, /*Force=*/true);
382
383 // Run loop strength reduction before anything else.
384 if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableLSR) {
385 // These passes do not use MSSA.
386 LoopPassManager LPM;
387 LPM.addPass(Pass: CanonicalizeFreezeInLoopsPass());
388 LPM.addPass(Pass: LoopStrengthReducePass());
389 if (Opt.EnableLoopTermFold)
390 LPM.addPass(Pass: LoopTermFoldPass());
391 addFunctionPass(Pass: createFunctionToLoopPassAdaptor(LPM: std::move(LPM),
392 /*UseMemorySSA=*/false),
393 PMW);
394 }
395
396 // Run GC lowering passes for builtin collectors
397 // TODO: add a pass insertion point here
398 addFunctionPass(Pass: GCLoweringPass(), PMW);
399 // Explicitly check to see if we should add ShadowStackGCLowering to avoid
400 // splitting the function pipeline if we do not have to.
401 if (runBeforeAdding(Name: ShadowStackGCLoweringPass::name())) {
402 flushFPMsToMPM(PMW);
403 addModulePass(Pass: ShadowStackGCLoweringPass(), PMW);
404 }
405
406 // Make sure that no unreachable blocks are instruction selected.
407 addFunctionPass(Pass: UnreachableBlockElimPass(), PMW);
408
409 // Prepare expensive constants for SelectionDAG.
410 if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableConstantHoisting)
411 addFunctionPass(Pass: ConstantHoistingPass(), PMW);
412
413 // Replace calls to LLVM intrinsics (e.g., exp, log) operating on vector
414 // operands with calls to the corresponding functions in a vector library.
415 if (getOptLevel() != CodeGenOptLevel::None)
416 addFunctionPass(Pass: ReplaceWithVeclib(), PMW);
417
418 if (getOptLevel() != CodeGenOptLevel::None &&
419 !Opt.DisablePartialLibcallInlining)
420 addFunctionPass(Pass: PartiallyInlineLibCallsPass(), PMW);
421
422 // Instrument function entry and exit, e.g. with calls to mcount().
423 addFunctionPass(Pass: EntryExitInstrumenterPass(/*PostInlining=*/true), PMW);
424
425 // Add scalarization of target's unsupported masked memory intrinsics pass.
426 // the unsupported intrinsic will be replaced with a chain of basic blocks,
427 // that stores/loads element one-by-one if the appropriate mask bit is set.
428 addFunctionPass(Pass: ScalarizeMaskedMemIntrinPass(), PMW);
429
430 // Expand reduction intrinsics into shuffle sequences if the target wants to.
431 if (!Opt.DisableExpandReductions)
432 addFunctionPass(Pass: ExpandReductionsPass(), PMW);
433
434 // Convert conditional moves to conditional jumps when profitable.
435 if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableSelectOptimize)
436 addFunctionPass(Pass: SelectOptimizePass(TM), PMW);
437
438 if (Opt.EnableGlobalMergeFunc) {
439 flushFPMsToMPM(PMW);
440 addModulePass(Pass: GlobalMergeFuncPass(), PMW);
441 }
442}
443
444/// Turn exception handling constructs into something the code generators can
445/// handle.
446void CodeGenPassBuilder::addPassesToHandleExceptions(PassManagerWrapper &PMW) {
447 const MCAsmInfo &MCAI = TM.getMCAsmInfo();
448 switch (MCAI.getExceptionHandlingType()) {
449 case ExceptionHandling::SjLj:
450 // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both
451 // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise,
452 // catch info can get misplaced when a selector ends up more than one block
453 // removed from the parent invoke(s). This could happen when a landing
454 // pad is shared by multiple invokes and is also a target of a normal
455 // edge from elsewhere.
456 addFunctionPass(Pass: SjLjEHPreparePass(&TM), PMW);
457 [[fallthrough]];
458 case ExceptionHandling::DwarfCFI:
459 case ExceptionHandling::ARM:
460 case ExceptionHandling::AIX:
461 case ExceptionHandling::ZOS:
462 addFunctionPass(Pass: DwarfEHPreparePass(TM), PMW);
463 break;
464 case ExceptionHandling::WinEH:
465 // We support using both GCC-style and MSVC-style exceptions on Windows, so
466 // add both preparation passes. Each pass will only actually run if it
467 // recognizes the personality function.
468 addFunctionPass(Pass: WinEHPreparePass(), PMW);
469 addFunctionPass(Pass: DwarfEHPreparePass(TM), PMW);
470 break;
471 case ExceptionHandling::Wasm:
472 // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs
473 // on catchpads and cleanuppads because it does not outline them into
474 // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we
475 // should remove PHIs there.
476 addFunctionPass(Pass: WinEHPreparePass(), PMW);
477 break;
478 case ExceptionHandling::Default:
479 case ExceptionHandling::None:
480 case ExceptionHandling::Emscripten:
481 // Emscripten EH is lowered earlier by WebAssemblyLowerEmscriptenEHSjLj, so
482 // by this point it needs no generic EH preparation, like the None case.
483 addFunctionPass(Pass: LowerInvokePass(), PMW);
484
485 // The lower invoke pass may create unreachable code. Remove it.
486 addFunctionPass(Pass: UnreachableBlockElimPass(), PMW);
487 break;
488 }
489}
490
491/// Add pass to prepare the LLVM IR for code generation. This should be done
492/// before exception handling preparation passes.
493void CodeGenPassBuilder::addCodeGenPrepare(PassManagerWrapper &PMW) {
494 if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableCGP)
495 addFunctionPass(Pass: CodeGenPreparePass(TM), PMW);
496 // TODO: Default ctor'd RewriteSymbolPass is no-op.
497 // addPass(RewriteSymbolPass());
498}
499
500/// Add common passes that perform LLVM IR to IR transforms in preparation for
501/// instruction selection.
502void CodeGenPassBuilder::addISelPrepare(PassManagerWrapper &PMW) {
503 addPreISel(PMW);
504
505 if (Opt.RequiresCodeGenSCCOrder && !AddInCGSCCOrder)
506 requireCGSCCOrder(PMW);
507
508 addFunctionPass(Pass: InlineAsmPreparePass(), PMW);
509 // Add both the safe stack and the stack protection passes: each of them will
510 // only protect functions that have corresponding attributes.
511 addFunctionPass(Pass: SafeStackPass(TM), PMW);
512 addFunctionPass(Pass: StackProtectorPass(TM), PMW);
513
514 if (Opt.PrintISelInput)
515 addFunctionPass(Pass: PrintFunctionPass(
516 dbgs(), "\n\n*** Final LLVM Code input to ISel ***\n"),
517 PMW);
518
519 // All passes which modify the LLVM IR are now complete; run the verifier
520 // to ensure that the IR is valid.
521 if (!Opt.DisableVerify)
522 addFunctionPass(Pass: VerifierPass(), PMW, /*Force=*/true);
523}
524
525Error CodeGenPassBuilder::addCoreISelPasses(PassManagerWrapper &PMW) {
526 // Enable FastISel with -fast-isel, but allow that to be overridden.
527 TM.setO0WantsFastISel(Opt.EnableFastISelOption !=
528 cl::boolOrDefault::BOU_FALSE);
529
530 // Determine an instruction selector.
531 enum class SelectorType { SelectionDAG, FastISel, GlobalISel };
532 SelectorType Selector;
533
534 if (Opt.EnableFastISelOption == cl::boolOrDefault::BOU_TRUE)
535 Selector = SelectorType::FastISel;
536 else if (Opt.EnableGlobalISelOption == cl::boolOrDefault::BOU_TRUE ||
537 (TM.Options.EnableGlobalISel &&
538 Opt.EnableGlobalISelOption != cl::boolOrDefault::BOU_FALSE))
539 Selector = SelectorType::GlobalISel;
540 else if (TM.getOptLevel() == CodeGenOptLevel::None && TM.getO0WantsFastISel())
541 Selector = SelectorType::FastISel;
542 else
543 Selector = SelectorType::SelectionDAG;
544
545 // Set consistently TM.Options.EnableFastISel and EnableGlobalISel.
546 if (Selector == SelectorType::FastISel) {
547 TM.setFastISel(true);
548 TM.setGlobalISel(false);
549 } else if (Selector == SelectorType::GlobalISel) {
550 TM.setFastISel(false);
551 TM.setGlobalISel(true);
552 }
553
554 // Add instruction selector passes.
555 if (Selector == SelectorType::GlobalISel) {
556 if (auto Err = addIRTranslator(PMW))
557 return Err;
558
559 addPreLegalizeMachineIR(PMW);
560
561 if (auto Err = addLegalizeMachineIR(PMW))
562 return Err;
563
564 // Before running the register bank selector, ask the target if it
565 // wants to run some passes.
566 addPreRegBankSelect(PMW);
567
568 if (auto Err = addRegBankSelect(PMW))
569 return Err;
570
571 addPreGlobalInstructionSelect(PMW);
572
573 if (auto Err = addGlobalInstructionSelect(PMW))
574 return Err;
575
576 // Pass to reset the MachineFunction if the ISel failed.
577 addMachineFunctionPass(
578 Pass: ResetMachineFunctionPass(reportDiagnosticWhenGlobalISelFallback(),
579 isGlobalISelAbortEnabled()),
580 PMW);
581
582 // Provide a fallback path when we do not want to abort on
583 // not-yet-supported input.
584 if (!isGlobalISelAbortEnabled())
585 if (auto Err = addInstSelector(PMW))
586 return Err;
587
588 } else if (auto Err = addInstSelector(PMW))
589 return Err;
590
591 // Expand pseudo-instructions emitted by ISel. Don't run the verifier before
592 // FinalizeISel.
593 addMachineFunctionPass(Pass: FinalizeISelPass(), PMW);
594
595 // // Print the instruction selected machine code...
596 // printAndVerify("After Instruction Selection");
597
598 return Error::success();
599}
600
601/// Add the complete set of target-independent postISel code generator passes.
602///
603/// This can be read as the standard order of major LLVM CodeGen stages. Stages
604/// with nontrivial configuration or multiple passes are broken out below in
605/// add%Stage routines.
606///
607/// Any CodeGenPassBuilder::addXX routine may be overriden by the Target. The
608/// addPre/Post methods with empty header implementations allow injecting
609/// target-specific fixups just before or after major stages. Additionally,
610/// targets have the flexibility to change pass order within a stage by
611/// overriding default implementation of add%Stage routines below. Each
612/// technique has maintainability tradeoffs because alternate pass orders are
613/// not well supported. addPre/Post works better if the target pass is easily
614/// tied to a common pass. But if it has subtle dependencies on multiple passes,
615/// the target should override the stage instead.
616Error CodeGenPassBuilder::addMachinePasses(PassManagerWrapper &PMW) {
617 // Add passes that optimize machine instructions in SSA form.
618 if (getOptLevel() != CodeGenOptLevel::None) {
619 addMachineSSAOptimization(PMW);
620 } else {
621 // If the target requests it, assign local variables to stack slots relative
622 // to one another and simplify frame index references where possible.
623 addMachineFunctionPass(Pass: LocalStackSlotAllocationPass(), PMW);
624 }
625
626 if (TM.Options.EnableIPRA) {
627 flushFPMsToMPM(PMW);
628 addModulePass(Pass: RequireAnalysisPass<PhysicalRegisterUsageAnalysis, Module>(),
629 PMW, /*Force=*/true);
630 addMachineFunctionPass(Pass: RegUsageInfoPropagationPass(), PMW);
631 }
632 // Run pre-ra passes.
633 addPreRegAlloc(PMW);
634
635 // Run register allocation and passes that are tightly coupled with it,
636 // including phi elimination and scheduling.
637 if (auto Err = Opt.OptimizeRegAlloc == cl::boolOrDefault::BOU_TRUE
638 ? addOptimizedRegAlloc(PMW)
639 : addFastRegAlloc(PMW))
640 return Err;
641
642 // Run post-ra passes.
643 addPostRegAlloc(PMW);
644
645 addMachineFunctionPass(Pass: RemoveRedundantDebugValuesPass(), PMW);
646 addMachineFunctionPass(Pass: FixupStatepointCallerSavedPass(), PMW);
647
648 // Insert prolog/epilog code. Eliminate abstract frame index references...
649 if (getOptLevel() != CodeGenOptLevel::None) {
650 addMachineFunctionPass(Pass: PostRAMachineSinkingPass(), PMW);
651 addMachineFunctionPass(Pass: ShrinkWrapPass(), PMW);
652 }
653
654 addMachineFunctionPass(Pass: PrologEpilogInserterPass(), PMW);
655
656 /// Add passes that optimize machine instructions after register allocation.
657 if (getOptLevel() != CodeGenOptLevel::None)
658 addMachineLateOptimization(PMW);
659
660 // Expand pseudo instructions before second scheduling pass.
661 addMachineFunctionPass(Pass: ExpandPostRAPseudosPass(), PMW);
662
663 // Run pre-sched2 passes.
664 addPreSched2(PMW);
665
666 if (Opt.EnableImplicitNullChecks)
667 addMachineFunctionPass(Pass: ImplicitNullChecksPass(), PMW);
668
669 // Second pass scheduler.
670 // Let Target optionally insert this pass by itself at some other
671 // point.
672 if (getOptLevel() != CodeGenOptLevel::None &&
673 !TM.targetSchedulesPostRAScheduling()) {
674 if (Opt.MISchedPostRA)
675 addMachineFunctionPass(Pass: PostMachineSchedulerPass(&TM), PMW);
676 else
677 addMachineFunctionPass(Pass: PostRASchedulerPass(&TM), PMW);
678 }
679
680 // GC
681 addGCPasses(PMW);
682
683 // Basic block placement.
684 if (getOptLevel() != CodeGenOptLevel::None)
685 addBlockPlacement(PMW);
686
687 // Insert before XRay Instrumentation.
688 addMachineFunctionPass(Pass: FEntryInserterPass(), PMW);
689
690 addMachineFunctionPass(Pass: XRayInstrumentationPass(), PMW);
691 addMachineFunctionPass(Pass: PatchableFunctionPass(), PMW);
692
693 addPreEmitPass(PMW);
694
695 if (TM.Options.EnableIPRA) {
696 // Collect register usage information and produce a register mask of
697 // clobbered registers, to be used to optimize call sites.
698 addMachineFunctionPass(Pass: RegUsageInfoCollectorPass(), PMW);
699 // If -print-regusage is specified, print the collected register usage info.
700 if (Opt.PrintRegUsage) {
701 flushFPMsToMPM(PMW);
702 addModulePass(Pass: PhysicalRegisterUsageInfoPrinterPass(errs()), PMW);
703 }
704 }
705
706 addMachineFunctionPass(Pass: FuncletLayoutPass(), PMW);
707
708 addMachineFunctionPass(Pass: RemoveLoadsIntoFakeUsesPass(), PMW);
709 addMachineFunctionPass(Pass: StackMapLivenessPass(), PMW);
710 addMachineFunctionPass(Pass: LiveDebugValuesPass(TM.shouldEmitDebugEntryValues()),
711 PMW);
712 addMachineFunctionPass(Pass: MachineSanitizerBinaryMetadataPass(), PMW);
713
714 if (TM.Options.EnableMachineOutliner &&
715 getOptLevel() != CodeGenOptLevel::None &&
716 Opt.EnableMachineOutliner != RunOutliner::NeverOutline) {
717 if (Opt.EnableMachineOutliner != RunOutliner::TargetDefault ||
718 TM.supportsDefaultOutlining()) {
719 flushFPMsToMPM(PMW);
720 addModulePass(Pass: MachineOutlinerPass(Opt.EnableMachineOutliner), PMW);
721 }
722 }
723
724 if (Opt.EnableGCEmptyBlocks)
725 addMachineFunctionPass(Pass: GCEmptyBasicBlocksPass(), PMW);
726
727 addPostBBSections(PMW);
728
729 addMachineFunctionPass(Pass: StackFrameLayoutAnalysisPass(), PMW);
730
731 // Add passes that directly emit MI after all other MI passes.
732 addPreEmitPass2(PMW);
733
734 return Error::success();
735}
736
737/// Add passes that optimize machine instructions in SSA form.
738void CodeGenPassBuilder::addMachineSSAOptimization(PassManagerWrapper &PMW) {
739 // Pre-ra tail duplication.
740 addMachineFunctionPass(Pass: EarlyTailDuplicatePass(), PMW);
741
742 // Optimize PHIs before DCE: removing dead PHI cycles may make more
743 // instructions dead.
744 addMachineFunctionPass(Pass: OptimizePHIsPass(), PMW);
745
746 // This pass merges large allocas. StackSlotColoring is a different pass
747 // which merges spill slots.
748 addMachineFunctionPass(Pass: StackColoringPass(), PMW);
749
750 // If the target requests it, assign local variables to stack slots relative
751 // to one another and simplify frame index references where possible.
752 addMachineFunctionPass(Pass: LocalStackSlotAllocationPass(), PMW);
753
754 // With optimization, dead code should already be eliminated. However
755 // there is one known exception: lowered code for arguments that are only
756 // used by tail calls, where the tail calls reuse the incoming stack
757 // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
758 addMachineFunctionPass(Pass: DeadMachineInstructionElimPass(), PMW);
759
760 // Allow targets to insert passes that improve instruction level parallelism,
761 // like if-conversion. Such passes will typically need dominator trees and
762 // loop info, just like LICM and CSE below.
763 addILPOpts(PMW);
764
765 addMachineFunctionPass(Pass: EarlyMachineLICMPass(), PMW);
766 addMachineFunctionPass(Pass: MachineCSEPass(), PMW);
767
768 addMachineFunctionPass(Pass: MachineSinkingPass(Opt.EnableSinkAndFold), PMW);
769
770 addMachineFunctionPass(Pass: PeepholeOptimizerPass(), PMW);
771 // Clean-up the dead code that may have been generated by peephole
772 // rewriting.
773 addMachineFunctionPass(Pass: DeadMachineInstructionElimPass(), PMW);
774}
775
776//===---------------------------------------------------------------------===//
777/// Register Allocation Pass Configuration
778//===---------------------------------------------------------------------===//
779
780/// Instantiate the default register allocator pass for this target for either
781/// the optimized or unoptimized allocation path. This will be added to the pass
782/// manager by addFastRegAlloc in the unoptimized case or addOptimizedRegAlloc
783/// in the optimized case.
784///
785/// A target that uses the standard regalloc pass order for fast or optimized
786/// allocation may still override this for per-target regalloc
787/// selection. But -regalloc-npm=... always takes precedence.
788/// If a target does not want to allow users to set -regalloc-npm=... at all,
789/// check if Opt.RegAlloc == RegAllocType::Unset.
790void CodeGenPassBuilder::addTargetRegisterAllocator(PassManagerWrapper &PMW,
791 bool Optimized) {
792 if (Optimized)
793 addMachineFunctionPass(Pass: RAGreedyPass(), PMW);
794 else
795 addMachineFunctionPass(Pass: RegAllocFastPass(), PMW);
796}
797
798/// Find and instantiate the register allocation pass requested by this target
799/// at the current optimization level. Different register allocators are
800/// defined as separate passes because they may require different analysis.
801///
802/// This helper ensures that the -regalloc-npm= option is always available,
803/// even for targets that override the default allocator.
804void CodeGenPassBuilder::addRegAllocPass(PassManagerWrapper &PMW,
805 bool Optimized) {
806 // Use the specified -regalloc-npm={basic|greedy|fast|pbqp}
807 if (Opt.RegAlloc > RegAllocType::Default) {
808 switch (Opt.RegAlloc) {
809 case RegAllocType::Fast:
810 addMachineFunctionPass(Pass: RegAllocFastPass(), PMW);
811 break;
812 case RegAllocType::Greedy:
813 addMachineFunctionPass(Pass: RAGreedyPass(), PMW);
814 break;
815 default:
816 reportFatalUsageError(reason: "register allocator not supported yet");
817 }
818 return;
819 }
820 // -regalloc=default or unspecified, so pick based on the optimization level
821 // or ask the target for the regalloc pass.
822 addTargetRegisterAllocator(PMW, Optimized);
823}
824
825Error CodeGenPassBuilder::addRegAssignAndRewriteFast(PassManagerWrapper &PMW) {
826 // TODO: Ensure allocator is default or fast.
827 addRegAllocPass(PMW, Optimized: false);
828 return Error::success();
829}
830
831Expected<bool>
832CodeGenPassBuilder::addRegAssignAndRewriteOptimized(PassManagerWrapper &PMW) {
833 // Add the selected register allocation pass.
834 addRegAllocPass(PMW, Optimized: true);
835
836 // Allow targets to change the register assignments before rewriting.
837 addPreRewrite(PMW);
838
839 // Finally rewrite virtual registers.
840 addMachineFunctionPass(Pass: VirtRegRewriterPass(), PMW);
841
842 return true;
843}
844
845/// Add the minimum set of target-independent passes that are required for
846/// register allocation. No coalescing or scheduling.
847Error CodeGenPassBuilder::addFastRegAlloc(PassManagerWrapper &PMW) {
848 addMachineFunctionPass(Pass: PHIEliminationPass(), PMW);
849 if (!TM.enableTiedFastRegAlloc())
850 addMachineFunctionPass(Pass: TwoAddressInstructionPass(), PMW);
851 return addRegAssignAndRewriteFast(PMW);
852}
853
854/// Add standard target-independent passes that are tightly coupled with
855/// optimized register allocation, including coalescing, machine instruction
856/// scheduling, and register allocation itself.
857Error CodeGenPassBuilder::addOptimizedRegAlloc(PassManagerWrapper &PMW) {
858 addMachineFunctionPass(Pass: DetectDeadLanesPass(), PMW);
859
860 addMachineFunctionPass(Pass: InitUndefPass(), PMW);
861
862 addMachineFunctionPass(Pass: ProcessImplicitDefsPass(), PMW);
863
864 // LiveVariables currently requires pure SSA form.
865 //
866 // FIXME: Once TwoAddressInstruction pass no longer uses kill flags,
867 // LiveVariables can be removed completely, and LiveIntervals can be directly
868 // computed. (We still either need to regenerate kill flags after regalloc, or
869 // preferably fix the scavenger to not depend on them).
870 // FIXME: UnreachableMachineBlockElim is a dependant pass of LiveVariables.
871 // When LiveVariables is removed this has to be removed/moved either.
872 // Explicit addition of UnreachableMachineBlockElim allows stopping before or
873 // after it with -stop-before/-stop-after.
874 addMachineFunctionPass(Pass: UnreachableMachineBlockElimPass(), PMW);
875 addMachineFunctionPass(
876 Pass: RequireAnalysisPass<LiveVariablesAnalysis, MachineFunction>(), PMW);
877
878 // LiveIntervals is computed unconditionally before TwoAddressInstruction so
879 // that pass can rely on it instead of LiveVariables. This is a step toward
880 // removing LiveVariables entirely.
881 addMachineFunctionPass(
882 Pass: RequireAnalysisPass<LiveIntervalsAnalysis, MachineFunction>(), PMW);
883
884 // Edge splitting is smarter with machine loop info.
885 addMachineFunctionPass(
886 Pass: RequireAnalysisPass<MachineLoopAnalysis, MachineFunction>(), PMW);
887 addMachineFunctionPass(Pass: PHIEliminationPass(), PMW);
888
889 addMachineFunctionPass(Pass: TwoAddressInstructionPass(), PMW);
890 addMachineFunctionPass(Pass: RegisterCoalescerPass(), PMW);
891
892 // The machine scheduler may accidentally create disconnected components
893 // when moving subregister definitions around, avoid this by splitting them to
894 // separate vregs before. Splitting can also improve reg. allocation quality.
895 addMachineFunctionPass(Pass: RenameIndependentSubregsPass(), PMW);
896
897 // PreRA instruction scheduling.
898 addMachineFunctionPass(Pass: MachineSchedulerPass(&TM), PMW);
899
900 Expected<bool> AddedPasses = addRegAssignAndRewriteOptimized(PMW);
901 if (!AddedPasses)
902 return AddedPasses.takeError();
903 if (!AddedPasses.get())
904 return Error::success();
905
906 addMachineFunctionPass(Pass: StackSlotColoringPass(), PMW);
907
908 // Allow targets to expand pseudo instructions depending on the choice of
909 // registers before MachineCopyPropagation.
910 addPostRewrite(PMW);
911
912 // Copy propagate to forward register uses and try to eliminate COPYs that
913 // were not coalesced.
914 addMachineFunctionPass(Pass: MachineCopyPropagationPass(), PMW);
915
916 // Run post-ra machine LICM to hoist reloads / remats.
917 //
918 // FIXME: can this move into MachineLateOptimization?
919 addMachineFunctionPass(Pass: MachineLICMPass(), PMW);
920
921 return Error::success();
922}
923
924//===---------------------------------------------------------------------===//
925/// Post RegAlloc Pass Configuration
926//===---------------------------------------------------------------------===//
927
928/// Add passes that optimize machine instructions after register allocation.
929void CodeGenPassBuilder::addMachineLateOptimization(PassManagerWrapper &PMW) {
930 // Cleanup of redundant (identical) address/immediate loads.
931 addMachineFunctionPass(Pass: MachineLateInstrsCleanupPass(), PMW);
932
933 // Branch folding must be run after regalloc and prolog/epilog insertion.
934 addMachineFunctionPass(Pass: BranchFolderPass(Opt.EnableTailMerge), PMW);
935
936 // Tail duplication.
937 // Note that duplicating tail just increases code size and degrades
938 // performance for targets that require Structured Control Flow.
939 // In addition it can also make CFG irreducible. Thus we disable it.
940 if (!TM.requiresStructuredCFG())
941 addMachineFunctionPass(Pass: TailDuplicatePass(), PMW);
942
943 // Copy propagation.
944 addMachineFunctionPass(Pass: MachineCopyPropagationPass(), PMW);
945}
946
947/// Add standard basic block placement passes.
948void CodeGenPassBuilder::addBlockPlacement(PassManagerWrapper &PMW) {
949 addMachineFunctionPass(Pass: MachineBlockPlacementPass(Opt.EnableTailMerge), PMW);
950 // Run a separate pass to collect block placement statistics.
951 if (Opt.EnableBlockPlacementStats)
952 addMachineFunctionPass(Pass: MachineBlockPlacementStatsPass(), PMW);
953}
954