1//===--- CodeGenPassBuilder.cpp --------------------------------------- ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines interfaces to access the target independent code
10// generation passes provided by the LLVM backend.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Passes/CodeGenPassBuilder.h"
15#include "llvm/ADT/SmallVector.h"
16#include "llvm/ADT/StringRef.h"
17#include "llvm/Analysis/CGSCCPassManager.h"
18#include "llvm/Analysis/ProfileSummaryInfo.h"
19#include "llvm/Analysis/RuntimeLibcallInfo.h"
20#include "llvm/CodeGen/AsmPrinter.h"
21#include "llvm/CodeGen/AsmPrinterAnalysis.h"
22#include "llvm/CodeGen/BranchFoldingPass.h"
23#include "llvm/CodeGen/CodeGenPrepare.h"
24#include "llvm/CodeGen/DeadMachineInstructionElim.h"
25#include "llvm/CodeGen/DetectDeadLanes.h"
26#include "llvm/CodeGen/DwarfEHPrepare.h"
27#include "llvm/CodeGen/ExpandIRInsts.h"
28#include "llvm/CodeGen/ExpandPostRAPseudos.h"
29#include "llvm/CodeGen/ExpandReductions.h"
30#include "llvm/CodeGen/FEntryInserter.h"
31#include "llvm/CodeGen/FinalizeISel.h"
32#include "llvm/CodeGen/FixupStatepointCallerSaved.h"
33#include "llvm/CodeGen/FuncletLayout.h"
34#include "llvm/CodeGen/GCEmptyBasicBlocks.h"
35#include "llvm/CodeGen/GCMetadata.h"
36#include "llvm/CodeGen/GlobalMerge.h"
37#include "llvm/CodeGen/GlobalMergeFunctions.h"
38#include "llvm/CodeGen/ImplicitNullChecks.h"
39#include "llvm/CodeGen/IndirectBrExpand.h"
40#include "llvm/CodeGen/InitUndef.h"
41#include "llvm/CodeGen/InlineAsmPrepare.h"
42#include "llvm/CodeGen/InterleavedAccess.h"
43#include "llvm/CodeGen/InterleavedLoadCombine.h"
44#include "llvm/CodeGen/LiveDebugValuesPass.h"
45#include "llvm/CodeGen/LiveIntervals.h"
46#include "llvm/CodeGen/LiveVariables.h"
47#include "llvm/CodeGen/LocalStackSlotAllocation.h"
48#include "llvm/CodeGen/LowerEmuTLS.h"
49#include "llvm/CodeGen/MIRPrinter.h"
50#include "llvm/CodeGen/MachineBlockPlacement.h"
51#include "llvm/CodeGen/MachineCSE.h"
52#include "llvm/CodeGen/MachineCopyPropagation.h"
53#include "llvm/CodeGen/MachineFunctionAnalysis.h"
54#include "llvm/CodeGen/MachineLICM.h"
55#include "llvm/CodeGen/MachineLateInstrsCleanup.h"
56#include "llvm/CodeGen/MachineModuleInfo.h"
57#include "llvm/CodeGen/MachinePassManager.h"
58#include "llvm/CodeGen/MachineScheduler.h"
59#include "llvm/CodeGen/MachineSink.h"
60#include "llvm/CodeGen/MachineVerifier.h"
61#include "llvm/CodeGen/OptimizePHIs.h"
62#include "llvm/CodeGen/PEI.h"
63#include "llvm/CodeGen/PHIElimination.h"
64#include "llvm/CodeGen/PatchableFunction.h"
65#include "llvm/CodeGen/PeepholeOptimizer.h"
66#include "llvm/CodeGen/PostRAMachineSink.h"
67#include "llvm/CodeGen/PostRASchedulerList.h"
68#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
69#include "llvm/CodeGen/ProcessImplicitDefs.h"
70#include "llvm/CodeGen/RegAllocEvictionAdvisor.h"
71#include "llvm/CodeGen/RegAllocFast.h"
72#include "llvm/CodeGen/RegAllocGreedyPass.h"
73#include "llvm/CodeGen/RegUsageInfoCollector.h"
74#include "llvm/CodeGen/RegUsageInfoPropagate.h"
75#include "llvm/CodeGen/RegisterCoalescerPass.h"
76#include "llvm/CodeGen/RegisterUsageInfo.h"
77#include "llvm/CodeGen/RemoveLoadsIntoFakeUses.h"
78#include "llvm/CodeGen/RemoveRedundantDebugValues.h"
79#include "llvm/CodeGen/RenameIndependentSubregs.h"
80#include "llvm/CodeGen/ReplaceWithVeclib.h"
81#include "llvm/CodeGen/ResetMachineFunctionPass.h"
82#include "llvm/CodeGen/SafeStack.h"
83#include "llvm/CodeGen/SanitizerBinaryMetadata.h"
84#include "llvm/CodeGen/SelectOptimize.h"
85#include "llvm/CodeGen/ShadowStackGCLowering.h"
86#include "llvm/CodeGen/ShrinkWrap.h"
87#include "llvm/CodeGen/SjLjEHPrepare.h"
88#include "llvm/CodeGen/StackColoring.h"
89#include "llvm/CodeGen/StackFrameLayoutAnalysisPass.h"
90#include "llvm/CodeGen/StackProtector.h"
91#include "llvm/CodeGen/StackSlotColoring.h"
92#include "llvm/CodeGen/TailDuplication.h"
93#include "llvm/CodeGen/TargetPassConfig.h"
94#include "llvm/CodeGen/TwoAddressInstructionPass.h"
95#include "llvm/CodeGen/UnreachableBlockElim.h"
96#include "llvm/CodeGen/VirtRegMap.h"
97#include "llvm/CodeGen/WinEHPrepare.h"
98#include "llvm/CodeGen/XRayInstrumentation.h"
99#include "llvm/IR/PassManager.h"
100#include "llvm/IR/Verifier.h"
101#include "llvm/IRPrinter/IRPrintingPasses.h"
102#include "llvm/MC/MCAsmInfo.h"
103#include "llvm/MC/MCStreamer.h"
104#include "llvm/MC/MCTargetOptions.h"
105#include "llvm/MC/TargetRegistry.h"
106#include "llvm/Passes/TriggerCrashPasses.h"
107#include "llvm/Support/CodeGen.h"
108#include "llvm/Support/Debug.h"
109#include "llvm/Support/Error.h"
110#include "llvm/Support/ErrorHandling.h"
111#include "llvm/Target/CGPassBuilderOption.h"
112#include "llvm/Target/TargetMachine.h"
113#include "llvm/Transforms/ObjCARC.h"
114#include "llvm/Transforms/Scalar/ConstantHoisting.h"
115#include "llvm/Transforms/Scalar/LoopPassManager.h"
116#include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
117#include "llvm/Transforms/Scalar/LoopTermFold.h"
118#include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
119#include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
120#include "llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h"
121#include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
122#include "llvm/Transforms/Utils/LowerInvoke.h"
123#include <cassert>
124#include <utility>
125
126using namespace llvm;
127
128namespace llvm {
129#define DUMMY_MACHINE_FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
130 AnalysisKey PASS_NAME::Key;
131#include "llvm/Passes/MachinePassRegistry.def"
132} // namespace llvm
133
134CodeGenPassBuilder::CodeGenPassBuilder(TargetMachine &TM,
135 const CGPassBuilderOption &Opts,
136 PassInstrumentationCallbacks *PIC)
137 : TM(TM), Opt(Opts), PIC(PIC) {
138 // Target could set CGPassBuilderOption::MISchedPostRA to true to achieve
139 // substitutePass(&PostRASchedulerID, &PostMachineSchedulerID)
140
141 // Target should override TM.Options.EnableIPRA in their target-specific
142 // LLVMTM ctor. See TargetMachine::setGlobalISel for example.
143 if (Opt.EnableIPRA) {
144 TM.Options.EnableIPRA = *Opt.EnableIPRA;
145 } else {
146 // If not explicitly specified, use target default.
147 TM.Options.EnableIPRA |= TM.useIPRA();
148 }
149
150 if (Opt.EnableGlobalISelAbort)
151 TM.Options.GlobalISelAbort = *Opt.EnableGlobalISelAbort;
152
153 if (Opt.EnableRegAllocFastTied != cl::boolOrDefault::BOU_UNSET)
154 TM.setEnableTiedFastRegAlloc(Opt.EnableRegAllocFastTied ==
155 cl::boolOrDefault::BOU_TRUE);
156
157 // An explicit RegAlloc choice implies its pipeline: only the fast
158 // allocator uses the unoptimized one.
159 if (Opt.OptimizeRegAlloc == cl::boolOrDefault::BOU_UNSET) {
160 bool Optimized = Opt.RegAlloc > RegAllocType::Default
161 ? Opt.RegAlloc != RegAllocType::Fast
162 : getOptLevel() != CodeGenOptLevel::None;
163 Opt.OptimizeRegAlloc =
164 Optimized ? cl::boolOrDefault::BOU_TRUE : cl::boolOrDefault::BOU_FALSE;
165 }
166}
167
168// Out-of-line to anchor the vtable in this translation unit.
169CodeGenPassBuilder::~CodeGenPassBuilder() = default;
170
171Error CodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) {
172 return make_error<StringError>(Args: "addInstSelector is not overridden",
173 Args: inconvertibleErrorCode());
174}
175
176Error CodeGenPassBuilder::addIRTranslator(PassManagerWrapper &PMW) {
177 return make_error<StringError>(Args: "addIRTranslator is not overridden",
178 Args: inconvertibleErrorCode());
179}
180
181Error CodeGenPassBuilder::addLegalizeMachineIR(PassManagerWrapper &PMW) {
182 return make_error<StringError>(Args: "addLegalizeMachineIR is not overridden",
183 Args: inconvertibleErrorCode());
184}
185
186Error CodeGenPassBuilder::addRegBankSelect(PassManagerWrapper &PMW) {
187 return make_error<StringError>(Args: "addRegBankSelect is not overridden",
188 Args: inconvertibleErrorCode());
189}
190
191Error CodeGenPassBuilder::addGlobalInstructionSelect(PassManagerWrapper &PMW) {
192 return make_error<StringError>(Args: "addGlobalInstructionSelect is not overridden",
193 Args: inconvertibleErrorCode());
194}
195
196void CodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) {
197 llvm_unreachable("addAsmPrinterBegin is not overriden");
198}
199
200void CodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
201 llvm_unreachable("addAsmPrinter is not overridden");
202}
203
204void CodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
205 llvm_unreachable("addAsmPrinterEnd is not overriden");
206}
207
208void CodeGenPassBuilder::flushFPMsToMPM(PassManagerWrapper &PMW,
209 bool FreeMachineFunctions) {
210 if (PMW.FPM.isEmpty() && PMW.MFPM.isEmpty())
211 return;
212 if (!PMW.MFPM.isEmpty()) {
213 PMW.FPM.addPass(
214 Pass: createFunctionToMachineFunctionPassAdaptor(Pass: std::move(PMW.MFPM)));
215 PMW.MFPM = MachineFunctionPassManager();
216 }
217 if (FreeMachineFunctions)
218 PMW.FPM.addPass(Pass: FreeMachineFunctionPass());
219 if (AddInCGSCCOrder) {
220 PMW.MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(
221 Pass: createCGSCCToFunctionPassAdaptor(Pass: std::move(PMW.FPM))));
222 } else {
223 PMW.MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(PMW.FPM)));
224 }
225 PMW.FPM = FunctionPassManager();
226}
227
228Error CodeGenPassBuilder::buildPipeline(
229 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
230 raw_pwrite_stream *DwoOut, CodeGenFileType FileType, MCContext &Ctx) {
231 auto StartStopInfo = TargetPassConfig::getStartStopInfo(TM, PIC&: *PIC);
232 if (!StartStopInfo)
233 return StartStopInfo.takeError();
234 setStartStopPasses(*StartStopInfo);
235
236 CompletesPipeline = StartStopInfo->StopPass.empty();
237 bool PrintAsm = CompletesPipeline;
238 bool PrintMIR = !PrintAsm && FileType != CodeGenFileType::Null;
239
240 PassManagerWrapper PMW(MPM);
241
242 addModulePass(Pass: RequireAnalysisPass<MachineModuleAnalysis, Module>(), PMW,
243 /*Force=*/true);
244 addModulePass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>(), PMW,
245 /*Force=*/true);
246 addModulePass(Pass: RequireAnalysisPass<CollectorMetadataAnalysis, Module>(), PMW,
247 /*Force=*/true);
248 addModulePass(Pass: RequireAnalysisPass<RuntimeLibraryAnalysis, Module>(), PMW,
249 /*Force=*/true);
250 addModulePass(Pass: RequireAnalysisPass<LibcallLoweringModuleAnalysis, Module>(),
251 PMW,
252 /*Force=*/true);
253 addISelPasses(PMW);
254 flushFPMsToMPM(PMW);
255
256 if (PrintAsm) {
257 Expected<std::unique_ptr<MCStreamer>> MCStreamerOrErr =
258 TM.createMCStreamer(Out, DwoOut, FileType, Ctx);
259 if (!MCStreamerOrErr)
260 return MCStreamerOrErr.takeError();
261 std::unique_ptr<AsmPrinter> Printer(
262 TM.getTarget().createAsmPrinter(TM, Streamer: std::move(*MCStreamerOrErr)));
263 if (!Printer)
264 return createStringError(Fmt: "failed to create AsmPrinter");
265 MAM.registerPass(PassBuilder: [&] { return AsmPrinterAnalysis(std::move(Printer)); });
266 addAsmPrinterBegin(PMW);
267 }
268
269 if (PrintMIR)
270 addModulePass(Pass: PrintMIRPreparePass(Out), PMW, /*Force=*/true);
271
272 if (auto Err = addCoreISelPasses(PMW))
273 return Err;
274
275 if (auto Err = addMachinePasses(PMW))
276 return Err;
277
278 if (PrintAsm ? !Opt.DisableVerify && TM.Options.EnableDefaultMachineVerifier
279 : Opt.VerifyMachineCode == MachineVerifierMode::End)
280 addMachineFunctionPass(Pass: MachineVerifierPass(), PMW, /*Force=*/!PrintAsm);
281
282 // We add AsmPrinter regardless if we are emitting MIR or Assembly as the
283 // final output so that -stop-before=<target>-asm-printer works. When printing
284 // MIR as the final output, we never end up running AsmPrinter.
285 addAsmPrinter(PMW);
286
287 if (PrintAsm) {
288 flushFPMsToMPM(PMW, /*FreeMachineFunctions=*/true);
289 addAsmPrinterEnd(PMW);
290 } else {
291 if (PrintMIR)
292 addMachineFunctionPass(Pass: PrintMIRPass(Out), PMW, /*Force=*/true);
293 flushFPMsToMPM(PMW, /*FreeMachineFunctions=*/true);
294 }
295
296 return verifyStartStop(Info: *StartStopInfo);
297}
298
299void CodeGenPassBuilder::setStartStopPasses(
300 const TargetPassConfig::StartStopInfo &Info) {
301 if (!Info.StartPass.empty()) {
302 Started = false;
303 BeforeCallbacks.emplace_back(Args: [this, &Info, AfterFlag = Info.StartAfter,
304 Count = 0u](StringRef ClassName) mutable {
305 if (Count == Info.StartInstanceNum) {
306 if (AfterFlag) {
307 AfterFlag = false;
308 Started = true;
309 }
310 return Started;
311 }
312
313 auto PassName = PIC->getPassNameForClassName(ClassName);
314 if (Info.StartPass == PassName && ++Count == Info.StartInstanceNum)
315 Started = !Info.StartAfter;
316
317 return Started;
318 });
319 }
320
321 if (!Info.StopPass.empty()) {
322 Stopped = false;
323 BeforeCallbacks.emplace_back(Args: [this, &Info, AfterFlag = Info.StopAfter,
324 Count = 0u](StringRef ClassName) mutable {
325 if (Count == Info.StopInstanceNum) {
326 if (AfterFlag) {
327 AfterFlag = false;
328 Stopped = true;
329 }
330 return !Stopped;
331 }
332
333 auto PassName = PIC->getPassNameForClassName(ClassName);
334 if (Info.StopPass == PassName && ++Count == Info.StopInstanceNum)
335 Stopped = !Info.StopAfter;
336 return !Stopped;
337 });
338 }
339}
340
341Error CodeGenPassBuilder::verifyStartStop(
342 const TargetPassConfig::StartStopInfo &Info) const {
343 if (Started && Stopped)
344 return Error::success();
345
346 if (!Started)
347 return make_error<StringError>(
348 Args: "Can't find start pass \"" + Info.StartPass + "\".",
349 Args: std::make_error_code(e: std::errc::invalid_argument));
350 if (!Stopped)
351 return make_error<StringError>(
352 Args: "Can't find stop pass \"" + Info.StopPass + "\".",
353 Args: std::make_error_code(e: std::errc::invalid_argument));
354 return Error::success();
355}
356
357void CodeGenPassBuilder::addISelPasses(PassManagerWrapper &PMW) {
358 addGlobalMergePass(PMW);
359 if (TM.useEmulatedTLS())
360 addModulePass(Pass: LowerEmuTLSPass(), PMW);
361
362 // ObjCARCContract operates on ObjC intrinsics and must run before
363 // PreISelIntrinsicLowering.
364 if (getOptLevel() != CodeGenOptLevel::None) {
365 addFunctionPass(Pass: ObjCARCContractPass(), PMW);
366 flushFPMsToMPM(PMW);
367 }
368 addModulePass(Pass: PreISelIntrinsicLoweringPass(&TM), PMW);
369 addFunctionPass(Pass: ExpandIRInstsPass(TM, getOptLevel()), PMW);
370
371 addIRPasses(PMW);
372 addCodeGenPrepare(PMW);
373 addPassesToHandleExceptions(PMW);
374 addISelPrepare(PMW);
375}
376
377/// Add common target configurable passes that perform LLVM IR to IR transforms
378/// following machine independent optimization.
379void CodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
380 // Before running any passes, run the verifier to determine if the input
381 // coming from the front-end and/or optimizer is valid.
382 if (!Opt.DisableVerify)
383 addFunctionPass(Pass: VerifierPass(), PMW, /*Force=*/true);
384
385 // Run loop strength reduction before anything else.
386 if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableLSR) {
387 // These passes do not use MSSA.
388 LoopPassManager LPM;
389 LPM.addPass(Pass: CanonicalizeFreezeInLoopsPass());
390 LPM.addPass(Pass: LoopStrengthReducePass());
391 if (Opt.EnableLoopTermFold)
392 LPM.addPass(Pass: LoopTermFoldPass());
393 addFunctionPass(Pass: createFunctionToLoopPassAdaptor(LPM: std::move(LPM),
394 /*UseMemorySSA=*/false),
395 PMW);
396 }
397
398 // Run GC lowering passes for builtin collectors
399 // TODO: add a pass insertion point here
400 addFunctionPass(Pass: GCLoweringPass(), PMW);
401 // Explicitly check to see if we should add ShadowStackGCLowering to avoid
402 // splitting the function pipeline if we do not have to.
403 if (runBeforeAdding(Name: ShadowStackGCLoweringPass::name())) {
404 flushFPMsToMPM(PMW);
405 addModulePass(Pass: ShadowStackGCLoweringPass(), PMW);
406 }
407
408 // Make sure that no unreachable blocks are instruction selected.
409 addFunctionPass(Pass: UnreachableBlockElimPass(), PMW);
410
411 // Prepare expensive constants for SelectionDAG.
412 if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableConstantHoisting)
413 addFunctionPass(Pass: ConstantHoistingPass(), PMW);
414
415 // Replace calls to LLVM intrinsics (e.g., exp, log) operating on vector
416 // operands with calls to the corresponding functions in a vector library.
417 if (getOptLevel() != CodeGenOptLevel::None)
418 addFunctionPass(Pass: ReplaceWithVeclib(), PMW);
419
420 if (getOptLevel() != CodeGenOptLevel::None &&
421 !Opt.DisablePartialLibcallInlining)
422 addFunctionPass(Pass: PartiallyInlineLibCallsPass(), PMW);
423
424 // Instrument function entry and exit, e.g. with calls to mcount().
425 addFunctionPass(Pass: EntryExitInstrumenterPass(/*PostInlining=*/true), PMW);
426
427 // Add scalarization of target's unsupported masked memory intrinsics pass.
428 // the unsupported intrinsic will be replaced with a chain of basic blocks,
429 // that stores/loads element one-by-one if the appropriate mask bit is set.
430 addFunctionPass(Pass: ScalarizeMaskedMemIntrinPass(), PMW);
431
432 // Expand reduction intrinsics into shuffle sequences if the target wants to.
433 if (!Opt.DisableExpandReductions)
434 addFunctionPass(Pass: ExpandReductionsPass(), PMW);
435
436 // Convert conditional moves to conditional jumps when profitable.
437 if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableSelectOptimize)
438 addFunctionPass(Pass: SelectOptimizePass(TM), PMW);
439
440 if (Opt.EnableGlobalMergeFunc) {
441 flushFPMsToMPM(PMW);
442 addModulePass(Pass: GlobalMergeFuncPass(), PMW);
443 }
444}
445
446/// Turn exception handling constructs into something the code generators can
447/// handle.
448void CodeGenPassBuilder::addPassesToHandleExceptions(PassManagerWrapper &PMW) {
449 const MCAsmInfo &MCAI = TM.getMCAsmInfo();
450 switch (MCAI.getExceptionHandlingType()) {
451 case ExceptionHandling::SjLj:
452 // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both
453 // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise,
454 // catch info can get misplaced when a selector ends up more than one block
455 // removed from the parent invoke(s). This could happen when a landing
456 // pad is shared by multiple invokes and is also a target of a normal
457 // edge from elsewhere.
458 addFunctionPass(Pass: SjLjEHPreparePass(&TM), PMW);
459 [[fallthrough]];
460 case ExceptionHandling::DwarfCFI:
461 case ExceptionHandling::ARM:
462 case ExceptionHandling::AIX:
463 case ExceptionHandling::ZOS:
464 addFunctionPass(Pass: DwarfEHPreparePass(TM), PMW);
465 break;
466 case ExceptionHandling::WinEH:
467 // We support using both GCC-style and MSVC-style exceptions on Windows, so
468 // add both preparation passes. Each pass will only actually run if it
469 // recognizes the personality function.
470 addFunctionPass(Pass: WinEHPreparePass(), PMW);
471 addFunctionPass(Pass: DwarfEHPreparePass(TM), PMW);
472 break;
473 case ExceptionHandling::Wasm:
474 // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs
475 // on catchpads and cleanuppads because it does not outline them into
476 // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we
477 // should remove PHIs there.
478 addFunctionPass(Pass: WinEHPreparePass(), PMW);
479 break;
480 case ExceptionHandling::Default:
481 case ExceptionHandling::None:
482 case ExceptionHandling::Emscripten:
483 // Emscripten EH is lowered earlier by WebAssemblyLowerEmscriptenEHSjLj, so
484 // by this point it needs no generic EH preparation, like the None case.
485 addFunctionPass(Pass: LowerInvokePass(), PMW);
486
487 // The lower invoke pass may create unreachable code. Remove it.
488 addFunctionPass(Pass: UnreachableBlockElimPass(), PMW);
489 break;
490 }
491}
492
493/// Add pass to prepare the LLVM IR for code generation. This should be done
494/// before exception handling preparation passes.
495void CodeGenPassBuilder::addCodeGenPrepare(PassManagerWrapper &PMW) {
496 if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableCGP)
497 addFunctionPass(Pass: CodeGenPreparePass(TM), PMW);
498 // TODO: Default ctor'd RewriteSymbolPass is no-op.
499 // addPass(RewriteSymbolPass());
500}
501
502/// Add common passes that perform LLVM IR to IR transforms in preparation for
503/// instruction selection.
504void CodeGenPassBuilder::addISelPrepare(PassManagerWrapper &PMW) {
505 addPreISel(PMW);
506
507 if (Opt.RequiresCodeGenSCCOrder && !AddInCGSCCOrder)
508 requireCGSCCOrder(PMW);
509
510 addFunctionPass(Pass: InlineAsmPreparePass(), PMW);
511 // Add both the safe stack and the stack protection passes: each of them will
512 // only protect functions that have corresponding attributes.
513 addFunctionPass(Pass: SafeStackPass(TM), PMW);
514 addFunctionPass(Pass: StackProtectorPass(TM), PMW);
515
516 if (Opt.PrintISelInput)
517 addFunctionPass(Pass: PrintFunctionPass(
518 dbgs(), "\n\n*** Final LLVM Code input to ISel ***\n"),
519 PMW);
520
521 // All passes which modify the LLVM IR are now complete; run the verifier
522 // to ensure that the IR is valid.
523 if (!Opt.DisableVerify)
524 addFunctionPass(Pass: VerifierPass(), PMW, /*Force=*/true);
525}
526
527Error CodeGenPassBuilder::addCoreISelPasses(PassManagerWrapper &PMW) {
528 // Enable FastISel with -fast-isel, but allow that to be overridden.
529 TM.setO0WantsFastISel(Opt.EnableFastISelOption !=
530 cl::boolOrDefault::BOU_FALSE);
531
532 // Determine an instruction selector.
533 enum class SelectorType { SelectionDAG, FastISel, GlobalISel };
534 SelectorType Selector;
535
536 if (Opt.EnableFastISelOption == cl::boolOrDefault::BOU_TRUE)
537 Selector = SelectorType::FastISel;
538 else if (Opt.EnableGlobalISelOption == cl::boolOrDefault::BOU_TRUE ||
539 (TM.Options.EnableGlobalISel &&
540 Opt.EnableGlobalISelOption != cl::boolOrDefault::BOU_FALSE))
541 Selector = SelectorType::GlobalISel;
542 else if (TM.getOptLevel() == CodeGenOptLevel::None && TM.getO0WantsFastISel())
543 Selector = SelectorType::FastISel;
544 else
545 Selector = SelectorType::SelectionDAG;
546
547 // Set consistently TM.Options.EnableFastISel and EnableGlobalISel.
548 if (Selector == SelectorType::FastISel) {
549 TM.setFastISel(true);
550 TM.setGlobalISel(false);
551 } else if (Selector == SelectorType::GlobalISel) {
552 TM.setFastISel(false);
553 TM.setGlobalISel(true);
554 }
555
556 // Add instruction selector passes.
557 if (Selector == SelectorType::GlobalISel) {
558 if (auto Err = addIRTranslator(PMW))
559 return Err;
560
561 addPreLegalizeMachineIR(PMW);
562
563 if (auto Err = addLegalizeMachineIR(PMW))
564 return Err;
565
566 // Before running the register bank selector, ask the target if it
567 // wants to run some passes.
568 addPreRegBankSelect(PMW);
569
570 if (auto Err = addRegBankSelect(PMW))
571 return Err;
572
573 addPreGlobalInstructionSelect(PMW);
574
575 if (auto Err = addGlobalInstructionSelect(PMW))
576 return Err;
577
578 // Pass to reset the MachineFunction if the ISel failed.
579 addMachineFunctionPass(
580 Pass: ResetMachineFunctionPass(reportDiagnosticWhenGlobalISelFallback(),
581 isGlobalISelAbortEnabled()),
582 PMW);
583
584 // Provide a fallback path when we do not want to abort on
585 // not-yet-supported input.
586 if (!isGlobalISelAbortEnabled())
587 if (auto Err = addInstSelector(PMW))
588 return Err;
589
590 } else if (auto Err = addInstSelector(PMW))
591 return Err;
592
593 // Expand pseudo-instructions emitted by ISel. Don't run the verifier before
594 // FinalizeISel.
595 addMachineFunctionPass(Pass: FinalizeISelPass(), PMW);
596
597 // // Print the instruction selected machine code...
598 // printAndVerify("After Instruction Selection");
599
600 return Error::success();
601}
602
603/// Add the complete set of target-independent postISel code generator passes.
604///
605/// This can be read as the standard order of major LLVM CodeGen stages. Stages
606/// with nontrivial configuration or multiple passes are broken out below in
607/// add%Stage routines.
608///
609/// Any CodeGenPassBuilder::addXX routine may be overriden by the Target. The
610/// addPre/Post methods with empty header implementations allow injecting
611/// target-specific fixups just before or after major stages. Additionally,
612/// targets have the flexibility to change pass order within a stage by
613/// overriding default implementation of add%Stage routines below. Each
614/// technique has maintainability tradeoffs because alternate pass orders are
615/// not well supported. addPre/Post works better if the target pass is easily
616/// tied to a common pass. But if it has subtle dependencies on multiple passes,
617/// the target should override the stage instead.
618Error CodeGenPassBuilder::addMachinePasses(PassManagerWrapper &PMW) {
619 // Add passes that optimize machine instructions in SSA form.
620 if (getOptLevel() != CodeGenOptLevel::None) {
621 addMachineSSAOptimization(PMW);
622 } else {
623 // If the target requests it, assign local variables to stack slots relative
624 // to one another and simplify frame index references where possible.
625 addMachineFunctionPass(Pass: LocalStackSlotAllocationPass(), PMW);
626 }
627
628 if (TM.Options.EnableIPRA) {
629 flushFPMsToMPM(PMW);
630 addModulePass(Pass: RequireAnalysisPass<PhysicalRegisterUsageAnalysis, Module>(),
631 PMW, /*Force=*/true);
632 addMachineFunctionPass(Pass: RegUsageInfoPropagationPass(), PMW);
633 }
634 // Run pre-ra passes.
635 addPreRegAlloc(PMW);
636
637 // Run register allocation and passes that are tightly coupled with it,
638 // including phi elimination and scheduling.
639 if (auto Err = Opt.OptimizeRegAlloc == cl::boolOrDefault::BOU_TRUE
640 ? addOptimizedRegAlloc(PMW)
641 : addFastRegAlloc(PMW))
642 return Err;
643
644 // Run post-ra passes.
645 addPostRegAlloc(PMW);
646
647 addMachineFunctionPass(Pass: RemoveRedundantDebugValuesPass(), PMW);
648 addMachineFunctionPass(Pass: FixupStatepointCallerSavedPass(), PMW);
649
650 // Insert prolog/epilog code. Eliminate abstract frame index references...
651 if (getOptLevel() != CodeGenOptLevel::None) {
652 addMachineFunctionPass(Pass: PostRAMachineSinkingPass(), PMW);
653 addMachineFunctionPass(Pass: ShrinkWrapPass(), PMW);
654 }
655
656 addMachineFunctionPass(Pass: PrologEpilogInserterPass(), PMW);
657
658 /// Add passes that optimize machine instructions after register allocation.
659 if (getOptLevel() != CodeGenOptLevel::None)
660 addMachineLateOptimization(PMW);
661
662 // Expand pseudo instructions before second scheduling pass.
663 addMachineFunctionPass(Pass: ExpandPostRAPseudosPass(), PMW);
664
665 // Run pre-sched2 passes.
666 addPreSched2(PMW);
667
668 if (Opt.EnableImplicitNullChecks)
669 addMachineFunctionPass(Pass: ImplicitNullChecksPass(), PMW);
670
671 // Second pass scheduler.
672 // Let Target optionally insert this pass by itself at some other
673 // point.
674 if (getOptLevel() != CodeGenOptLevel::None &&
675 !TM.targetSchedulesPostRAScheduling()) {
676 if (Opt.MISchedPostRA)
677 addMachineFunctionPass(Pass: PostMachineSchedulerPass(&TM), PMW);
678 else
679 addMachineFunctionPass(Pass: PostRASchedulerPass(&TM), PMW);
680 }
681
682 // GC
683 addGCPasses(PMW);
684
685 // Basic block placement.
686 if (getOptLevel() != CodeGenOptLevel::None)
687 addBlockPlacement(PMW);
688
689 // Insert before XRay Instrumentation.
690 addMachineFunctionPass(Pass: FEntryInserterPass(), PMW);
691
692 addMachineFunctionPass(Pass: XRayInstrumentationPass(), PMW);
693 addMachineFunctionPass(Pass: PatchableFunctionPass(), PMW);
694
695 addPreEmitPass(PMW);
696
697 if (TM.Options.EnableIPRA) {
698 // Collect register usage information and produce a register mask of
699 // clobbered registers, to be used to optimize call sites.
700 addMachineFunctionPass(Pass: RegUsageInfoCollectorPass(), PMW);
701 // If -print-regusage is specified, print the collected register usage info.
702 if (Opt.PrintRegUsage) {
703 flushFPMsToMPM(PMW);
704 addModulePass(Pass: PhysicalRegisterUsageInfoPrinterPass(errs()), PMW);
705 }
706 }
707
708 addMachineFunctionPass(Pass: FuncletLayoutPass(), PMW);
709
710 addMachineFunctionPass(Pass: RemoveLoadsIntoFakeUsesPass(), PMW);
711 addMachineFunctionPass(Pass: StackMapLivenessPass(), PMW);
712 addMachineFunctionPass(Pass: LiveDebugValuesPass(TM.shouldEmitDebugEntryValues()),
713 PMW);
714 addMachineFunctionPass(Pass: MachineSanitizerBinaryMetadataPass(), PMW);
715
716 if (TM.Options.EnableMachineOutliner &&
717 getOptLevel() != CodeGenOptLevel::None &&
718 Opt.EnableMachineOutliner != RunOutliner::NeverOutline) {
719 if (Opt.EnableMachineOutliner != RunOutliner::TargetDefault ||
720 TM.supportsDefaultOutlining()) {
721 flushFPMsToMPM(PMW);
722 addModulePass(Pass: MachineOutlinerPass(Opt.EnableMachineOutliner), PMW);
723 }
724 }
725
726 if (Opt.EnableGCEmptyBlocks)
727 addMachineFunctionPass(Pass: GCEmptyBasicBlocksPass(), PMW);
728
729 addPostBBSections(PMW);
730
731 addMachineFunctionPass(Pass: StackFrameLayoutAnalysisPass(), PMW);
732
733 // Add passes that directly emit MI after all other MI passes.
734 addPreEmitPass2(PMW);
735
736 return Error::success();
737}
738
739/// Add passes that optimize machine instructions in SSA form.
740void CodeGenPassBuilder::addMachineSSAOptimization(PassManagerWrapper &PMW) {
741 // Pre-ra tail duplication.
742 addMachineFunctionPass(Pass: EarlyTailDuplicatePass(), PMW);
743
744 // Optimize PHIs before DCE: removing dead PHI cycles may make more
745 // instructions dead.
746 addMachineFunctionPass(Pass: OptimizePHIsPass(), PMW);
747
748 // This pass merges large allocas. StackSlotColoring is a different pass
749 // which merges spill slots.
750 addMachineFunctionPass(Pass: StackColoringPass(), PMW);
751
752 // If the target requests it, assign local variables to stack slots relative
753 // to one another and simplify frame index references where possible.
754 addMachineFunctionPass(Pass: LocalStackSlotAllocationPass(), PMW);
755
756 // With optimization, dead code should already be eliminated. However
757 // there is one known exception: lowered code for arguments that are only
758 // used by tail calls, where the tail calls reuse the incoming stack
759 // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
760 addMachineFunctionPass(Pass: DeadMachineInstructionElimPass(), PMW);
761
762 // Allow targets to insert passes that improve instruction level parallelism,
763 // like if-conversion. Such passes will typically need dominator trees and
764 // loop info, just like LICM and CSE below.
765 addILPOpts(PMW);
766
767 addMachineFunctionPass(Pass: EarlyMachineLICMPass(), PMW);
768 addMachineFunctionPass(Pass: MachineCSEPass(), PMW);
769
770 addMachineFunctionPass(Pass: MachineSinkingPass(Opt.EnableSinkAndFold), PMW);
771
772 addMachineFunctionPass(Pass: PeepholeOptimizerPass(), PMW);
773 // Clean-up the dead code that may have been generated by peephole
774 // rewriting.
775 addMachineFunctionPass(Pass: DeadMachineInstructionElimPass(), PMW);
776}
777
778//===---------------------------------------------------------------------===//
779/// Register Allocation Pass Configuration
780//===---------------------------------------------------------------------===//
781
782/// Instantiate the default register allocator pass for this target for either
783/// the optimized or unoptimized allocation path. This will be added to the pass
784/// manager by addFastRegAlloc in the unoptimized case or addOptimizedRegAlloc
785/// in the optimized case.
786///
787/// A target that uses the standard regalloc pass order for fast or optimized
788/// allocation may still override this for per-target regalloc
789/// selection. But -regalloc-npm=... always takes precedence.
790/// If a target does not want to allow users to set -regalloc-npm=... at all,
791/// check if Opt.RegAlloc == RegAllocType::Unset.
792void CodeGenPassBuilder::addTargetRegisterAllocator(PassManagerWrapper &PMW,
793 bool Optimized) {
794 if (Optimized)
795 addMachineFunctionPass(Pass: RAGreedyPass(), PMW);
796 else
797 addMachineFunctionPass(Pass: RegAllocFastPass(), PMW);
798}
799
800/// Find and instantiate the register allocation pass requested by this target
801/// at the current optimization level. Different register allocators are
802/// defined as separate passes because they may require different analysis.
803///
804/// This helper ensures that the -regalloc-npm= option is always available,
805/// even for targets that override the default allocator.
806void CodeGenPassBuilder::addRegAllocPass(PassManagerWrapper &PMW,
807 bool Optimized) {
808 // Use the specified -regalloc-npm={basic|greedy|fast|pbqp}
809 if (Opt.RegAlloc > RegAllocType::Default) {
810 switch (Opt.RegAlloc) {
811 case RegAllocType::Fast:
812 addMachineFunctionPass(Pass: RegAllocFastPass(), PMW);
813 break;
814 case RegAllocType::Greedy:
815 addMachineFunctionPass(Pass: RAGreedyPass(), PMW);
816 break;
817 default:
818 reportFatalUsageError(reason: "register allocator not supported yet");
819 }
820 return;
821 }
822 // -regalloc=default or unspecified, so pick based on the optimization level
823 // or ask the target for the regalloc pass.
824 addTargetRegisterAllocator(PMW, Optimized);
825}
826
827Error CodeGenPassBuilder::addRegAssignAndRewriteFast(PassManagerWrapper &PMW) {
828 // TODO: Ensure allocator is default or fast.
829 addRegAllocPass(PMW, Optimized: false);
830 return Error::success();
831}
832
833Expected<bool>
834CodeGenPassBuilder::addRegAssignAndRewriteOptimized(PassManagerWrapper &PMW) {
835 // Add the selected register allocation pass.
836 addRegAllocPass(PMW, Optimized: true);
837
838 // Allow targets to change the register assignments before rewriting.
839 addPreRewrite(PMW);
840
841 // Finally rewrite virtual registers.
842 addMachineFunctionPass(Pass: VirtRegRewriterPass(), PMW);
843
844 return true;
845}
846
847/// Add the minimum set of target-independent passes that are required for
848/// register allocation. No coalescing or scheduling.
849Error CodeGenPassBuilder::addFastRegAlloc(PassManagerWrapper &PMW) {
850 addMachineFunctionPass(Pass: PHIEliminationPass(), PMW);
851 if (!TM.enableTiedFastRegAlloc())
852 addMachineFunctionPass(Pass: TwoAddressInstructionPass(), PMW);
853 return addRegAssignAndRewriteFast(PMW);
854}
855
856/// Add standard target-independent passes that are tightly coupled with
857/// optimized register allocation, including coalescing, machine instruction
858/// scheduling, and register allocation itself.
859Error CodeGenPassBuilder::addOptimizedRegAlloc(PassManagerWrapper &PMW) {
860 addMachineFunctionPass(Pass: DetectDeadLanesPass(), PMW);
861
862 addMachineFunctionPass(Pass: InitUndefPass(), PMW);
863
864 addMachineFunctionPass(Pass: ProcessImplicitDefsPass(), PMW);
865
866 // LiveVariables currently requires pure SSA form.
867 //
868 // FIXME: Once TwoAddressInstruction pass no longer uses kill flags,
869 // LiveVariables can be removed completely, and LiveIntervals can be directly
870 // computed. (We still either need to regenerate kill flags after regalloc, or
871 // preferably fix the scavenger to not depend on them).
872 // FIXME: UnreachableMachineBlockElim is a dependant pass of LiveVariables.
873 // When LiveVariables is removed this has to be removed/moved either.
874 // Explicit addition of UnreachableMachineBlockElim allows stopping before or
875 // after it with -stop-before/-stop-after.
876 addMachineFunctionPass(Pass: UnreachableMachineBlockElimPass(), PMW);
877 addMachineFunctionPass(
878 Pass: RequireAnalysisPass<LiveVariablesAnalysis, MachineFunction>(), PMW);
879
880 // If enabled, run the SSA machine scheduler just before PHI elimination.
881 if (Opt.EnableSSAMachineScheduler) {
882 addMachineFunctionPass(
883 Pass: RequireAnalysisPass<LiveIntervalsAnalysis, MachineFunction>(), PMW);
884 addMachineFunctionPass(Pass: SSAMachineSchedulerPass(&TM), PMW);
885 }
886
887 // LiveIntervals is computed unconditionally before TwoAddressInstruction so
888 // that pass can rely on it instead of LiveVariables. This is a step toward
889 // removing LiveVariables entirely.
890 addMachineFunctionPass(
891 Pass: RequireAnalysisPass<LiveIntervalsAnalysis, MachineFunction>(), PMW);
892
893 // Edge splitting is smarter with machine loop info.
894 addMachineFunctionPass(
895 Pass: RequireAnalysisPass<MachineLoopAnalysis, MachineFunction>(), PMW);
896 addMachineFunctionPass(Pass: PHIEliminationPass(), PMW);
897
898 addMachineFunctionPass(Pass: TwoAddressInstructionPass(), PMW);
899 addMachineFunctionPass(Pass: RegisterCoalescerPass(), PMW);
900
901 // The machine scheduler may accidentally create disconnected components
902 // when moving subregister definitions around, avoid this by splitting them to
903 // separate vregs before. Splitting can also improve reg. allocation quality.
904 addMachineFunctionPass(Pass: RenameIndependentSubregsPass(), PMW);
905
906 // PreRA instruction scheduling.
907 addMachineFunctionPass(Pass: MachineSchedulerPass(&TM), PMW);
908
909 Expected<bool> AddedPasses = addRegAssignAndRewriteOptimized(PMW);
910 if (!AddedPasses)
911 return AddedPasses.takeError();
912 if (!AddedPasses.get())
913 return Error::success();
914
915 addMachineFunctionPass(Pass: StackSlotColoringPass(), PMW);
916
917 // Allow targets to expand pseudo instructions depending on the choice of
918 // registers before MachineCopyPropagation.
919 addPostRewrite(PMW);
920
921 // Copy propagate to forward register uses and try to eliminate COPYs that
922 // were not coalesced.
923 addMachineFunctionPass(Pass: MachineCopyPropagationPass(), PMW);
924
925 // Run post-ra machine LICM to hoist reloads / remats.
926 //
927 // FIXME: can this move into MachineLateOptimization?
928 addMachineFunctionPass(Pass: MachineLICMPass(), PMW);
929
930 return Error::success();
931}
932
933//===---------------------------------------------------------------------===//
934/// Post RegAlloc Pass Configuration
935//===---------------------------------------------------------------------===//
936
937/// Add passes that optimize machine instructions after register allocation.
938void CodeGenPassBuilder::addMachineLateOptimization(PassManagerWrapper &PMW) {
939 // Cleanup of redundant (identical) address/immediate loads.
940 addMachineFunctionPass(Pass: MachineLateInstrsCleanupPass(), PMW);
941
942 // Branch folding must be run after regalloc and prolog/epilog insertion.
943 addMachineFunctionPass(Pass: BranchFolderPass(Opt.EnableTailMerge), PMW);
944
945 // Tail duplication.
946 // Note that duplicating tail just increases code size and degrades
947 // performance for targets that require Structured Control Flow.
948 // In addition it can also make CFG irreducible. Thus we disable it.
949 if (!TM.requiresStructuredCFG())
950 addMachineFunctionPass(Pass: TailDuplicatePass(), PMW);
951
952 // Copy propagation.
953 addMachineFunctionPass(Pass: MachineCopyPropagationPass(), PMW);
954}
955
956/// Add standard basic block placement passes.
957void CodeGenPassBuilder::addBlockPlacement(PassManagerWrapper &PMW) {
958 addMachineFunctionPass(Pass: MachineBlockPlacementPass(Opt.EnableTailMerge), PMW);
959 // Run a separate pass to collect block placement statistics.
960 if (Opt.EnableBlockPlacementStats)
961 addMachineFunctionPass(Pass: MachineBlockPlacementStatsPass(), PMW);
962}
963