1//===--- CodeGenPassBuilder.cpp --------------------------------------- ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines interfaces to access the target independent code
10// generation passes provided by the LLVM backend.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Passes/CodeGenPassBuilder.h"
15#include "llvm/ADT/SmallVector.h"
16#include "llvm/ADT/StringRef.h"
17#include "llvm/Analysis/BasicAliasAnalysis.h"
18#include "llvm/Analysis/CGSCCPassManager.h"
19#include "llvm/Analysis/ProfileSummaryInfo.h"
20#include "llvm/Analysis/RuntimeLibcallInfo.h"
21#include "llvm/Analysis/ScopedNoAliasAA.h"
22#include "llvm/Analysis/TargetTransformInfo.h"
23#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
24#include "llvm/CodeGen/AsmPrinter.h"
25#include "llvm/CodeGen/AsmPrinterAnalysis.h"
26#include "llvm/CodeGen/BranchFoldingPass.h"
27#include "llvm/CodeGen/CodeGenPrepare.h"
28#include "llvm/CodeGen/DeadMachineInstructionElim.h"
29#include "llvm/CodeGen/DetectDeadLanes.h"
30#include "llvm/CodeGen/DwarfEHPrepare.h"
31#include "llvm/CodeGen/ExpandIRInsts.h"
32#include "llvm/CodeGen/ExpandPostRAPseudos.h"
33#include "llvm/CodeGen/ExpandReductions.h"
34#include "llvm/CodeGen/FEntryInserter.h"
35#include "llvm/CodeGen/FinalizeISel.h"
36#include "llvm/CodeGen/FixupStatepointCallerSaved.h"
37#include "llvm/CodeGen/FuncletLayout.h"
38#include "llvm/CodeGen/GCEmptyBasicBlocks.h"
39#include "llvm/CodeGen/GCMetadata.h"
40#include "llvm/CodeGen/GlobalMerge.h"
41#include "llvm/CodeGen/GlobalMergeFunctions.h"
42#include "llvm/CodeGen/ImplicitNullChecks.h"
43#include "llvm/CodeGen/IndirectBrExpand.h"
44#include "llvm/CodeGen/InitUndef.h"
45#include "llvm/CodeGen/InlineAsmPrepare.h"
46#include "llvm/CodeGen/InterleavedAccess.h"
47#include "llvm/CodeGen/InterleavedLoadCombine.h"
48#include "llvm/CodeGen/LiveDebugValuesPass.h"
49#include "llvm/CodeGen/LiveIntervals.h"
50#include "llvm/CodeGen/LiveVariables.h"
51#include "llvm/CodeGen/LocalStackSlotAllocation.h"
52#include "llvm/CodeGen/LowerEmuTLS.h"
53#include "llvm/CodeGen/MIRPrinter.h"
54#include "llvm/CodeGen/MachineBlockPlacement.h"
55#include "llvm/CodeGen/MachineCSE.h"
56#include "llvm/CodeGen/MachineCopyPropagation.h"
57#include "llvm/CodeGen/MachineFunctionAnalysis.h"
58#include "llvm/CodeGen/MachineLICM.h"
59#include "llvm/CodeGen/MachineLateInstrsCleanup.h"
60#include "llvm/CodeGen/MachineModuleInfo.h"
61#include "llvm/CodeGen/MachinePassManager.h"
62#include "llvm/CodeGen/MachineScheduler.h"
63#include "llvm/CodeGen/MachineSink.h"
64#include "llvm/CodeGen/MachineVerifier.h"
65#include "llvm/CodeGen/OptimizePHIs.h"
66#include "llvm/CodeGen/PEI.h"
67#include "llvm/CodeGen/PHIElimination.h"
68#include "llvm/CodeGen/PatchableFunction.h"
69#include "llvm/CodeGen/PeepholeOptimizer.h"
70#include "llvm/CodeGen/PostRAMachineSink.h"
71#include "llvm/CodeGen/PostRASchedulerList.h"
72#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
73#include "llvm/CodeGen/ProcessImplicitDefs.h"
74#include "llvm/CodeGen/RegAllocEvictionAdvisor.h"
75#include "llvm/CodeGen/RegAllocFast.h"
76#include "llvm/CodeGen/RegAllocGreedyPass.h"
77#include "llvm/CodeGen/RegUsageInfoCollector.h"
78#include "llvm/CodeGen/RegUsageInfoPropagate.h"
79#include "llvm/CodeGen/RegisterCoalescerPass.h"
80#include "llvm/CodeGen/RegisterUsageInfo.h"
81#include "llvm/CodeGen/RemoveLoadsIntoFakeUses.h"
82#include "llvm/CodeGen/RemoveRedundantDebugValues.h"
83#include "llvm/CodeGen/RenameIndependentSubregs.h"
84#include "llvm/CodeGen/ReplaceWithVeclib.h"
85#include "llvm/CodeGen/SafeStack.h"
86#include "llvm/CodeGen/SanitizerBinaryMetadata.h"
87#include "llvm/CodeGen/SelectOptimize.h"
88#include "llvm/CodeGen/ShadowStackGCLowering.h"
89#include "llvm/CodeGen/ShrinkWrap.h"
90#include "llvm/CodeGen/SjLjEHPrepare.h"
91#include "llvm/CodeGen/StackColoring.h"
92#include "llvm/CodeGen/StackFrameLayoutAnalysisPass.h"
93#include "llvm/CodeGen/StackProtector.h"
94#include "llvm/CodeGen/StackSlotColoring.h"
95#include "llvm/CodeGen/TailDuplication.h"
96#include "llvm/CodeGen/TargetPassConfig.h"
97#include "llvm/CodeGen/TwoAddressInstructionPass.h"
98#include "llvm/CodeGen/UnreachableBlockElim.h"
99#include "llvm/CodeGen/VirtRegMap.h"
100#include "llvm/CodeGen/WasmEHPrepare.h"
101#include "llvm/CodeGen/WinEHPrepare.h"
102#include "llvm/CodeGen/XRayInstrumentation.h"
103#include "llvm/IR/PassManager.h"
104#include "llvm/IR/Verifier.h"
105#include "llvm/IRPrinter/IRPrintingPasses.h"
106#include "llvm/MC/MCAsmInfo.h"
107#include "llvm/MC/MCStreamer.h"
108#include "llvm/MC/MCTargetOptions.h"
109#include "llvm/MC/TargetRegistry.h"
110#include "llvm/Support/CodeGen.h"
111#include "llvm/Support/Debug.h"
112#include "llvm/Support/Error.h"
113#include "llvm/Support/ErrorHandling.h"
114#include "llvm/Target/CGPassBuilderOption.h"
115#include "llvm/Target/TargetMachine.h"
116#include "llvm/Transforms/ObjCARC.h"
117#include "llvm/Transforms/Scalar/ConstantHoisting.h"
118#include "llvm/Transforms/Scalar/LoopPassManager.h"
119#include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
120#include "llvm/Transforms/Scalar/LoopTermFold.h"
121#include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
122#include "llvm/Transforms/Scalar/MergeICmps.h"
123#include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
124#include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
125#include "llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h"
126#include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
127#include "llvm/Transforms/Utils/LowerInvoke.h"
128#include <cassert>
129#include <utility>
130
131using namespace llvm;
132
133namespace llvm {
134#define DUMMY_MACHINE_FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
135 AnalysisKey PASS_NAME::Key;
136#include "llvm/Passes/MachinePassRegistry.def"
137} // namespace llvm
138
139CodeGenPassBuilder::CodeGenPassBuilder(TargetMachine &TM,
140 const CGPassBuilderOption &Opts,
141 PassInstrumentationCallbacks *PIC)
142 : TM(TM), Opt(Opts), PIC(PIC) {
143 // Target could set CGPassBuilderOption::MISchedPostRA to true to achieve
144 // substitutePass(&PostRASchedulerID, &PostMachineSchedulerID)
145
146 // Target should override TM.Options.EnableIPRA in their target-specific
147 // LLVMTM ctor. See TargetMachine::setGlobalISel for example.
148 if (Opt.EnableIPRA) {
149 TM.Options.EnableIPRA = *Opt.EnableIPRA;
150 } else {
151 // If not explicitly specified, use target default.
152 TM.Options.EnableIPRA |= TM.useIPRA();
153 }
154
155 if (Opt.EnableGlobalISelAbort)
156 TM.Options.GlobalISelAbort = *Opt.EnableGlobalISelAbort;
157
158 // An explicit RegAlloc choice implies its pipeline: only the fast
159 // allocator uses the unoptimized one.
160 if (Opt.OptimizeRegAlloc == cl::boolOrDefault::BOU_UNSET) {
161 bool Optimized = Opt.RegAlloc > RegAllocType::Default
162 ? Opt.RegAlloc != RegAllocType::Fast
163 : getOptLevel() != CodeGenOptLevel::None;
164 Opt.OptimizeRegAlloc =
165 Optimized ? cl::boolOrDefault::BOU_TRUE : cl::boolOrDefault::BOU_FALSE;
166 }
167}
168
169// Out-of-line to anchor the vtable in this translation unit.
170CodeGenPassBuilder::~CodeGenPassBuilder() = default;
171
172Error CodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) {
173 return make_error<StringError>(Args: "addInstSelector is not overridden",
174 Args: inconvertibleErrorCode());
175}
176
177Error CodeGenPassBuilder::addIRTranslator(PassManagerWrapper &PMW) {
178 return make_error<StringError>(Args: "addIRTranslator is not overridden",
179 Args: inconvertibleErrorCode());
180}
181
182Error CodeGenPassBuilder::addLegalizeMachineIR(PassManagerWrapper &PMW) {
183 return make_error<StringError>(Args: "addLegalizeMachineIR is not overridden",
184 Args: inconvertibleErrorCode());
185}
186
187Error CodeGenPassBuilder::addRegBankSelect(PassManagerWrapper &PMW) {
188 return make_error<StringError>(Args: "addRegBankSelect is not overridden",
189 Args: inconvertibleErrorCode());
190}
191
192Error CodeGenPassBuilder::addGlobalInstructionSelect(PassManagerWrapper &PMW) {
193 return make_error<StringError>(Args: "addGlobalInstructionSelect is not overridden",
194 Args: inconvertibleErrorCode());
195}
196
197void CodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) {
198 llvm_unreachable("addAsmPrinterBegin is not overriden");
199}
200
201void CodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
202 llvm_unreachable("addAsmPrinter is not overridden");
203}
204
205void CodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
206 llvm_unreachable("addAsmPrinterEnd is not overriden");
207}
208
209void CodeGenPassBuilder::flushFPMsToMPM(PassManagerWrapper &PMW,
210 bool FreeMachineFunctions) {
211 if (PMW.FPM.isEmpty() && PMW.MFPM.isEmpty())
212 return;
213 if (!PMW.MFPM.isEmpty()) {
214 PMW.FPM.addPass(
215 Pass: createFunctionToMachineFunctionPassAdaptor(Pass: std::move(PMW.MFPM)));
216 PMW.MFPM = MachineFunctionPassManager();
217 }
218 if (FreeMachineFunctions)
219 PMW.FPM.addPass(Pass: FreeMachineFunctionPass());
220 if (AddInCGSCCOrder) {
221 PMW.MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(
222 Pass: createCGSCCToFunctionPassAdaptor(Pass: std::move(PMW.FPM))));
223 } else {
224 PMW.MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(PMW.FPM)));
225 }
226 PMW.FPM = FunctionPassManager();
227}
228
229Error CodeGenPassBuilder::buildPipeline(
230 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
231 raw_pwrite_stream *DwoOut, CodeGenFileType FileType, MCContext &Ctx) {
232 auto StartStopInfo = TargetPassConfig::getStartStopInfo(PIC&: *PIC);
233 if (!StartStopInfo)
234 return StartStopInfo.takeError();
235 setStartStopPasses(*StartStopInfo);
236
237 bool PrintAsm = TargetPassConfig::willCompleteCodeGenPipeline();
238 bool PrintMIR = !PrintAsm && FileType != CodeGenFileType::Null;
239
240 PassManagerWrapper PMW(MPM);
241
242 addModulePass(Pass: RequireAnalysisPass<MachineModuleAnalysis, Module>(), PMW,
243 /*Force=*/true);
244 addModulePass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>(), PMW,
245 /*Force=*/true);
246 addModulePass(Pass: RequireAnalysisPass<CollectorMetadataAnalysis, Module>(), PMW,
247 /*Force=*/true);
248 addModulePass(Pass: RequireAnalysisPass<RuntimeLibraryAnalysis, Module>(), PMW,
249 /*Force=*/true);
250 addModulePass(Pass: RequireAnalysisPass<LibcallLoweringModuleAnalysis, Module>(),
251 PMW,
252 /*Force=*/true);
253 addISelPasses(PMW);
254 flushFPMsToMPM(PMW);
255
256 if (PrintAsm) {
257 Expected<std::unique_ptr<MCStreamer>> MCStreamerOrErr =
258 TM.createMCStreamer(Out, DwoOut, FileType, Ctx);
259 if (!MCStreamerOrErr)
260 return MCStreamerOrErr.takeError();
261 std::unique_ptr<AsmPrinter> Printer(
262 TM.getTarget().createAsmPrinter(TM, Streamer: std::move(*MCStreamerOrErr)));
263 if (!Printer)
264 return createStringError(Fmt: "failed to create AsmPrinter");
265 MAM.registerPass(PassBuilder: [&] { return AsmPrinterAnalysis(std::move(Printer)); });
266 addAsmPrinterBegin(PMW);
267 }
268
269 if (PrintMIR)
270 addModulePass(Pass: PrintMIRPreparePass(Out), PMW, /*Force=*/true);
271
272 if (auto Err = addCoreISelPasses(PMW))
273 return Err;
274
275 if (auto Err = addMachinePasses(PMW))
276 return Err;
277
278 if (!Opt.DisableVerify && TM.Options.EnableDefaultMachineVerifier)
279 addMachineFunctionPass(Pass: MachineVerifierPass(), PMW);
280
281 // We add AsmPrinter regardless if we are emitting MIR or Assembly as the
282 // final output so that -stop-before=<target>-asm-printer works. When printing
283 // MIR as the final output, we never end up running AsmPrinter.
284 addAsmPrinter(PMW);
285
286 if (PrintAsm) {
287 flushFPMsToMPM(PMW, /*FreeMachineFunctions=*/true);
288 addAsmPrinterEnd(PMW);
289 } else {
290 if (PrintMIR)
291 addMachineFunctionPass(Pass: PrintMIRPass(Out), PMW, /*Force=*/true);
292 flushFPMsToMPM(PMW, /*FreeMachineFunctions=*/true);
293 }
294
295 return verifyStartStop(Info: *StartStopInfo);
296}
297
298void CodeGenPassBuilder::setStartStopPasses(
299 const TargetPassConfig::StartStopInfo &Info) {
300 if (!Info.StartPass.empty()) {
301 Started = false;
302 BeforeCallbacks.emplace_back(Args: [this, &Info, AfterFlag = Info.StartAfter,
303 Count = 0u](StringRef ClassName) mutable {
304 if (Count == Info.StartInstanceNum) {
305 if (AfterFlag) {
306 AfterFlag = false;
307 Started = true;
308 }
309 return Started;
310 }
311
312 auto PassName = PIC->getPassNameForClassName(ClassName);
313 if (Info.StartPass == PassName && ++Count == Info.StartInstanceNum)
314 Started = !Info.StartAfter;
315
316 return Started;
317 });
318 }
319
320 if (!Info.StopPass.empty()) {
321 Stopped = false;
322 BeforeCallbacks.emplace_back(Args: [this, &Info, AfterFlag = Info.StopAfter,
323 Count = 0u](StringRef ClassName) mutable {
324 if (Count == Info.StopInstanceNum) {
325 if (AfterFlag) {
326 AfterFlag = false;
327 Stopped = true;
328 }
329 return !Stopped;
330 }
331
332 auto PassName = PIC->getPassNameForClassName(ClassName);
333 if (Info.StopPass == PassName && ++Count == Info.StopInstanceNum)
334 Stopped = !Info.StopAfter;
335 return !Stopped;
336 });
337 }
338}
339
340Error CodeGenPassBuilder::verifyStartStop(
341 const TargetPassConfig::StartStopInfo &Info) const {
342 if (Started && Stopped)
343 return Error::success();
344
345 if (!Started)
346 return make_error<StringError>(
347 Args: "Can't find start pass \"" + Info.StartPass + "\".",
348 Args: std::make_error_code(e: std::errc::invalid_argument));
349 if (!Stopped)
350 return make_error<StringError>(
351 Args: "Can't find stop pass \"" + Info.StopPass + "\".",
352 Args: std::make_error_code(e: std::errc::invalid_argument));
353 return Error::success();
354}
355
356void CodeGenPassBuilder::addISelPasses(PassManagerWrapper &PMW) {
357 addGlobalMergePass(PMW);
358 if (TM.useEmulatedTLS())
359 addModulePass(Pass: LowerEmuTLSPass(), PMW);
360
361 // ObjCARCContract operates on ObjC intrinsics and must run before
362 // PreISelIntrinsicLowering.
363 if (getOptLevel() != CodeGenOptLevel::None) {
364 addFunctionPass(Pass: ObjCARCContractPass(), PMW);
365 flushFPMsToMPM(PMW);
366 }
367 addModulePass(Pass: PreISelIntrinsicLoweringPass(&TM), PMW);
368 addFunctionPass(Pass: ExpandIRInstsPass(TM, getOptLevel()), PMW);
369
370 addIRPasses(PMW);
371 addCodeGenPrepare(PMW);
372 addPassesToHandleExceptions(PMW);
373 addISelPrepare(PMW);
374}
375
376/// Add common target configurable passes that perform LLVM IR to IR transforms
377/// following machine independent optimization.
378void CodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
379 // Before running any passes, run the verifier to determine if the input
380 // coming from the front-end and/or optimizer is valid.
381 if (!Opt.DisableVerify)
382 addFunctionPass(Pass: VerifierPass(), PMW, /*Force=*/true);
383
384 // Run loop strength reduction before anything else.
385 if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableLSR) {
386 // These passes do not use MSSA.
387 LoopPassManager LPM;
388 LPM.addPass(Pass: CanonicalizeFreezeInLoopsPass());
389 LPM.addPass(Pass: LoopStrengthReducePass());
390 if (Opt.EnableLoopTermFold)
391 LPM.addPass(Pass: LoopTermFoldPass());
392 addFunctionPass(Pass: createFunctionToLoopPassAdaptor(LPM: std::move(LPM),
393 /*UseMemorySSA=*/false),
394 PMW);
395 }
396
397 // Run GC lowering passes for builtin collectors
398 // TODO: add a pass insertion point here
399 addFunctionPass(Pass: GCLoweringPass(), PMW);
400 // Explicitly check to see if we should add ShadowStackGCLowering to avoid
401 // splitting the function pipeline if we do not have to.
402 if (runBeforeAdding(Name: ShadowStackGCLoweringPass::name())) {
403 flushFPMsToMPM(PMW);
404 addModulePass(Pass: ShadowStackGCLoweringPass(), PMW);
405 }
406
407 // Make sure that no unreachable blocks are instruction selected.
408 addFunctionPass(Pass: UnreachableBlockElimPass(), PMW);
409
410 // Prepare expensive constants for SelectionDAG.
411 if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableConstantHoisting)
412 addFunctionPass(Pass: ConstantHoistingPass(), PMW);
413
414 // Replace calls to LLVM intrinsics (e.g., exp, log) operating on vector
415 // operands with calls to the corresponding functions in a vector library.
416 if (getOptLevel() != CodeGenOptLevel::None)
417 addFunctionPass(Pass: ReplaceWithVeclib(), PMW);
418
419 if (getOptLevel() != CodeGenOptLevel::None &&
420 !Opt.DisablePartialLibcallInlining)
421 addFunctionPass(Pass: PartiallyInlineLibCallsPass(), PMW);
422
423 // Instrument function entry and exit, e.g. with calls to mcount().
424 addFunctionPass(Pass: EntryExitInstrumenterPass(/*PostInlining=*/true), PMW);
425
426 // Add scalarization of target's unsupported masked memory intrinsics pass.
427 // the unsupported intrinsic will be replaced with a chain of basic blocks,
428 // that stores/loads element one-by-one if the appropriate mask bit is set.
429 addFunctionPass(Pass: ScalarizeMaskedMemIntrinPass(), PMW);
430
431 // Expand reduction intrinsics into shuffle sequences if the target wants to.
432 if (!Opt.DisableExpandReductions)
433 addFunctionPass(Pass: ExpandReductionsPass(), PMW);
434
435 // Convert conditional moves to conditional jumps when profitable.
436 if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableSelectOptimize)
437 addFunctionPass(Pass: SelectOptimizePass(TM), PMW);
438
439 if (Opt.EnableGlobalMergeFunc) {
440 flushFPMsToMPM(PMW);
441 addModulePass(Pass: GlobalMergeFuncPass(), PMW);
442 }
443}
444
445/// Turn exception handling constructs into something the code generators can
446/// handle.
447void CodeGenPassBuilder::addPassesToHandleExceptions(PassManagerWrapper &PMW) {
448 const MCAsmInfo &MCAI = TM.getMCAsmInfo();
449 switch (MCAI.getExceptionHandlingType()) {
450 case ExceptionHandling::SjLj:
451 // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both
452 // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise,
453 // catch info can get misplaced when a selector ends up more than one block
454 // removed from the parent invoke(s). This could happen when a landing
455 // pad is shared by multiple invokes and is also a target of a normal
456 // edge from elsewhere.
457 addFunctionPass(Pass: SjLjEHPreparePass(&TM), PMW);
458 [[fallthrough]];
459 case ExceptionHandling::DwarfCFI:
460 case ExceptionHandling::ARM:
461 case ExceptionHandling::AIX:
462 case ExceptionHandling::ZOS:
463 addFunctionPass(Pass: DwarfEHPreparePass(TM), PMW);
464 break;
465 case ExceptionHandling::WinEH:
466 // We support using both GCC-style and MSVC-style exceptions on Windows, so
467 // add both preparation passes. Each pass will only actually run if it
468 // recognizes the personality function.
469 addFunctionPass(Pass: WinEHPreparePass(), PMW);
470 addFunctionPass(Pass: DwarfEHPreparePass(TM), PMW);
471 break;
472 case ExceptionHandling::Wasm:
473 // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs
474 // on catchpads and cleanuppads because it does not outline them into
475 // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we
476 // should remove PHIs there.
477 addFunctionPass(Pass: WinEHPreparePass(/*DemoteCatchSwitchPHIOnly=*/false), PMW);
478 addFunctionPass(Pass: WasmEHPreparePass(), PMW);
479 break;
480 case ExceptionHandling::None:
481 addFunctionPass(Pass: LowerInvokePass(), PMW);
482
483 // The lower invoke pass may create unreachable code. Remove it.
484 addFunctionPass(Pass: UnreachableBlockElimPass(), PMW);
485 break;
486 }
487}
488
489/// Add pass to prepare the LLVM IR for code generation. This should be done
490/// before exception handling preparation passes.
491void CodeGenPassBuilder::addCodeGenPrepare(PassManagerWrapper &PMW) {
492 if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableCGP)
493 addFunctionPass(Pass: CodeGenPreparePass(TM), PMW);
494 // TODO: Default ctor'd RewriteSymbolPass is no-op.
495 // addPass(RewriteSymbolPass());
496}
497
498/// Add common passes that perform LLVM IR to IR transforms in preparation for
499/// instruction selection.
500void CodeGenPassBuilder::addISelPrepare(PassManagerWrapper &PMW) {
501 addPreISel(PMW);
502
503 if (Opt.RequiresCodeGenSCCOrder && !AddInCGSCCOrder)
504 requireCGSCCOrder(PMW);
505
506 addFunctionPass(Pass: InlineAsmPreparePass(), PMW);
507 // Add both the safe stack and the stack protection passes: each of them will
508 // only protect functions that have corresponding attributes.
509 addFunctionPass(Pass: SafeStackPass(TM), PMW);
510 addFunctionPass(Pass: StackProtectorPass(TM), PMW);
511
512 if (Opt.PrintISelInput)
513 addFunctionPass(Pass: PrintFunctionPass(
514 dbgs(), "\n\n*** Final LLVM Code input to ISel ***\n"),
515 PMW);
516
517 // All passes which modify the LLVM IR are now complete; run the verifier
518 // to ensure that the IR is valid.
519 if (!Opt.DisableVerify)
520 addFunctionPass(Pass: VerifierPass(), PMW, /*Force=*/true);
521}
522
523Error CodeGenPassBuilder::addCoreISelPasses(PassManagerWrapper &PMW) {
524 // Enable FastISel with -fast-isel, but allow that to be overridden.
525 TM.setO0WantsFastISel(Opt.EnableFastISelOption !=
526 cl::boolOrDefault::BOU_FALSE);
527
528 // Determine an instruction selector.
529 enum class SelectorType { SelectionDAG, FastISel, GlobalISel };
530 SelectorType Selector;
531
532 if (Opt.EnableFastISelOption == cl::boolOrDefault::BOU_TRUE)
533 Selector = SelectorType::FastISel;
534 else if (Opt.EnableGlobalISelOption == cl::boolOrDefault::BOU_TRUE ||
535 (TM.Options.EnableGlobalISel &&
536 Opt.EnableGlobalISelOption != cl::boolOrDefault::BOU_FALSE))
537 Selector = SelectorType::GlobalISel;
538 else if (TM.getOptLevel() == CodeGenOptLevel::None && TM.getO0WantsFastISel())
539 Selector = SelectorType::FastISel;
540 else
541 Selector = SelectorType::SelectionDAG;
542
543 // Set consistently TM.Options.EnableFastISel and EnableGlobalISel.
544 if (Selector == SelectorType::FastISel) {
545 TM.setFastISel(true);
546 TM.setGlobalISel(false);
547 } else if (Selector == SelectorType::GlobalISel) {
548 TM.setFastISel(false);
549 TM.setGlobalISel(true);
550 }
551
552 // Add instruction selector passes.
553 if (Selector == SelectorType::GlobalISel) {
554 if (auto Err = addIRTranslator(PMW))
555 return Err;
556
557 addPreLegalizeMachineIR(PMW);
558
559 if (auto Err = addLegalizeMachineIR(PMW))
560 return Err;
561
562 // Before running the register bank selector, ask the target if it
563 // wants to run some passes.
564 addPreRegBankSelect(PMW);
565
566 if (auto Err = addRegBankSelect(PMW))
567 return Err;
568
569 addPreGlobalInstructionSelect(PMW);
570
571 if (auto Err = addGlobalInstructionSelect(PMW))
572 return Err;
573
574 // Pass to reset the MachineFunction if the ISel failed.
575 addMachineFunctionPass(
576 Pass: ResetMachineFunctionPass(reportDiagnosticWhenGlobalISelFallback(),
577 isGlobalISelAbortEnabled()),
578 PMW);
579
580 // Provide a fallback path when we do not want to abort on
581 // not-yet-supported input.
582 if (!isGlobalISelAbortEnabled())
583 if (auto Err = addInstSelector(PMW))
584 return Err;
585
586 } else if (auto Err = addInstSelector(PMW))
587 return Err;
588
589 // Expand pseudo-instructions emitted by ISel. Don't run the verifier before
590 // FinalizeISel.
591 addMachineFunctionPass(Pass: FinalizeISelPass(), PMW);
592
593 // // Print the instruction selected machine code...
594 // printAndVerify("After Instruction Selection");
595
596 return Error::success();
597}
598
599/// Add the complete set of target-independent postISel code generator passes.
600///
601/// This can be read as the standard order of major LLVM CodeGen stages. Stages
602/// with nontrivial configuration or multiple passes are broken out below in
603/// add%Stage routines.
604///
605/// Any CodeGenPassBuilder::addXX routine may be overriden by the Target. The
606/// addPre/Post methods with empty header implementations allow injecting
607/// target-specific fixups just before or after major stages. Additionally,
608/// targets have the flexibility to change pass order within a stage by
609/// overriding default implementation of add%Stage routines below. Each
610/// technique has maintainability tradeoffs because alternate pass orders are
611/// not well supported. addPre/Post works better if the target pass is easily
612/// tied to a common pass. But if it has subtle dependencies on multiple passes,
613/// the target should override the stage instead.
614Error CodeGenPassBuilder::addMachinePasses(PassManagerWrapper &PMW) {
615 // Add passes that optimize machine instructions in SSA form.
616 if (getOptLevel() != CodeGenOptLevel::None) {
617 addMachineSSAOptimization(PMW);
618 } else {
619 // If the target requests it, assign local variables to stack slots relative
620 // to one another and simplify frame index references where possible.
621 addMachineFunctionPass(Pass: LocalStackSlotAllocationPass(), PMW);
622 }
623
624 if (TM.Options.EnableIPRA) {
625 flushFPMsToMPM(PMW);
626 addModulePass(Pass: RequireAnalysisPass<PhysicalRegisterUsageAnalysis, Module>(),
627 PMW, /*Force=*/true);
628 addMachineFunctionPass(Pass: RegUsageInfoPropagationPass(), PMW);
629 }
630 // Run pre-ra passes.
631 addPreRegAlloc(PMW);
632
633 // Run register allocation and passes that are tightly coupled with it,
634 // including phi elimination and scheduling.
635 if (auto Err = Opt.OptimizeRegAlloc == cl::boolOrDefault::BOU_TRUE
636 ? addOptimizedRegAlloc(PMW)
637 : addFastRegAlloc(PMW))
638 return Err;
639
640 // Run post-ra passes.
641 addPostRegAlloc(PMW);
642
643 addMachineFunctionPass(Pass: RemoveRedundantDebugValuesPass(), PMW);
644 addMachineFunctionPass(Pass: FixupStatepointCallerSavedPass(), PMW);
645
646 // Insert prolog/epilog code. Eliminate abstract frame index references...
647 if (getOptLevel() != CodeGenOptLevel::None) {
648 addMachineFunctionPass(Pass: PostRAMachineSinkingPass(), PMW);
649 addMachineFunctionPass(Pass: ShrinkWrapPass(), PMW);
650 }
651
652 addMachineFunctionPass(Pass: PrologEpilogInserterPass(), PMW);
653
654 /// Add passes that optimize machine instructions after register allocation.
655 if (getOptLevel() != CodeGenOptLevel::None)
656 addMachineLateOptimization(PMW);
657
658 // Expand pseudo instructions before second scheduling pass.
659 addMachineFunctionPass(Pass: ExpandPostRAPseudosPass(), PMW);
660
661 // Run pre-sched2 passes.
662 addPreSched2(PMW);
663
664 if (Opt.EnableImplicitNullChecks)
665 addMachineFunctionPass(Pass: ImplicitNullChecksPass(), PMW);
666
667 // Second pass scheduler.
668 // Let Target optionally insert this pass by itself at some other
669 // point.
670 if (getOptLevel() != CodeGenOptLevel::None &&
671 !TM.targetSchedulesPostRAScheduling()) {
672 if (Opt.MISchedPostRA)
673 addMachineFunctionPass(Pass: PostMachineSchedulerPass(&TM), PMW);
674 else
675 addMachineFunctionPass(Pass: PostRASchedulerPass(&TM), PMW);
676 }
677
678 // GC
679 addGCPasses(PMW);
680
681 // Basic block placement.
682 if (getOptLevel() != CodeGenOptLevel::None)
683 addBlockPlacement(PMW);
684
685 // Insert before XRay Instrumentation.
686 addMachineFunctionPass(Pass: FEntryInserterPass(), PMW);
687
688 addMachineFunctionPass(Pass: XRayInstrumentationPass(), PMW);
689 addMachineFunctionPass(Pass: PatchableFunctionPass(), PMW);
690
691 addPreEmitPass(PMW);
692
693 if (TM.Options.EnableIPRA) {
694 // Collect register usage information and produce a register mask of
695 // clobbered registers, to be used to optimize call sites.
696 addMachineFunctionPass(Pass: RegUsageInfoCollectorPass(), PMW);
697 // If -print-regusage is specified, print the collected register usage info.
698 if (Opt.PrintRegUsage) {
699 flushFPMsToMPM(PMW);
700 addModulePass(Pass: PhysicalRegisterUsageInfoPrinterPass(errs()), PMW);
701 }
702 }
703
704 addMachineFunctionPass(Pass: FuncletLayoutPass(), PMW);
705
706 addMachineFunctionPass(Pass: RemoveLoadsIntoFakeUsesPass(), PMW);
707 addMachineFunctionPass(Pass: StackMapLivenessPass(), PMW);
708 addMachineFunctionPass(
709 Pass: LiveDebugValuesPass(TM.Options.ShouldEmitDebugEntryValues()), PMW);
710 addMachineFunctionPass(Pass: MachineSanitizerBinaryMetadataPass(), PMW);
711
712 if (TM.Options.EnableMachineOutliner &&
713 getOptLevel() != CodeGenOptLevel::None &&
714 Opt.EnableMachineOutliner != RunOutliner::NeverOutline) {
715 if (Opt.EnableMachineOutliner != RunOutliner::TargetDefault ||
716 TM.Options.SupportsDefaultOutlining) {
717 flushFPMsToMPM(PMW);
718 addModulePass(Pass: MachineOutlinerPass(Opt.EnableMachineOutliner), PMW);
719 }
720 }
721
722 if (Opt.EnableGCEmptyBlocks)
723 addMachineFunctionPass(Pass: GCEmptyBasicBlocksPass(), PMW);
724
725 addPostBBSections(PMW);
726
727 addMachineFunctionPass(Pass: StackFrameLayoutAnalysisPass(), PMW);
728
729 // Add passes that directly emit MI after all other MI passes.
730 addPreEmitPass2(PMW);
731
732 return Error::success();
733}
734
735/// Add passes that optimize machine instructions in SSA form.
736void CodeGenPassBuilder::addMachineSSAOptimization(PassManagerWrapper &PMW) {
737 // Pre-ra tail duplication.
738 addMachineFunctionPass(Pass: EarlyTailDuplicatePass(), PMW);
739
740 // Optimize PHIs before DCE: removing dead PHI cycles may make more
741 // instructions dead.
742 addMachineFunctionPass(Pass: OptimizePHIsPass(), PMW);
743
744 // This pass merges large allocas. StackSlotColoring is a different pass
745 // which merges spill slots.
746 addMachineFunctionPass(Pass: StackColoringPass(), PMW);
747
748 // If the target requests it, assign local variables to stack slots relative
749 // to one another and simplify frame index references where possible.
750 addMachineFunctionPass(Pass: LocalStackSlotAllocationPass(), PMW);
751
752 // With optimization, dead code should already be eliminated. However
753 // there is one known exception: lowered code for arguments that are only
754 // used by tail calls, where the tail calls reuse the incoming stack
755 // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
756 addMachineFunctionPass(Pass: DeadMachineInstructionElimPass(), PMW);
757
758 // Allow targets to insert passes that improve instruction level parallelism,
759 // like if-conversion. Such passes will typically need dominator trees and
760 // loop info, just like LICM and CSE below.
761 addILPOpts(PMW);
762
763 addMachineFunctionPass(Pass: EarlyMachineLICMPass(), PMW);
764 addMachineFunctionPass(Pass: MachineCSEPass(), PMW);
765
766 addMachineFunctionPass(Pass: MachineSinkingPass(Opt.EnableSinkAndFold), PMW);
767
768 addMachineFunctionPass(Pass: PeepholeOptimizerPass(), PMW);
769 // Clean-up the dead code that may have been generated by peephole
770 // rewriting.
771 addMachineFunctionPass(Pass: DeadMachineInstructionElimPass(), PMW);
772}
773
774//===---------------------------------------------------------------------===//
775/// Register Allocation Pass Configuration
776//===---------------------------------------------------------------------===//
777
778/// Instantiate the default register allocator pass for this target for either
779/// the optimized or unoptimized allocation path. This will be added to the pass
780/// manager by addFastRegAlloc in the unoptimized case or addOptimizedRegAlloc
781/// in the optimized case.
782///
783/// A target that uses the standard regalloc pass order for fast or optimized
784/// allocation may still override this for per-target regalloc
785/// selection. But -regalloc-npm=... always takes precedence.
786/// If a target does not want to allow users to set -regalloc-npm=... at all,
787/// check if Opt.RegAlloc == RegAllocType::Unset.
788void CodeGenPassBuilder::addTargetRegisterAllocator(PassManagerWrapper &PMW,
789 bool Optimized) {
790 if (Optimized)
791 addMachineFunctionPass(Pass: RAGreedyPass(), PMW);
792 else
793 addMachineFunctionPass(Pass: RegAllocFastPass(), PMW);
794}
795
796/// Find and instantiate the register allocation pass requested by this target
797/// at the current optimization level. Different register allocators are
798/// defined as separate passes because they may require different analysis.
799///
800/// This helper ensures that the -regalloc-npm= option is always available,
801/// even for targets that override the default allocator.
802void CodeGenPassBuilder::addRegAllocPass(PassManagerWrapper &PMW,
803 bool Optimized) {
804 // Use the specified -regalloc-npm={basic|greedy|fast|pbqp}
805 if (Opt.RegAlloc > RegAllocType::Default) {
806 switch (Opt.RegAlloc) {
807 case RegAllocType::Fast:
808 addMachineFunctionPass(Pass: RegAllocFastPass(), PMW);
809 break;
810 case RegAllocType::Greedy:
811 addMachineFunctionPass(Pass: RAGreedyPass(), PMW);
812 break;
813 default:
814 reportFatalUsageError(reason: "register allocator not supported yet");
815 }
816 return;
817 }
818 // -regalloc=default or unspecified, so pick based on the optimization level
819 // or ask the target for the regalloc pass.
820 addTargetRegisterAllocator(PMW, Optimized);
821}
822
823Error CodeGenPassBuilder::addRegAssignAndRewriteFast(PassManagerWrapper &PMW) {
824 // TODO: Ensure allocator is default or fast.
825 addRegAllocPass(PMW, Optimized: false);
826 return Error::success();
827}
828
829Expected<bool>
830CodeGenPassBuilder::addRegAssignAndRewriteOptimized(PassManagerWrapper &PMW) {
831 // Add the selected register allocation pass.
832 addRegAllocPass(PMW, Optimized: true);
833
834 // Allow targets to change the register assignments before rewriting.
835 addPreRewrite(PMW);
836
837 // Finally rewrite virtual registers.
838 addMachineFunctionPass(Pass: VirtRegRewriterPass(), PMW);
839
840 return true;
841}
842
843/// Add the minimum set of target-independent passes that are required for
844/// register allocation. No coalescing or scheduling.
845Error CodeGenPassBuilder::addFastRegAlloc(PassManagerWrapper &PMW) {
846 addMachineFunctionPass(Pass: PHIEliminationPass(), PMW);
847 addMachineFunctionPass(Pass: TwoAddressInstructionPass(), PMW);
848 return addRegAssignAndRewriteFast(PMW);
849}
850
851/// Add standard target-independent passes that are tightly coupled with
852/// optimized register allocation, including coalescing, machine instruction
853/// scheduling, and register allocation itself.
854Error CodeGenPassBuilder::addOptimizedRegAlloc(PassManagerWrapper &PMW) {
855 addMachineFunctionPass(Pass: DetectDeadLanesPass(), PMW);
856
857 addMachineFunctionPass(Pass: InitUndefPass(), PMW);
858
859 addMachineFunctionPass(Pass: ProcessImplicitDefsPass(), PMW);
860
861 // LiveVariables currently requires pure SSA form.
862 //
863 // FIXME: Once TwoAddressInstruction pass no longer uses kill flags,
864 // LiveVariables can be removed completely, and LiveIntervals can be directly
865 // computed. (We still either need to regenerate kill flags after regalloc, or
866 // preferably fix the scavenger to not depend on them).
867 // FIXME: UnreachableMachineBlockElim is a dependant pass of LiveVariables.
868 // When LiveVariables is removed this has to be removed/moved either.
869 // Explicit addition of UnreachableMachineBlockElim allows stopping before or
870 // after it with -stop-before/-stop-after.
871 addMachineFunctionPass(Pass: UnreachableMachineBlockElimPass(), PMW);
872 addMachineFunctionPass(
873 Pass: RequireAnalysisPass<LiveVariablesAnalysis, MachineFunction>(), PMW);
874
875 // Edge splitting is smarter with machine loop info.
876 addMachineFunctionPass(
877 Pass: RequireAnalysisPass<MachineLoopAnalysis, MachineFunction>(), PMW);
878 addMachineFunctionPass(Pass: PHIEliminationPass(), PMW);
879
880 // Eventually, we want to run LiveIntervals before PHI elimination.
881 if (Opt.EarlyLiveIntervals)
882 addMachineFunctionPass(
883 Pass: RequireAnalysisPass<LiveIntervalsAnalysis, MachineFunction>(), PMW);
884
885 addMachineFunctionPass(Pass: TwoAddressInstructionPass(), PMW);
886 addMachineFunctionPass(Pass: RegisterCoalescerPass(), PMW);
887
888 // The machine scheduler may accidentally create disconnected components
889 // when moving subregister definitions around, avoid this by splitting them to
890 // separate vregs before. Splitting can also improve reg. allocation quality.
891 addMachineFunctionPass(Pass: RenameIndependentSubregsPass(), PMW);
892
893 // PreRA instruction scheduling.
894 addMachineFunctionPass(Pass: MachineSchedulerPass(&TM), PMW);
895
896 Expected<bool> AddedPasses = addRegAssignAndRewriteOptimized(PMW);
897 if (!AddedPasses)
898 return AddedPasses.takeError();
899 if (!AddedPasses.get())
900 return Error::success();
901
902 addMachineFunctionPass(Pass: StackSlotColoringPass(), PMW);
903
904 // Allow targets to expand pseudo instructions depending on the choice of
905 // registers before MachineCopyPropagation.
906 addPostRewrite(PMW);
907
908 // Copy propagate to forward register uses and try to eliminate COPYs that
909 // were not coalesced.
910 addMachineFunctionPass(Pass: MachineCopyPropagationPass(), PMW);
911
912 // Run post-ra machine LICM to hoist reloads / remats.
913 //
914 // FIXME: can this move into MachineLateOptimization?
915 addMachineFunctionPass(Pass: MachineLICMPass(), PMW);
916
917 return Error::success();
918}
919
920//===---------------------------------------------------------------------===//
921/// Post RegAlloc Pass Configuration
922//===---------------------------------------------------------------------===//
923
924/// Add passes that optimize machine instructions after register allocation.
925void CodeGenPassBuilder::addMachineLateOptimization(PassManagerWrapper &PMW) {
926 // Cleanup of redundant (identical) address/immediate loads.
927 addMachineFunctionPass(Pass: MachineLateInstrsCleanupPass(), PMW);
928
929 // Branch folding must be run after regalloc and prolog/epilog insertion.
930 addMachineFunctionPass(Pass: BranchFolderPass(Opt.EnableTailMerge), PMW);
931
932 // Tail duplication.
933 // Note that duplicating tail just increases code size and degrades
934 // performance for targets that require Structured Control Flow.
935 // In addition it can also make CFG irreducible. Thus we disable it.
936 if (!TM.requiresStructuredCFG())
937 addMachineFunctionPass(Pass: TailDuplicatePass(), PMW);
938
939 // Copy propagation.
940 addMachineFunctionPass(Pass: MachineCopyPropagationPass(), PMW);
941}
942
943/// Add standard basic block placement passes.
944void CodeGenPassBuilder::addBlockPlacement(PassManagerWrapper &PMW) {
945 addMachineFunctionPass(Pass: MachineBlockPlacementPass(Opt.EnableTailMerge), PMW);
946 // Run a separate pass to collect block placement statistics.
947 if (Opt.EnableBlockPlacementStats)
948 addMachineFunctionPass(Pass: MachineBlockPlacementStatsPass(), PMW);
949}
950