1//===--- CodeGenPassBuilder.cpp --------------------------------------- ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines interfaces to access the target independent code
10// generation passes provided by the LLVM backend.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Passes/CodeGenPassBuilder.h"
15#include "llvm/ADT/SmallVector.h"
16#include "llvm/ADT/StringRef.h"
17#include "llvm/Analysis/CGSCCPassManager.h"
18#include "llvm/Analysis/ProfileSummaryInfo.h"
19#include "llvm/Analysis/RuntimeLibcallInfo.h"
20#include "llvm/CodeGen/AsmPrinter.h"
21#include "llvm/CodeGen/AsmPrinterAnalysis.h"
22#include "llvm/CodeGen/BranchFoldingPass.h"
23#include "llvm/CodeGen/CFIFixup.h"
24#include "llvm/CodeGen/CodeGenPrepare.h"
25#include "llvm/CodeGen/DeadMachineInstructionElim.h"
26#include "llvm/CodeGen/DetectDeadLanes.h"
27#include "llvm/CodeGen/DwarfEHPrepare.h"
28#include "llvm/CodeGen/ExpandIRInsts.h"
29#include "llvm/CodeGen/ExpandPostRAPseudos.h"
30#include "llvm/CodeGen/ExpandReductions.h"
31#include "llvm/CodeGen/FEntryInserter.h"
32#include "llvm/CodeGen/FinalizeISel.h"
33#include "llvm/CodeGen/FixupStatepointCallerSaved.h"
34#include "llvm/CodeGen/FuncletLayout.h"
35#include "llvm/CodeGen/GCEmptyBasicBlocks.h"
36#include "llvm/CodeGen/GCMetadata.h"
37#include "llvm/CodeGen/GlobalMerge.h"
38#include "llvm/CodeGen/GlobalMergeFunctions.h"
39#include "llvm/CodeGen/ImplicitNullChecks.h"
40#include "llvm/CodeGen/IndirectBrExpand.h"
41#include "llvm/CodeGen/InitUndef.h"
42#include "llvm/CodeGen/InlineAsmPrepare.h"
43#include "llvm/CodeGen/InterleavedAccess.h"
44#include "llvm/CodeGen/InterleavedLoadCombine.h"
45#include "llvm/CodeGen/LiveDebugValuesPass.h"
46#include "llvm/CodeGen/LiveIntervals.h"
47#include "llvm/CodeGen/LocalStackSlotAllocation.h"
48#include "llvm/CodeGen/LowerEmuTLS.h"
49#include "llvm/CodeGen/MIRPrinter.h"
50#include "llvm/CodeGen/MachineBlockPlacement.h"
51#include "llvm/CodeGen/MachineCSE.h"
52#include "llvm/CodeGen/MachineCopyPropagation.h"
53#include "llvm/CodeGen/MachineFunctionAnalysis.h"
54#include "llvm/CodeGen/MachineLICM.h"
55#include "llvm/CodeGen/MachineLateInstrsCleanup.h"
56#include "llvm/CodeGen/MachineModuleInfo.h"
57#include "llvm/CodeGen/MachinePassManager.h"
58#include "llvm/CodeGen/MachinePipeliner.h"
59#include "llvm/CodeGen/MachineScheduler.h"
60#include "llvm/CodeGen/MachineSink.h"
61#include "llvm/CodeGen/MachineVerifier.h"
62#include "llvm/CodeGen/OptimizePHIs.h"
63#include "llvm/CodeGen/PEI.h"
64#include "llvm/CodeGen/PHIElimination.h"
65#include "llvm/CodeGen/PatchableFunction.h"
66#include "llvm/CodeGen/PeepholeOptimizer.h"
67#include "llvm/CodeGen/PostRAMachineSink.h"
68#include "llvm/CodeGen/PostRASchedulerList.h"
69#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
70#include "llvm/CodeGen/ProcessImplicitDefs.h"
71#include "llvm/CodeGen/RegAllocEvictionAdvisor.h"
72#include "llvm/CodeGen/RegAllocFast.h"
73#include "llvm/CodeGen/RegAllocGreedyPass.h"
74#include "llvm/CodeGen/RegUsageInfoCollector.h"
75#include "llvm/CodeGen/RegUsageInfoPropagate.h"
76#include "llvm/CodeGen/RegisterCoalescerPass.h"
77#include "llvm/CodeGen/RegisterUsageInfo.h"
78#include "llvm/CodeGen/RemoveLoadsIntoFakeUses.h"
79#include "llvm/CodeGen/RemoveRedundantDebugValues.h"
80#include "llvm/CodeGen/RenameIndependentSubregs.h"
81#include "llvm/CodeGen/ReplaceWithVeclib.h"
82#include "llvm/CodeGen/ResetMachineFunctionPass.h"
83#include "llvm/CodeGen/SafeStack.h"
84#include "llvm/CodeGen/SanitizerBinaryMetadata.h"
85#include "llvm/CodeGen/SelectOptimize.h"
86#include "llvm/CodeGen/ShadowStackGCLowering.h"
87#include "llvm/CodeGen/ShrinkWrap.h"
88#include "llvm/CodeGen/SjLjEHPrepare.h"
89#include "llvm/CodeGen/StackColoring.h"
90#include "llvm/CodeGen/StackFrameLayoutAnalysisPass.h"
91#include "llvm/CodeGen/StackProtector.h"
92#include "llvm/CodeGen/StackSlotColoring.h"
93#include "llvm/CodeGen/TailDuplication.h"
94#include "llvm/CodeGen/TargetPassConfig.h"
95#include "llvm/CodeGen/TwoAddressInstructionPass.h"
96#include "llvm/CodeGen/UnreachableBlockElim.h"
97#include "llvm/CodeGen/VirtRegMap.h"
98#include "llvm/CodeGen/WasmEHPrepare.h"
99#include "llvm/CodeGen/WinEHPrepare.h"
100#include "llvm/CodeGen/XRayInstrumentation.h"
101#include "llvm/IR/PassManager.h"
102#include "llvm/IR/Verifier.h"
103#include "llvm/IRPrinter/IRPrintingPasses.h"
104#include "llvm/MC/MCAsmInfo.h"
105#include "llvm/MC/MCStreamer.h"
106#include "llvm/MC/MCTargetOptions.h"
107#include "llvm/MC/TargetRegistry.h"
108#include "llvm/Support/CodeGen.h"
109#include "llvm/Support/Debug.h"
110#include "llvm/Support/Error.h"
111#include "llvm/Support/ErrorHandling.h"
112#include "llvm/Target/CGPassBuilderOption.h"
113#include "llvm/Target/TargetMachine.h"
114#include "llvm/Transforms/ObjCARC.h"
115#include "llvm/Transforms/Scalar/ConstantHoisting.h"
116#include "llvm/Transforms/Scalar/LoopPassManager.h"
117#include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
118#include "llvm/Transforms/Scalar/LoopTermFold.h"
119#include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
120#include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
121#include "llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h"
122#include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
123#include "llvm/Transforms/Utils/LowerInvoke.h"
124#include "llvm/Transforms/Utils/ProfileVerify.h"
125#include <cassert>
126#include <utility>
127
128using namespace llvm;
129
130namespace llvm {
131extern cl::opt<bool> EnableProfcheckInPipeline;
132
133#define DUMMY_MACHINE_FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
134 AnalysisKey PASS_NAME::Key;
135#include "llvm/Passes/MachinePassRegistry.def"
136} // namespace llvm
137
138CodeGenPassBuilder::CodeGenPassBuilder(TargetMachine &TM,
139 const CGPassBuilderOption &Opts,
140 PassInstrumentationCallbacks *PIC)
141 : TM(TM), Opt(Opts), PIC(PIC) {
142 // Target could set CGPassBuilderOption::MISchedPostRA to true to achieve
143 // substitutePass(&PostRASchedulerID, &PostMachineSchedulerID)
144
145 // Target should override TM.Options.EnableIPRA in their target-specific
146 // LLVMTM ctor. See TargetMachine::setGlobalISel for example.
147 if (Opt.EnableIPRA) {
148 TM.Options.EnableIPRA = *Opt.EnableIPRA;
149 } else {
150 // If not explicitly specified, use target default.
151 TM.Options.EnableIPRA |= TM.useIPRA();
152 }
153
154 if (Opt.EnableGlobalISelAbort)
155 TM.Options.GlobalISelAbort = *Opt.EnableGlobalISelAbort;
156
157 if (Opt.EnableRegAllocFastTied != cl::boolOrDefault::BOU_UNSET)
158 TM.setEnableTiedFastRegAlloc(Opt.EnableRegAllocFastTied ==
159 cl::boolOrDefault::BOU_TRUE);
160
161 // An explicit RegAlloc choice implies its pipeline: only the fast
162 // allocator uses the unoptimized one.
163 if (Opt.OptimizeRegAlloc == cl::boolOrDefault::BOU_UNSET) {
164 bool Optimized = Opt.RegAlloc > RegAllocType::Default
165 ? Opt.RegAlloc != RegAllocType::Fast
166 : getOptLevel() != CodeGenOptLevel::None;
167 Opt.OptimizeRegAlloc =
168 Optimized ? cl::boolOrDefault::BOU_TRUE : cl::boolOrDefault::BOU_FALSE;
169 }
170}
171
172// Out-of-line to anchor the vtable in this translation unit.
173CodeGenPassBuilder::~CodeGenPassBuilder() = default;
174
175Error CodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) {
176 return make_error<StringError>(Args: "addInstSelector is not overridden",
177 Args: inconvertibleErrorCode());
178}
179
180Error CodeGenPassBuilder::addIRTranslator(PassManagerWrapper &PMW) {
181 return make_error<StringError>(Args: "addIRTranslator is not overridden",
182 Args: inconvertibleErrorCode());
183}
184
185Error CodeGenPassBuilder::addLegalizeMachineIR(PassManagerWrapper &PMW) {
186 return make_error<StringError>(Args: "addLegalizeMachineIR is not overridden",
187 Args: inconvertibleErrorCode());
188}
189
190Error CodeGenPassBuilder::addRegBankSelect(PassManagerWrapper &PMW) {
191 return make_error<StringError>(Args: "addRegBankSelect is not overridden",
192 Args: inconvertibleErrorCode());
193}
194
195Error CodeGenPassBuilder::addGlobalInstructionSelect(PassManagerWrapper &PMW) {
196 return make_error<StringError>(Args: "addGlobalInstructionSelect is not overridden",
197 Args: inconvertibleErrorCode());
198}
199
200void CodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) {
201 llvm_unreachable("addAsmPrinterBegin is not overriden");
202}
203
204void CodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
205 llvm_unreachable("addAsmPrinter is not overridden");
206}
207
208void CodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
209 llvm_unreachable("addAsmPrinterEnd is not overriden");
210}
211
212void CodeGenPassBuilder::flushFPMsToMPM(PassManagerWrapper &PMW,
213 bool FreeMachineFunctions) {
214 if (PMW.FPM.isEmpty() && PMW.MFPM.isEmpty())
215 return;
216 if (!PMW.MFPM.isEmpty()) {
217 PMW.FPM.addPass(
218 Pass: createFunctionToMachineFunctionPassAdaptor(Pass: std::move(PMW.MFPM)));
219 PMW.MFPM = MachineFunctionPassManager();
220 }
221 if (FreeMachineFunctions)
222 PMW.FPM.addPass(Pass: FreeMachineFunctionPass());
223 if (AddInCGSCCOrder) {
224 PMW.MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(
225 Pass: createCGSCCToFunctionPassAdaptor(Pass: std::move(PMW.FPM))));
226 } else {
227 PMW.MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(PMW.FPM)));
228 }
229 PMW.FPM = FunctionPassManager();
230}
231
232Error CodeGenPassBuilder::buildPipeline(
233 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
234 raw_pwrite_stream *DwoOut, CodeGenFileType FileType, MCContext &Ctx) {
235 auto StartStopInfo = TargetPassConfig::getStartStopInfo(TM, PIC&: *PIC);
236 if (!StartStopInfo)
237 return StartStopInfo.takeError();
238 setStartStopPasses(*StartStopInfo);
239
240 CompletesPipeline = StartStopInfo->StopPass.empty();
241 bool PrintAsm = CompletesPipeline;
242 bool PrintMIR = !PrintAsm && FileType != CodeGenFileType::Null;
243
244 PassManagerWrapper PMW(MPM);
245
246 addModulePass(Pass: RequireAnalysisPass<MachineModuleAnalysis, Module>(), PMW,
247 /*Force=*/true);
248 addModulePass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>(), PMW,
249 /*Force=*/true);
250 addModulePass(Pass: RequireAnalysisPass<CollectorMetadataAnalysis, Module>(), PMW,
251 /*Force=*/true);
252 addModulePass(Pass: RequireAnalysisPass<RuntimeLibraryAnalysis, Module>(), PMW,
253 /*Force=*/true);
254 addModulePass(Pass: RequireAnalysisPass<LibcallLoweringModuleAnalysis, Module>(),
255 PMW,
256 /*Force=*/true);
257 addISelPasses(PMW);
258 flushFPMsToMPM(PMW);
259
260 if (PrintAsm) {
261 Expected<std::unique_ptr<MCStreamer>> MCStreamerOrErr =
262 TM.createMCStreamer(Out, DwoOut, FileType, Ctx);
263 if (!MCStreamerOrErr)
264 return MCStreamerOrErr.takeError();
265 std::unique_ptr<AsmPrinter> Printer(
266 TM.getTarget().createAsmPrinter(TM, Streamer: std::move(*MCStreamerOrErr)));
267 if (!Printer)
268 return createStringError(Fmt: "failed to create AsmPrinter");
269 MAM.registerPass(PassBuilder: [&] { return AsmPrinterAnalysis(std::move(Printer)); });
270 addAsmPrinterBegin(PMW);
271 }
272
273 if (auto Err = addCoreISelPasses(PMW))
274 return Err;
275
276 if (auto Err = addMachinePasses(PMW))
277 return Err;
278
279 if (PrintAsm ? !Opt.DisableVerify && TM.Options.EnableDefaultMachineVerifier
280 : Opt.VerifyMachineCode == MachineVerifierMode::End)
281 addMachineFunctionPass(Pass: MachineVerifierPass(), PMW, /*Force=*/!PrintAsm);
282
283 // We add AsmPrinter regardless if we are emitting MIR or Assembly as the
284 // final output so that -stop-before=<target>-asm-printer works. When printing
285 // MIR as the final output, we never end up running AsmPrinter.
286 addAsmPrinter(PMW);
287
288 flushFPMsToMPM(PMW, /*FreeMachineFunctions=*/!PrintMIR);
289 if (PrintAsm)
290 addAsmPrinterEnd(PMW);
291 else if (PrintMIR)
292 addModulePass(Pass: PrintMIRPass(Out), PMW, /*Force=*/true);
293
294 return verifyStartStop(Info: *StartStopInfo);
295}
296
297void CodeGenPassBuilder::setStartStopPasses(
298 const TargetPassConfig::StartStopInfo &Info) {
299 if (!Info.StartPass.empty()) {
300 Started = false;
301 BeforeCallbacks.emplace_back(Args: [this, &Info, AfterFlag = Info.StartAfter,
302 Count = 0u](StringRef ClassName) mutable {
303 if (Count == Info.StartInstanceNum) {
304 if (AfterFlag) {
305 AfterFlag = false;
306 Started = true;
307 }
308 return Started;
309 }
310
311 auto PassName = PIC->getPassNameForClassName(ClassName);
312 if (Info.StartPass == PassName && ++Count == Info.StartInstanceNum)
313 Started = !Info.StartAfter;
314
315 return Started;
316 });
317 }
318
319 if (!Info.StopPass.empty()) {
320 Stopped = false;
321 BeforeCallbacks.emplace_back(Args: [this, &Info, AfterFlag = Info.StopAfter,
322 Count = 0u](StringRef ClassName) mutable {
323 if (Count == Info.StopInstanceNum) {
324 if (AfterFlag) {
325 AfterFlag = false;
326 Stopped = true;
327 }
328 return !Stopped;
329 }
330
331 auto PassName = PIC->getPassNameForClassName(ClassName);
332 if (Info.StopPass == PassName && ++Count == Info.StopInstanceNum)
333 Stopped = !Info.StopAfter;
334 return !Stopped;
335 });
336 }
337}
338
339Error CodeGenPassBuilder::verifyStartStop(
340 const TargetPassConfig::StartStopInfo &Info) const {
341 if (Started && Stopped)
342 return Error::success();
343
344 if (!Started)
345 return make_error<StringError>(
346 Args: "Can't find start pass \"" + Info.StartPass + "\".",
347 Args: std::make_error_code(e: std::errc::invalid_argument));
348 if (!Stopped)
349 return make_error<StringError>(
350 Args: "Can't find stop pass \"" + Info.StopPass + "\".",
351 Args: std::make_error_code(e: std::errc::invalid_argument));
352 return Error::success();
353}
354
355void CodeGenPassBuilder::addISelPasses(PassManagerWrapper &PMW) {
356 if (EnableProfcheckInPipeline) {
357 addFunctionPass(Pass: ProfileInjectorPass(), PMW);
358 flushFPMsToMPM(PMW);
359 }
360
361 addGlobalMergePass(PMW);
362 if (TM.useEmulatedTLS())
363 addModulePass(Pass: LowerEmuTLSPass(), PMW);
364
365 // ObjCARCContract operates on ObjC intrinsics and must run before
366 // PreISelIntrinsicLowering.
367 if (getOptLevel() != CodeGenOptLevel::None) {
368 addFunctionPass(Pass: ObjCARCContractPass(), PMW);
369 flushFPMsToMPM(PMW);
370 }
371 addModulePass(Pass: PreISelIntrinsicLoweringPass(TM), PMW);
372 addFunctionPass(Pass: ExpandIRInstsPass(TM, getOptLevel()), PMW);
373
374 addIRPasses(PMW);
375 addCodeGenPrepare(PMW);
376 addPassesToHandleExceptions(PMW);
377 addISelPrepare(PMW);
378}
379
380/// Add common target configurable passes that perform LLVM IR to IR transforms
381/// following machine independent optimization.
382void CodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
383 // Before running any passes, run the verifier to determine if the input
384 // coming from the front-end and/or optimizer is valid.
385 if (!Opt.DisableVerify)
386 addFunctionPass(Pass: VerifierPass(), PMW, /*Force=*/true);
387
388 // Run loop strength reduction before anything else.
389 if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableLSR) {
390 // These passes do not use MSSA.
391 LoopPassManager LPM;
392 LPM.addPass(Pass: CanonicalizeFreezeInLoopsPass());
393 LPM.addPass(Pass: LoopStrengthReducePass());
394 if (Opt.EnableLoopTermFold)
395 LPM.addPass(Pass: LoopTermFoldPass());
396 addFunctionPass(Pass: createFunctionToLoopPassAdaptor(LPM: std::move(LPM),
397 /*UseMemorySSA=*/false),
398 PMW);
399 }
400
401 // Run GC lowering passes for builtin collectors
402 // TODO: add a pass insertion point here
403 addFunctionPass(Pass: GCLoweringPass(), PMW);
404 // Explicitly check to see if we should add ShadowStackGCLowering to avoid
405 // splitting the function pipeline if we do not have to.
406 if (runBeforeAdding(Name: ShadowStackGCLoweringPass::name())) {
407 flushFPMsToMPM(PMW);
408 addModulePass(Pass: ShadowStackGCLoweringPass(), PMW);
409 }
410
411 // Make sure that no unreachable blocks are instruction selected.
412 addFunctionPass(Pass: UnreachableBlockElimPass(), PMW);
413
414 // Prepare expensive constants for SelectionDAG.
415 if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableConstantHoisting)
416 addFunctionPass(Pass: ConstantHoistingPass(), PMW);
417
418 // Replace calls to LLVM intrinsics (e.g., exp, log) operating on vector
419 // operands with calls to the corresponding functions in a vector library.
420 if (getOptLevel() != CodeGenOptLevel::None)
421 addFunctionPass(Pass: ReplaceWithVeclib(), PMW);
422
423 if (getOptLevel() != CodeGenOptLevel::None &&
424 !Opt.DisablePartialLibcallInlining)
425 addFunctionPass(Pass: PartiallyInlineLibCallsPass(), PMW);
426
427 // Instrument function entry and exit, e.g. with calls to mcount().
428 addFunctionPass(Pass: EntryExitInstrumenterPass(/*PostInlining=*/true), PMW);
429
430 // Add scalarization of target's unsupported masked memory intrinsics pass.
431 // the unsupported intrinsic will be replaced with a chain of basic blocks,
432 // that stores/loads element one-by-one if the appropriate mask bit is set.
433 addFunctionPass(Pass: ScalarizeMaskedMemIntrinPass(), PMW);
434
435 // Expand reduction intrinsics into shuffle sequences if the target wants to.
436 if (!Opt.DisableExpandReductions)
437 addFunctionPass(Pass: ExpandReductionsPass(), PMW);
438
439 // Convert conditional moves to conditional jumps when profitable.
440 if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableSelectOptimize)
441 addFunctionPass(Pass: SelectOptimizePass(TM), PMW);
442
443 if (Opt.EnableGlobalMergeFunc) {
444 flushFPMsToMPM(PMW);
445 addModulePass(Pass: GlobalMergeFuncPass(), PMW);
446 }
447}
448
449/// Turn exception handling constructs into something the code generators can
450/// handle.
451void CodeGenPassBuilder::addPassesToHandleExceptions(PassManagerWrapper &PMW) {
452 // The module's exception model is not known when the pipeline is built, so
453 // schedule the preparation pass for every model the target supports. Each
454 // pass does nothing for other models.
455 bool HasEHPrep = false;
456
457 // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both
458 // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise,
459 // catch info can get misplaced when a selector ends up more than one block
460 // removed from the parent invoke(s). This could happen when a landing
461 // pad is shared by multiple invokes and is also a target of a normal
462 // edge from elsewhere.
463 if (TM.supportsExceptionModel(Model: ExceptionHandling::SjLj)) {
464 addFunctionPass(Pass: SjLjEHPreparePass(), PMW);
465 HasEHPrep = true;
466 }
467 // We support using both GCC-style and MSVC-style exceptions on Windows, so
468 // WinEHPrepare runs in addition to DwarfEHPrepare below. Each pass will only
469 // actually run if it recognizes the personality function.
470 //
471 // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs
472 // on catchpads and cleanuppads because it does not outline them into
473 // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we
474 // should remove PHIs there.
475 if (TM.supportsExceptionModel(Model: ExceptionHandling::WinEH) ||
476 TM.supportsExceptionModel(Model: ExceptionHandling::Wasm)) {
477 addFunctionPass(Pass: WinEHPreparePass(), PMW);
478 HasEHPrep = true;
479 }
480 if (TM.supportsExceptionModel(Model: ExceptionHandling::Wasm))
481 addFunctionPass(Pass: WasmEHPreparePass(), PMW);
482 if (TM.supportsExceptionModel(Model: ExceptionHandling::DwarfCFI)) {
483 addFunctionPass(Pass: DwarfEHPreparePass(TM), PMW);
484 HasEHPrep = true;
485 }
486
487 // If the target supports no exception model, nothing else lowers invokes.
488 if (!HasEHPrep) {
489 addFunctionPass(Pass: LowerInvokePass(), PMW);
490
491 // The lower invoke pass may create unreachable code. Remove it.
492 addFunctionPass(Pass: UnreachableBlockElimPass(), PMW);
493 }
494}
495
496/// Add pass to prepare the LLVM IR for code generation. This should be done
497/// before exception handling preparation passes.
498void CodeGenPassBuilder::addCodeGenPrepare(PassManagerWrapper &PMW) {
499 if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableCGP)
500 addFunctionPass(Pass: CodeGenPreparePass(TM), PMW);
501 // TODO: Default ctor'd RewriteSymbolPass is no-op.
502 // addPass(RewriteSymbolPass());
503}
504
505/// Add common passes that perform LLVM IR to IR transforms in preparation for
506/// instruction selection.
507void CodeGenPassBuilder::addISelPrepare(PassManagerWrapper &PMW) {
508 addPreISel(PMW);
509
510 if (Opt.RequiresCodeGenSCCOrder && !AddInCGSCCOrder)
511 requireCGSCCOrder(PMW);
512
513 addFunctionPass(Pass: InlineAsmPreparePass(), PMW);
514 // Add both the safe stack and the stack protection passes: each of them will
515 // only protect functions that have corresponding attributes.
516 addFunctionPass(Pass: SafeStackPass(TM), PMW);
517 addFunctionPass(Pass: StackProtectorPass(TM), PMW);
518
519 if (Opt.PrintISelInput)
520 addFunctionPass(Pass: PrintFunctionPass(
521 dbgs(), "\n\n*** Final LLVM Code input to ISel ***\n"),
522 PMW);
523
524 if (EnableProfcheckInPipeline) {
525 flushFPMsToMPM(PMW);
526 addModulePass(Pass: ProfileVerifierPass(), PMW);
527 }
528
529 // All passes which modify the LLVM IR are now complete; run the verifier
530 // to ensure that the IR is valid.
531 if (!Opt.DisableVerify)
532 addFunctionPass(Pass: VerifierPass(), PMW, /*Force=*/true);
533}
534
535Error CodeGenPassBuilder::addCoreISelPasses(PassManagerWrapper &PMW) {
536 // Enable FastISel with -fast-isel, but allow that to be overridden.
537 TM.setO0WantsFastISel(Opt.EnableFastISelOption !=
538 cl::boolOrDefault::BOU_FALSE);
539
540 // Determine an instruction selector.
541 enum class SelectorType { SelectionDAG, FastISel, GlobalISel };
542 SelectorType Selector;
543
544 if (Opt.EnableFastISelOption == cl::boolOrDefault::BOU_TRUE)
545 Selector = SelectorType::FastISel;
546 else if (Opt.EnableGlobalISelOption == cl::boolOrDefault::BOU_TRUE ||
547 (TM.Options.EnableGlobalISel &&
548 Opt.EnableGlobalISelOption != cl::boolOrDefault::BOU_FALSE))
549 Selector = SelectorType::GlobalISel;
550 else if (TM.getOptLevel() == CodeGenOptLevel::None && TM.getO0WantsFastISel())
551 Selector = SelectorType::FastISel;
552 else
553 Selector = SelectorType::SelectionDAG;
554
555 // Set consistently TM.Options.EnableFastISel and EnableGlobalISel.
556 if (Selector == SelectorType::FastISel) {
557 TM.setFastISel(true);
558 TM.setGlobalISel(false);
559 } else if (Selector == SelectorType::GlobalISel) {
560 TM.setFastISel(false);
561 TM.setGlobalISel(true);
562 }
563
564 // Add instruction selector passes.
565 if (Selector == SelectorType::GlobalISel) {
566 if (auto Err = addIRTranslator(PMW))
567 return Err;
568
569 addPreLegalizeMachineIR(PMW);
570
571 if (auto Err = addLegalizeMachineIR(PMW))
572 return Err;
573
574 // Before running the register bank selector, ask the target if it
575 // wants to run some passes.
576 addPreRegBankSelect(PMW);
577
578 if (auto Err = addRegBankSelect(PMW))
579 return Err;
580
581 addPreGlobalInstructionSelect(PMW);
582
583 if (auto Err = addGlobalInstructionSelect(PMW))
584 return Err;
585
586 // Pass to reset the MachineFunction if the ISel failed.
587 addMachineFunctionPass(
588 Pass: ResetMachineFunctionPass(reportDiagnosticWhenGlobalISelFallback(),
589 isGlobalISelAbortEnabled()),
590 PMW);
591
592 // Provide a fallback path when we do not want to abort on
593 // not-yet-supported input.
594 if (!isGlobalISelAbortEnabled())
595 if (auto Err = addInstSelector(PMW))
596 return Err;
597
598 } else if (auto Err = addInstSelector(PMW))
599 return Err;
600
601 // Expand pseudo-instructions emitted by ISel. Don't run the verifier before
602 // FinalizeISel.
603 addMachineFunctionPass(Pass: FinalizeISelPass(), PMW);
604
605 // // Print the instruction selected machine code...
606 // printAndVerify("After Instruction Selection");
607
608 return Error::success();
609}
610
611/// Add the complete set of target-independent postISel code generator passes.
612///
613/// This can be read as the standard order of major LLVM CodeGen stages. Stages
614/// with nontrivial configuration or multiple passes are broken out below in
615/// add%Stage routines.
616///
617/// Any CodeGenPassBuilder::addXX routine may be overriden by the Target. The
618/// addPre/Post methods with empty header implementations allow injecting
619/// target-specific fixups just before or after major stages. Additionally,
620/// targets have the flexibility to change pass order within a stage by
621/// overriding default implementation of add%Stage routines below. Each
622/// technique has maintainability tradeoffs because alternate pass orders are
623/// not well supported. addPre/Post works better if the target pass is easily
624/// tied to a common pass. But if it has subtle dependencies on multiple passes,
625/// the target should override the stage instead.
626Error CodeGenPassBuilder::addMachinePasses(PassManagerWrapper &PMW) {
627 // Add passes that optimize machine instructions in SSA form.
628 if (getOptLevel() != CodeGenOptLevel::None) {
629 addMachineSSAOptimization(PMW);
630 } else {
631 // If the target requests it, assign local variables to stack slots relative
632 // to one another and simplify frame index references where possible.
633 addMachineFunctionPass(Pass: LocalStackSlotAllocationPass(), PMW);
634 }
635
636 if (TM.Options.EnableIPRA) {
637 flushFPMsToMPM(PMW);
638 addModulePass(Pass: RequireAnalysisPass<PhysicalRegisterUsageAnalysis, Module>(),
639 PMW, /*Force=*/true);
640 addMachineFunctionPass(Pass: RegUsageInfoPropagationPass(), PMW);
641 }
642
643 if (getOptLevel() != CodeGenOptLevel::None &&
644 (Opt.EnableMachinePipeliner == cl::boolOrDefault::BOU_UNSET
645 ? isMachinePipelinerEnabled()
646 : Opt.EnableMachinePipeliner == cl::boolOrDefault::BOU_TRUE))
647 addMachineFunctionPass(Pass: MachinePipelinerPass(), PMW);
648
649 // Run pre-ra passes.
650 addPreRegAlloc(PMW);
651
652 // Run register allocation and passes that are tightly coupled with it,
653 // including phi elimination and scheduling.
654 if (auto Err = Opt.OptimizeRegAlloc == cl::boolOrDefault::BOU_TRUE
655 ? addOptimizedRegAlloc(PMW)
656 : addFastRegAlloc(PMW))
657 return Err;
658
659 // Run post-ra passes.
660 addPostRegAlloc(PMW);
661
662 addMachineFunctionPass(Pass: RemoveRedundantDebugValuesPass(), PMW);
663 addMachineFunctionPass(Pass: FixupStatepointCallerSavedPass(), PMW);
664
665 // Insert prolog/epilog code. Eliminate abstract frame index references...
666 if (getOptLevel() != CodeGenOptLevel::None) {
667 addMachineFunctionPass(Pass: PostRAMachineSinkingPass(), PMW);
668 addMachineFunctionPass(Pass: ShrinkWrapPass(), PMW);
669 }
670
671 addMachineFunctionPass(Pass: PrologEpilogInserterPass(), PMW);
672
673 /// Add passes that optimize machine instructions after register allocation.
674 if (getOptLevel() != CodeGenOptLevel::None)
675 addMachineLateOptimization(PMW);
676
677 // Expand pseudo instructions before second scheduling pass.
678 addMachineFunctionPass(Pass: ExpandPostRAPseudosPass(), PMW);
679
680 // Run pre-sched2 passes.
681 addPreSched2(PMW);
682
683 if (Opt.EnableImplicitNullChecks)
684 addMachineFunctionPass(Pass: ImplicitNullChecksPass(), PMW);
685
686 // Second pass scheduler.
687 // Let Target optionally insert this pass by itself at some other
688 // point.
689 if (getOptLevel() != CodeGenOptLevel::None &&
690 !TM.targetSchedulesPostRAScheduling()) {
691 if (Opt.MISchedPostRA)
692 addMachineFunctionPass(Pass: PostMachineSchedulerPass(&TM), PMW);
693 else
694 addMachineFunctionPass(Pass: PostRASchedulerPass(&TM), PMW);
695 }
696
697 // GC
698 addGCPasses(PMW);
699
700 // Basic block placement.
701 if (getOptLevel() != CodeGenOptLevel::None)
702 addBlockPlacement(PMW);
703
704 // Insert before XRay Instrumentation.
705 addMachineFunctionPass(Pass: FEntryInserterPass(), PMW);
706
707 addMachineFunctionPass(Pass: XRayInstrumentationPass(), PMW);
708 addMachineFunctionPass(Pass: PatchableFunctionPass(), PMW);
709
710 addPreEmitPass(PMW);
711
712 if (TM.Options.EnableIPRA) {
713 // Collect register usage information and produce a register mask of
714 // clobbered registers, to be used to optimize call sites.
715 addMachineFunctionPass(Pass: RegUsageInfoCollectorPass(), PMW);
716 // If -print-regusage is specified, print the collected register usage info.
717 if (Opt.PrintRegUsage) {
718 flushFPMsToMPM(PMW);
719 addModulePass(Pass: PhysicalRegisterUsageInfoPrinterPass(errs()), PMW);
720 }
721 }
722
723 addMachineFunctionPass(Pass: FuncletLayoutPass(), PMW);
724
725 addMachineFunctionPass(Pass: RemoveLoadsIntoFakeUsesPass(), PMW);
726 addMachineFunctionPass(Pass: StackMapLivenessPass(), PMW);
727 addMachineFunctionPass(Pass: LiveDebugValuesPass(), PMW);
728 addMachineFunctionPass(Pass: MachineSanitizerBinaryMetadataPass(), PMW);
729
730 if (TM.Options.EnableMachineOutliner &&
731 getOptLevel() != CodeGenOptLevel::None &&
732 Opt.EnableMachineOutliner != RunOutliner::NeverOutline) {
733 if (Opt.EnableMachineOutliner != RunOutliner::TargetDefault ||
734 TM.supportsDefaultOutlining()) {
735 flushFPMsToMPM(PMW);
736 addModulePass(Pass: MachineOutlinerPass(Opt.EnableMachineOutliner), PMW);
737 }
738 }
739
740 if (Opt.EnableGCEmptyBlocks)
741 addMachineFunctionPass(Pass: GCEmptyBasicBlocksPass(), PMW);
742
743 addPostBBSections(PMW);
744
745 if (!Opt.DisableCFIFixup && TM.Options.EnableCFIFixup)
746 addMachineFunctionPass(Pass: CFIFixupPass(), PMW);
747
748 addMachineFunctionPass(Pass: StackFrameLayoutAnalysisPass(), PMW);
749
750 // Add passes that directly emit MI after all other MI passes.
751 addPreEmitPass2(PMW);
752
753 return Error::success();
754}
755
756/// Add passes that optimize machine instructions in SSA form.
757void CodeGenPassBuilder::addMachineSSAOptimization(PassManagerWrapper &PMW) {
758 // Pre-ra tail duplication.
759 addMachineFunctionPass(Pass: EarlyTailDuplicatePass(), PMW);
760
761 // Optimize PHIs before DCE: removing dead PHI cycles may make more
762 // instructions dead.
763 addMachineFunctionPass(Pass: OptimizePHIsPass(), PMW);
764
765 // This pass merges large allocas. StackSlotColoring is a different pass
766 // which merges spill slots.
767 addMachineFunctionPass(Pass: StackColoringPass(), PMW);
768
769 // If the target requests it, assign local variables to stack slots relative
770 // to one another and simplify frame index references where possible.
771 addMachineFunctionPass(Pass: LocalStackSlotAllocationPass(), PMW);
772
773 // With optimization, dead code should already be eliminated. However
774 // there is one known exception: lowered code for arguments that are only
775 // used by tail calls, where the tail calls reuse the incoming stack
776 // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
777 addMachineFunctionPass(Pass: DeadMachineInstructionElimPass(), PMW);
778
779 // Allow targets to insert passes that improve instruction level parallelism,
780 // like if-conversion. Such passes will typically need dominator trees and
781 // loop info, just like LICM and CSE below.
782 addILPOpts(PMW);
783
784 addMachineFunctionPass(Pass: EarlyMachineLICMPass(), PMW);
785 addMachineFunctionPass(Pass: MachineCSEPass(), PMW);
786
787 addMachineFunctionPass(Pass: MachineSinkingPass(Opt.EnableSinkAndFold), PMW);
788
789 addMachineFunctionPass(Pass: PeepholeOptimizerPass(), PMW);
790 // Clean-up the dead code that may have been generated by peephole
791 // rewriting.
792 addMachineFunctionPass(Pass: DeadMachineInstructionElimPass(), PMW);
793}
794
795//===---------------------------------------------------------------------===//
796/// Register Allocation Pass Configuration
797//===---------------------------------------------------------------------===//
798
799/// Instantiate the default register allocator pass for this target for either
800/// the optimized or unoptimized allocation path. This will be added to the pass
801/// manager by addFastRegAlloc in the unoptimized case or addOptimizedRegAlloc
802/// in the optimized case.
803///
804/// A target that uses the standard regalloc pass order for fast or optimized
805/// allocation may still override this for per-target regalloc
806/// selection. But -regalloc-npm=... always takes precedence.
807/// If a target does not want to allow users to set -regalloc-npm=... at all,
808/// check if Opt.RegAlloc == RegAllocType::Unset.
809void CodeGenPassBuilder::addTargetRegisterAllocator(PassManagerWrapper &PMW,
810 bool Optimized) {
811 if (Optimized)
812 addMachineFunctionPass(Pass: RAGreedyPass(), PMW);
813 else
814 addMachineFunctionPass(Pass: RegAllocFastPass(), PMW);
815}
816
817/// Find and instantiate the register allocation pass requested by this target
818/// at the current optimization level. Different register allocators are
819/// defined as separate passes because they may require different analysis.
820///
821/// This helper ensures that the -regalloc-npm= option is always available,
822/// even for targets that override the default allocator.
823void CodeGenPassBuilder::addRegAllocPass(PassManagerWrapper &PMW,
824 bool Optimized) {
825 // Use the specified -regalloc-npm={basic|greedy|fast|pbqp}
826 if (Opt.RegAlloc > RegAllocType::Default) {
827 switch (Opt.RegAlloc) {
828 case RegAllocType::Fast:
829 addMachineFunctionPass(Pass: RegAllocFastPass(), PMW);
830 break;
831 case RegAllocType::Greedy:
832 addMachineFunctionPass(Pass: RAGreedyPass(), PMW);
833 break;
834 default:
835 reportFatalUsageError(reason: "register allocator not supported yet");
836 }
837 return;
838 }
839 // -regalloc=default or unspecified, so pick based on the optimization level
840 // or ask the target for the regalloc pass.
841 addTargetRegisterAllocator(PMW, Optimized);
842}
843
844Error CodeGenPassBuilder::addRegAssignAndRewriteFast(PassManagerWrapper &PMW) {
845 // TODO: Ensure allocator is default or fast.
846 addRegAllocPass(PMW, Optimized: false);
847 return Error::success();
848}
849
850Expected<bool>
851CodeGenPassBuilder::addRegAssignAndRewriteOptimized(PassManagerWrapper &PMW) {
852 // Add the selected register allocation pass.
853 addRegAllocPass(PMW, Optimized: true);
854
855 // Allow targets to change the register assignments before rewriting.
856 addPreRewrite(PMW);
857
858 // Finally rewrite virtual registers.
859 addMachineFunctionPass(Pass: VirtRegRewriterPass(), PMW);
860
861 return true;
862}
863
864/// Add the minimum set of target-independent passes that are required for
865/// register allocation. No coalescing or scheduling.
866Error CodeGenPassBuilder::addFastRegAlloc(PassManagerWrapper &PMW) {
867 addMachineFunctionPass(Pass: PHIEliminationPass(), PMW);
868 if (!TM.enableTiedFastRegAlloc())
869 addMachineFunctionPass(Pass: TwoAddressInstructionPass(), PMW);
870 return addRegAssignAndRewriteFast(PMW);
871}
872
873/// Add standard target-independent passes that are tightly coupled with
874/// optimized register allocation, including coalescing, machine instruction
875/// scheduling, and register allocation itself.
876Error CodeGenPassBuilder::addOptimizedRegAlloc(PassManagerWrapper &PMW) {
877 addMachineFunctionPass(Pass: DetectDeadLanesPass(), PMW);
878
879 addMachineFunctionPass(Pass: InitUndefPass(), PMW);
880
881 addMachineFunctionPass(Pass: ProcessImplicitDefsPass(), PMW);
882
883 // LiveIntervals requires pure SSA form and no unreachable blocks.
884 // Explicit addition of UnreachableMachineBlockElim allows stopping before or
885 // after it with -stop-before/-stop-after.
886 addMachineFunctionPass(Pass: UnreachableMachineBlockElimPass(), PMW);
887
888 // If enabled, run the SSA machine scheduler just before PHI elimination.
889 if (Opt.EnableSSAMachineScheduler) {
890 addMachineFunctionPass(
891 Pass: RequireAnalysisPass<LiveIntervalsAnalysis, MachineFunction>(), PMW);
892 addMachineFunctionPass(Pass: SSAMachineSchedulerPass(&TM), PMW);
893 }
894
895 addMachineFunctionPass(
896 Pass: RequireAnalysisPass<LiveIntervalsAnalysis, MachineFunction>(), PMW);
897
898 // Edge splitting is smarter with machine loop info.
899 addMachineFunctionPass(
900 Pass: RequireAnalysisPass<MachineLoopAnalysis, MachineFunction>(), PMW);
901 addMachineFunctionPass(Pass: PHIEliminationPass(), PMW);
902
903 addMachineFunctionPass(Pass: TwoAddressInstructionPass(), PMW);
904 addMachineFunctionPass(Pass: RegisterCoalescerPass(), PMW);
905
906 // The machine scheduler may accidentally create disconnected components
907 // when moving subregister definitions around, avoid this by splitting them to
908 // separate vregs before. Splitting can also improve reg. allocation quality.
909 addMachineFunctionPass(Pass: RenameIndependentSubregsPass(), PMW);
910
911 // PreRA instruction scheduling.
912 addMachineFunctionPass(Pass: MachineSchedulerPass(&TM), PMW);
913
914 Expected<bool> AddedPasses = addRegAssignAndRewriteOptimized(PMW);
915 if (!AddedPasses)
916 return AddedPasses.takeError();
917 if (!AddedPasses.get())
918 return Error::success();
919
920 addMachineFunctionPass(Pass: StackSlotColoringPass(), PMW);
921
922 // Allow targets to expand pseudo instructions depending on the choice of
923 // registers before MachineCopyPropagation.
924 addPostRewrite(PMW);
925
926 // Copy propagate to forward register uses and try to eliminate COPYs that
927 // were not coalesced.
928 addMachineFunctionPass(Pass: MachineCopyPropagationPass(), PMW);
929
930 // Run post-ra machine LICM to hoist reloads / remats.
931 //
932 // FIXME: can this move into MachineLateOptimization?
933 addMachineFunctionPass(Pass: MachineLICMPass(), PMW);
934
935 return Error::success();
936}
937
938//===---------------------------------------------------------------------===//
939/// Post RegAlloc Pass Configuration
940//===---------------------------------------------------------------------===//
941
942/// Add passes that optimize machine instructions after register allocation.
943void CodeGenPassBuilder::addMachineLateOptimization(PassManagerWrapper &PMW) {
944 // Cleanup of redundant (identical) address/immediate loads.
945 addMachineFunctionPass(Pass: MachineLateInstrsCleanupPass(), PMW);
946
947 // Branch folding must be run after regalloc and prolog/epilog insertion.
948 addMachineFunctionPass(Pass: BranchFolderPass(Opt.EnableTailMerge), PMW);
949
950 // Tail duplication.
951 // Note that duplicating tail just increases code size and degrades
952 // performance for targets that require Structured Control Flow.
953 // In addition it can also make CFG irreducible. Thus we disable it.
954 if (!TM.requiresStructuredCFG())
955 addMachineFunctionPass(Pass: TailDuplicatePass(), PMW);
956
957 // Copy propagation.
958 addMachineFunctionPass(Pass: MachineCopyPropagationPass(), PMW);
959}
960
961/// Add standard basic block placement passes.
962void CodeGenPassBuilder::addBlockPlacement(PassManagerWrapper &PMW) {
963 addMachineFunctionPass(Pass: MachineBlockPlacementPass(Opt.EnableTailMerge), PMW);
964 // Run a separate pass to collect block placement statistics.
965 if (Opt.EnableBlockPlacementStats)
966 addMachineFunctionPass(Pass: MachineBlockPlacementStatsPass(), PMW);
967}
968