1//===- Construction of pass pipelines -------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9///
10/// This file provides the implementation of the PassBuilder based on our
11/// static pass registry as well as related functionality. It also provides
12/// helpers to aid in analyzing, debugging, and testing passes and pass
13/// pipelines.
14///
15//===----------------------------------------------------------------------===//
16
17#include "PassesOptions.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/Analysis/AliasAnalysis.h"
20#include "llvm/Analysis/BasicAliasAnalysis.h"
21#include "llvm/Analysis/CGSCCPassManager.h"
22#include "llvm/Analysis/CallGraphSCCPass.h"
23#include "llvm/Analysis/FunctionPropertiesAnalysis.h"
24#include "llvm/Analysis/GlobalsModRef.h"
25#include "llvm/Analysis/InlineAdvisor.h"
26#include "llvm/Analysis/InstCount.h"
27#include "llvm/Analysis/ProfileSummaryInfo.h"
28#include "llvm/Analysis/ScopedNoAliasAA.h"
29#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
30#include "llvm/IR/PassManager.h"
31#include "llvm/IR/Verifier.h"
32#include "llvm/Pass.h"
33#include "llvm/Passes/OptimizationLevel.h"
34#include "llvm/Passes/PassBuilder.h"
35#include "llvm/Passes/TriggerCrashPasses.h"
36#include "llvm/Support/CommandLine.h"
37#include "llvm/Support/ErrorHandling.h"
38#include "llvm/Support/PGOOptions.h"
39#include "llvm/Target/TargetMachine.h"
40#include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
41#include "llvm/Transforms/Coroutines/CoroAnnotationElide.h"
42#include "llvm/Transforms/Coroutines/CoroCleanup.h"
43#include "llvm/Transforms/Coroutines/CoroConditionalWrapper.h"
44#include "llvm/Transforms/Coroutines/CoroEarly.h"
45#include "llvm/Transforms/Coroutines/CoroElide.h"
46#include "llvm/Transforms/Coroutines/CoroSplit.h"
47#include "llvm/Transforms/HipStdPar/HipStdPar.h"
48#include "llvm/Transforms/IPO/AlwaysInliner.h"
49#include "llvm/Transforms/IPO/Annotation2Metadata.h"
50#include "llvm/Transforms/IPO/ArgumentPromotion.h"
51#include "llvm/Transforms/IPO/Attributor.h"
52#include "llvm/Transforms/IPO/CalledValuePropagation.h"
53#include "llvm/Transforms/IPO/ConstantMerge.h"
54#include "llvm/Transforms/IPO/CrossDSOCFI.h"
55#include "llvm/Transforms/IPO/DeadArgumentElimination.h"
56#include "llvm/Transforms/IPO/ElimAvailExtern.h"
57#include "llvm/Transforms/IPO/EmbedBitcodePass.h"
58#include "llvm/Transforms/IPO/ExpandVariadics.h"
59#include "llvm/Transforms/IPO/FatLTOCleanup.h"
60#include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
61#include "llvm/Transforms/IPO/FunctionAttrs.h"
62#include "llvm/Transforms/IPO/GlobalDCE.h"
63#include "llvm/Transforms/IPO/GlobalOpt.h"
64#include "llvm/Transforms/IPO/GlobalSplit.h"
65#include "llvm/Transforms/IPO/HotColdSplitting.h"
66#include "llvm/Transforms/IPO/InferFunctionAttrs.h"
67#include "llvm/Transforms/IPO/Inliner.h"
68#include "llvm/Transforms/IPO/Instrumentor.h"
69#include "llvm/Transforms/IPO/LowerTypeTests.h"
70#include "llvm/Transforms/IPO/MemProfContextDisambiguation.h"
71#include "llvm/Transforms/IPO/MergeFunctions.h"
72#include "llvm/Transforms/IPO/ModuleInliner.h"
73#include "llvm/Transforms/IPO/OpenMPOpt.h"
74#include "llvm/Transforms/IPO/PartialInlining.h"
75#include "llvm/Transforms/IPO/SCCP.h"
76#include "llvm/Transforms/IPO/SampleProfile.h"
77#include "llvm/Transforms/IPO/SampleProfileProbe.h"
78#include "llvm/Transforms/IPO/WholeProgramDevirt.h"
79#include "llvm/Transforms/InstCombine/InstCombine.h"
80#include "llvm/Transforms/Instrumentation/AllocToken.h"
81#include "llvm/Transforms/Instrumentation/CGProfile.h"
82#include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
83#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
84#include "llvm/Transforms/Instrumentation/MemProfInstrumentation.h"
85#include "llvm/Transforms/Instrumentation/MemProfUse.h"
86#include "llvm/Transforms/Instrumentation/PGOCtxProfFlattening.h"
87#include "llvm/Transforms/Instrumentation/PGOCtxProfLowering.h"
88#include "llvm/Transforms/Instrumentation/PGOForceFunctionAttrs.h"
89#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
90#include "llvm/Transforms/Scalar/ADCE.h"
91#include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
92#include "llvm/Transforms/Scalar/AnnotationRemarks.h"
93#include "llvm/Transforms/Scalar/BDCE.h"
94#include "llvm/Transforms/Scalar/CallSiteSplitting.h"
95#include "llvm/Transforms/Scalar/ConstraintElimination.h"
96#include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
97#include "llvm/Transforms/Scalar/DFAJumpThreading.h"
98#include "llvm/Transforms/Scalar/DeadStoreElimination.h"
99#include "llvm/Transforms/Scalar/DivRemPairs.h"
100#include "llvm/Transforms/Scalar/DropUnnecessaryAssumes.h"
101#include "llvm/Transforms/Scalar/EarlyCSE.h"
102#include "llvm/Transforms/Scalar/ExpandMemCmp.h"
103#include "llvm/Transforms/Scalar/Float2Int.h"
104#include "llvm/Transforms/Scalar/GVN.h"
105#include "llvm/Transforms/Scalar/GVNHoist.h"
106#include "llvm/Transforms/Scalar/GVNSink.h"
107#include "llvm/Transforms/Scalar/IndVarSimplify.h"
108#include "llvm/Transforms/Scalar/InferAlignment.h"
109#include "llvm/Transforms/Scalar/InstSimplifyPass.h"
110#include "llvm/Transforms/Scalar/JumpTableToSwitch.h"
111#include "llvm/Transforms/Scalar/JumpThreading.h"
112#include "llvm/Transforms/Scalar/LICM.h"
113#include "llvm/Transforms/Scalar/LoopDeletion.h"
114#include "llvm/Transforms/Scalar/LoopDistribute.h"
115#include "llvm/Transforms/Scalar/LoopFlatten.h"
116#include "llvm/Transforms/Scalar/LoopFuse.h"
117#include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
118#include "llvm/Transforms/Scalar/LoopInstSimplify.h"
119#include "llvm/Transforms/Scalar/LoopInterchange.h"
120#include "llvm/Transforms/Scalar/LoopLoadElimination.h"
121#include "llvm/Transforms/Scalar/LoopPassManager.h"
122#include "llvm/Transforms/Scalar/LoopRotation.h"
123#include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
124#include "llvm/Transforms/Scalar/LoopSink.h"
125#include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
126#include "llvm/Transforms/Scalar/LoopUnrollPass.h"
127#include "llvm/Transforms/Scalar/LoopVersioningLICM.h"
128#include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
129#include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
130#include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h"
131#include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
132#include "llvm/Transforms/Scalar/MergeICmps.h"
133#include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"
134#include "llvm/Transforms/Scalar/NewGVN.h"
135#include "llvm/Transforms/Scalar/Reassociate.h"
136#include "llvm/Transforms/Scalar/SCCP.h"
137#include "llvm/Transforms/Scalar/SROA.h"
138#include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
139#include "llvm/Transforms/Scalar/SimplifyCFG.h"
140#include "llvm/Transforms/Scalar/SpeculativeExecution.h"
141#include "llvm/Transforms/Scalar/TailRecursionElimination.h"
142#include "llvm/Transforms/Scalar/WarnMissedTransforms.h"
143#include "llvm/Transforms/Utils/AddDiscriminators.h"
144#include "llvm/Transforms/Utils/AssignGUID.h"
145#include "llvm/Transforms/Utils/CanonicalizeAliases.h"
146#include "llvm/Transforms/Utils/CountVisits.h"
147#include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
148#include "llvm/Transforms/Utils/ExtraPassManager.h"
149#include "llvm/Transforms/Utils/InjectTLIMappings.h"
150#include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
151#include "llvm/Transforms/Utils/LowerCommentStringPass.h"
152#include "llvm/Transforms/Utils/Mem2Reg.h"
153#include "llvm/Transforms/Utils/MoveAutoInit.h"
154#include "llvm/Transforms/Utils/NameAnonGlobals.h"
155#include "llvm/Transforms/Utils/ProfileVerify.h"
156#include "llvm/Transforms/Utils/RelLookupTableConverter.h"
157#include "llvm/Transforms/Utils/SimplifyCFGOptions.h"
158#include "llvm/Transforms/Vectorize/LoopVectorize.h"
159#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
160#include "llvm/Transforms/Vectorize/VectorCombine.h"
161
162using namespace llvm;
163
164namespace llvm {
165extern cl::opt<std::string> UseCtxProfile;
166
167extern cl::opt<bool> EnableMemProfContextDisambiguation;
168
169extern cl::opt<bool> EnableProfcheckInPipeline;
170} // namespace llvm
171
172PipelineTuningOptions PipelineTuningOptions::resolve(
173 const cl::LibraryOptionsContext &LibraryOptions) const {
174 const PassesOptions &Opts = LibraryOptions.get<PassesOptions>();
175 PipelineTuningOptions R = *this;
176 R.ForgetAllSCEVInLoopUnroll = ForgetAllSCEVInLoopUnroll.value_or(
177 u: getForgetSCEVInLoopUnroll(LibraryOptions));
178 R.LicmMssaOptCap = LicmMssaOptCap.value_or(u: getLicmMssaOptCap(LibraryOptions));
179 R.LicmMssaNoAccForPromotionCap = LicmMssaNoAccForPromotionCap.value_or(
180 u: getLicmMssaNoAccForPromotionCap(LibraryOptions));
181 R.MergeFunctions = MergeFunctions.value_or(u: Opts.enable_merge_functions);
182 R.MaxDevirtIterations =
183 MaxDevirtIterations.value_or(u: getMaxDevirtIterations(LibraryOptions));
184 R.EagerlyInvalidateAnalyses =
185 EagerlyInvalidateAnalyses.value_or(u: Opts.eagerly_invalidate_analyses);
186 R.DevirtualizeSpeculatively = DevirtualizeSpeculatively.value_or(
187 u: Opts.enable_devirtualize_speculatively);
188 return R;
189}
190
191void PassBuilder::invokePeepholeEPCallbacks(FunctionPassManager &FPM,
192 OptimizationLevel Level) {
193 for (auto &C : PeepholeEPCallbacks)
194 C(FPM, Level);
195}
196void PassBuilder::invokeLateLoopOptimizationsEPCallbacks(
197 LoopPassManager &LPM, OptimizationLevel Level) {
198 for (auto &C : LateLoopOptimizationsEPCallbacks)
199 C(LPM, Level);
200}
201void PassBuilder::invokeLoopOptimizerEndEPCallbacks(LoopPassManager &LPM,
202 OptimizationLevel Level) {
203 for (auto &C : LoopOptimizerEndEPCallbacks)
204 C(LPM, Level);
205}
206void PassBuilder::invokeScalarOptimizerLateEPCallbacks(
207 FunctionPassManager &FPM, OptimizationLevel Level) {
208 for (auto &C : ScalarOptimizerLateEPCallbacks)
209 C(FPM, Level);
210}
211void PassBuilder::invokeCGSCCOptimizerLateEPCallbacks(CGSCCPassManager &CGPM,
212 OptimizationLevel Level) {
213 for (auto &C : CGSCCOptimizerLateEPCallbacks)
214 C(CGPM, Level);
215}
216void PassBuilder::invokeVectorizerStartEPCallbacks(FunctionPassManager &FPM,
217 OptimizationLevel Level) {
218 for (auto &C : VectorizerStartEPCallbacks)
219 C(FPM, Level);
220}
221void PassBuilder::invokeVectorizerEndEPCallbacks(FunctionPassManager &FPM,
222 OptimizationLevel Level) {
223 for (auto &C : VectorizerEndEPCallbacks)
224 C(FPM, Level);
225}
226void PassBuilder::invokeOptimizerEarlyEPCallbacks(ModulePassManager &MPM,
227 OptimizationLevel Level,
228 ThinOrFullLTOPhase Phase) {
229 for (auto &C : OptimizerEarlyEPCallbacks)
230 C(MPM, Level, Phase);
231}
232void PassBuilder::invokeOptimizerLastEPCallbacks(ModulePassManager &MPM,
233 OptimizationLevel Level,
234 ThinOrFullLTOPhase Phase) {
235 for (auto &C : OptimizerLastEPCallbacks)
236 C(MPM, Level, Phase);
237}
238void PassBuilder::invokeFullLinkTimeOptimizationEarlyEPCallbacks(
239 ModulePassManager &MPM, OptimizationLevel Level) {
240 for (auto &C : FullLinkTimeOptimizationEarlyEPCallbacks)
241 C(MPM, Level);
242}
243void PassBuilder::invokeFullLinkTimeOptimizationLastEPCallbacks(
244 ModulePassManager &MPM, OptimizationLevel Level) {
245 for (auto &C : FullLinkTimeOptimizationLastEPCallbacks)
246 C(MPM, Level);
247}
248void PassBuilder::invokeThinLinkTimeOptimizationEarlyEPCallbacks(
249 ModulePassManager &MPM, OptimizationLevel Level) {
250 for (auto &C : ThinLinkTimeOptimizationEarlyEPCallbacks)
251 C(MPM, Level);
252}
253void PassBuilder::invokeThinLinkTimeOptimizationLastEPCallbacks(
254 ModulePassManager &MPM, OptimizationLevel Level) {
255 for (auto &C : ThinLinkTimeOptimizationLastEPCallbacks)
256 C(MPM, Level);
257}
258void PassBuilder::invokePipelineStartEPCallbacks(ModulePassManager &MPM,
259 OptimizationLevel Level) {
260 for (auto &C : PipelineStartEPCallbacks)
261 C(MPM, Level);
262}
263void PassBuilder::invokePipelineEarlySimplificationEPCallbacks(
264 ModulePassManager &MPM, OptimizationLevel Level, ThinOrFullLTOPhase Phase) {
265 for (auto &C : PipelineEarlySimplificationEPCallbacks)
266 C(MPM, Level, Phase);
267}
268
269// Get IR stats with InstCount before/after the optimization pipeline
270static void instructionCountersPass(ModulePassManager &MPM,
271 bool IsPreOptimization) {
272 if (AreStatisticsEnabled()) {
273 MPM.addPass(
274 Pass: createModuleToFunctionPassAdaptor(Pass: InstCountPass(IsPreOptimization)));
275 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
276 Pass: FunctionPropertiesStatisticsPass(IsPreOptimization)));
277 }
278}
279
280// Add passes to run at the start of the pipeline. This includes things like
281// statistics passes.
282static void addPipelineStartPasses(ModulePassManager &MPM,
283 const PassesOptions &Opts) {
284 instructionCountersPass(MPM, /*IsPreOptimization=*/true);
285 if (EnableProfcheckInPipeline)
286 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: ProfileInjectorPass()));
287}
288
289// Add passes to run at the end of the pipeline. This includes things like
290// statistics passes.
291static void addPipelineEndPasses(ModulePassManager &MPM,
292 const PassesOptions &Opts) {
293 instructionCountersPass(MPM, /*IsPreOptimization=*/false);
294 if (EnableProfcheckInPipeline)
295 MPM.addPass(Pass: ProfileVerifierPass());
296}
297
298// Helper to add AnnotationRemarksPass.
299static void addAnnotationRemarksPass(ModulePassManager &MPM) {
300 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: AnnotationRemarksPass()));
301}
302
303// Helper to check if the current compilation phase is preparing for LTO
304static bool isLTOPreLink(ThinOrFullLTOPhase Phase) {
305 return Phase == ThinOrFullLTOPhase::ThinLTOPreLink ||
306 Phase == ThinOrFullLTOPhase::FullLTOPreLink;
307}
308
309// Helper to check if the current compilation phase is preparing for FullLTO
310[[maybe_unused]] static bool isFullLTOPreLink(ThinOrFullLTOPhase Phase) {
311 return Phase == ThinOrFullLTOPhase::FullLTOPreLink;
312}
313
314// Helper to check if the current compilation phase is preparing for ThinLTO
315static bool isThinLTOPreLink(ThinOrFullLTOPhase Phase) {
316 return Phase == ThinOrFullLTOPhase::ThinLTOPreLink;
317}
318
319// Helper to check if the current compilation phase is LTO backend
320static bool isLTOPostLink(ThinOrFullLTOPhase Phase) {
321 return Phase == ThinOrFullLTOPhase::ThinLTOPostLink ||
322 Phase == ThinOrFullLTOPhase::FullLTOPostLink;
323}
324
325// Helper to check if the current compilation phase is FullLTO backend
326static bool isFullLTOPostLink(ThinOrFullLTOPhase Phase) {
327 return Phase == ThinOrFullLTOPhase::FullLTOPostLink;
328}
329
330// Helper to check if the current compilation phase is ThinLTO backend
331static bool isThinLTOPostLink(ThinOrFullLTOPhase Phase) {
332 return Phase == ThinOrFullLTOPhase::ThinLTOPostLink;
333}
334
335// Helper to wrap conditionally Coro passes.
336static CoroConditionalWrapper buildCoroWrapper(ThinOrFullLTOPhase Phase) {
337 // TODO: Skip passes according to Phase.
338 ModulePassManager CoroPM;
339 CoroPM.addPass(Pass: CoroEarlyPass());
340 CGSCCPassManager CGPM;
341 CGPM.addPass(Pass: CoroSplitPass());
342 CoroPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: std::move(CGPM)));
343 CoroPM.addPass(Pass: CoroCleanupPass());
344 CoroPM.addPass(Pass: GlobalDCEPass());
345 return CoroConditionalWrapper(std::move(CoroPM));
346}
347
348static InlineParams getInlineParamsFromOptLevel(const LLVMContext &Context,
349 OptimizationLevel Level) {
350 return getInlineParamsFromOptLevel(C: Context, OptLevel: static_cast<unsigned>(Level));
351}
352
353static void addModuleInlinerPass(ModulePassManager &MPM,
354 const LLVMContext &Context,
355 OptimizationLevel Level,
356 ThinOrFullLTOPhase Phase) {
357 const PassesOptions &Opts = getPassesOptions(C: Context);
358 InlineParams IP = ::getInlineParamsFromOptLevel(Context, Level);
359 if (Opts.enable_module_inliner)
360 MPM.addPass(Pass: ModuleInlinerPass(IP, Opts.enable_ml_inliner, Phase));
361 else
362 MPM.addPass(Pass: ModuleInlinerWrapperPass(
363 IP,
364 /* MandatoryFirst */ true,
365 InlineContext{.LTOPhase: Phase, .Pass: InlinePass::CGSCCInliner}));
366}
367
368// TODO: Investigate the cost/benefit of tail call elimination on debugging.
369FunctionPassManager
370PassBuilder::buildO1FunctionSimplificationPipeline(OptimizationLevel Level,
371 ThinOrFullLTOPhase Phase) {
372
373 FunctionPassManager FPM;
374
375 if (AreStatisticsEnabled())
376 FPM.addPass(Pass: CountVisitsPass());
377
378 // Form SSA out of local memory accesses after breaking apart aggregates into
379 // scalars.
380 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
381
382 // Catch trivial redundancies
383 FPM.addPass(Pass: EarlyCSEPass(true /* Enable mem-ssa. */));
384
385 // Hoisting of scalars and load expressions.
386 FPM.addPass(
387 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
388 FPM.addPass(Pass: InstCombinePass());
389
390 FPM.addPass(Pass: LibCallsShrinkWrapPass());
391
392 invokePeepholeEPCallbacks(FPM, Level);
393
394 FPM.addPass(
395 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
396
397 // Form canonically associated expression trees, and simplify the trees using
398 // basic mathematical properties. For example, this will form (nearly)
399 // minimal multiplication trees.
400 FPM.addPass(Pass: ReassociatePass());
401
402 // Add the primary loop simplification pipeline.
403 // FIXME: Currently this is split into two loop pass pipelines because we run
404 // some function passes in between them. These can and should be removed
405 // and/or replaced by scheduling the loop pass equivalents in the correct
406 // positions. But those equivalent passes aren't powerful enough yet.
407 // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
408 // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
409 // fully replace `SimplifyCFGPass`, and the closest to the other we have is
410 // `LoopInstSimplify`.
411 LoopPassManager LPM1, LPM2;
412
413 // Simplify the loop body. We do this initially to clean up after other loop
414 // passes run, either when iterating on a loop or on inner loops with
415 // implications on the outer loop.
416 LPM1.addPass(Pass: LoopInstSimplifyPass());
417 LPM1.addPass(Pass: LoopSimplifyCFGPass());
418
419 // Try to remove as much code from the loop header as possible,
420 // to reduce amount of IR that will have to be duplicated. However,
421 // do not perform speculative hoisting the first time as LICM
422 // will destroy metadata that may not need to be destroyed if run
423 // after loop rotation.
424 // TODO: Investigate promotion cap for O1.
425 LPM1.addPass(Pass: LICMPass(*PTO.LicmMssaOptCap, *PTO.LicmMssaNoAccForPromotionCap,
426 /*AllowSpeculation=*/false));
427
428 LPM1.addPass(
429 Pass: LoopRotatePass(/*EnableHeaderDuplication=*/true, isLTOPreLink(Phase)));
430 // TODO: Investigate promotion cap for O1.
431 LPM1.addPass(Pass: LICMPass(*PTO.LicmMssaOptCap, *PTO.LicmMssaNoAccForPromotionCap,
432 /*AllowSpeculation=*/true));
433 LPM1.addPass(Pass: SimpleLoopUnswitchPass());
434 if (Opts.enable_loop_flatten)
435 LPM1.addPass(Pass: LoopFlattenPass());
436
437 LPM2.addPass(Pass: LoopIdiomRecognizePass());
438 LPM2.addPass(Pass: IndVarSimplifyPass());
439
440 invokeLateLoopOptimizationsEPCallbacks(LPM&: LPM2, Level);
441
442 LPM2.addPass(Pass: LoopDeletionPass());
443
444 // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
445 // because it changes IR to makes profile annotation in back compile
446 // inaccurate. The normal unroller doesn't pay attention to forced full unroll
447 // attributes so we need to make sure and allow the full unroll pass to pay
448 // attention to it.
449 if (!isThinLTOPreLink(Phase) || !PGOOpt ||
450 PGOOpt->Action != PGOOptions::SampleUse)
451 LPM2.addPass(Pass: LoopFullUnrollPass(static_cast<int>(Level),
452 /* OnlyWhenForced= */ !PTO.LoopUnrolling,
453 *PTO.ForgetAllSCEVInLoopUnroll));
454
455 invokeLoopOptimizerEndEPCallbacks(LPM&: LPM2, Level);
456
457 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM1),
458 /*UseMemorySSA=*/true));
459 FPM.addPass(
460 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
461 FPM.addPass(Pass: InstCombinePass());
462 // The loop passes in LPM2 (LoopFullUnrollPass) do not preserve MemorySSA.
463 // *All* loop passes must preserve it, in order to be able to use it.
464 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM2),
465 /*UseMemorySSA=*/false));
466
467 // Delete small array after loop unroll.
468 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
469
470 // Specially optimize memory movement as it doesn't look like dataflow in SSA.
471 FPM.addPass(Pass: MemCpyOptPass());
472
473 // Sparse conditional constant propagation.
474 // FIXME: It isn't clear why we do this *after* loop passes rather than
475 // before...
476 FPM.addPass(Pass: SCCPPass());
477
478 // Delete dead bit computations (instcombine runs after to fold away the dead
479 // computations, and then ADCE will run later to exploit any new DCE
480 // opportunities that creates).
481 FPM.addPass(Pass: BDCEPass());
482
483 // Run instcombine after redundancy and dead bit elimination to exploit
484 // opportunities opened up by them.
485 FPM.addPass(Pass: InstCombinePass());
486 invokePeepholeEPCallbacks(FPM, Level);
487
488 FPM.addPass(Pass: CoroElidePass());
489
490 invokeScalarOptimizerLateEPCallbacks(FPM, Level);
491
492 // Finally, do an expensive DCE pass to catch all the dead code exposed by
493 // the simplifications and basic cleanup after all the simplifications.
494 // TODO: Investigate if this is too expensive.
495 FPM.addPass(Pass: ADCEPass());
496 FPM.addPass(
497 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
498 FPM.addPass(Pass: InstCombinePass());
499 invokePeepholeEPCallbacks(FPM, Level);
500
501 return FPM;
502}
503
504FunctionPassManager
505PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
506 ThinOrFullLTOPhase Phase) {
507 assert(Level != OptimizationLevel::O0 && "Must request optimizations!");
508
509 // The O1 pipeline has a separate pipeline creation function to simplify
510 // construction readability.
511 if (Level == OptimizationLevel::O1)
512 return buildO1FunctionSimplificationPipeline(Level, Phase);
513
514 FunctionPassManager FPM;
515
516 if (AreStatisticsEnabled())
517 FPM.addPass(Pass: CountVisitsPass());
518
519 // Form SSA out of local memory accesses after breaking apart aggregates into
520 // scalars.
521 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
522
523 // Catch trivial redundancies
524 FPM.addPass(Pass: EarlyCSEPass(true /* Enable mem-ssa. */));
525
526 // Hoisting of scalars and load expressions.
527 if (Opts.enable_gvn_hoist)
528 FPM.addPass(Pass: GVNHoistPass());
529
530 // Global value numbering based sinking.
531 if (Opts.enable_gvn_sink) {
532 FPM.addPass(Pass: GVNSinkPass());
533 FPM.addPass(
534 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
535 }
536
537 // Speculative execution if the target has divergent branches; otherwise nop.
538 FPM.addPass(Pass: SpeculativeExecutionPass(/* OnlyIfDivergentTarget =*/true));
539
540 // Optimize based on known information about branches, and cleanup afterward.
541 FPM.addPass(Pass: JumpThreadingPass());
542 FPM.addPass(Pass: CorrelatedValuePropagationPass());
543
544 // Jump table to switch conversion.
545 if (Opts.enable_jump_table_to_switch)
546 FPM.addPass(Pass: JumpTableToSwitchPass(/*InLTO=*/isLTOPostLink(Phase)));
547
548 FPM.addPass(
549 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
550 FPM.addPass(Pass: InstCombinePass());
551 FPM.addPass(Pass: AggressiveInstCombinePass());
552 FPM.addPass(Pass: LibCallsShrinkWrapPass());
553
554 invokePeepholeEPCallbacks(FPM, Level);
555
556 // For PGO use pipeline, try to optimize memory intrinsics such as memcpy
557 // using the size value profile. Don't perform this when optimizing for size.
558 if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse)
559 FPM.addPass(Pass: PGOMemOPSizeOpt());
560
561 FPM.addPass(Pass: TailCallElimPass(/*UpdateFunctionEntryCount=*/
562 isInstrumentedPGOUse()));
563 FPM.addPass(
564 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
565
566 // Form canonically associated expression trees, and simplify the trees using
567 // basic mathematical properties. For example, this will form (nearly)
568 // minimal multiplication trees.
569 FPM.addPass(Pass: ReassociatePass());
570
571 if (Opts.enable_constraint_elimination)
572 FPM.addPass(Pass: ConstraintEliminationPass());
573
574 // Add the primary loop simplification pipeline.
575 // FIXME: Currently this is split into two loop pass pipelines because we run
576 // some function passes in between them. These can and should be removed
577 // and/or replaced by scheduling the loop pass equivalents in the correct
578 // positions. But those equivalent passes aren't powerful enough yet.
579 // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
580 // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
581 // fully replace `SimplifyCFGPass`, and the closest to the other we have is
582 // `LoopInstSimplify`.
583 LoopPassManager LPM1, LPM2;
584
585 // Simplify the loop body. We do this initially to clean up after other loop
586 // passes run, either when iterating on a loop or on inner loops with
587 // implications on the outer loop.
588 LPM1.addPass(Pass: LoopInstSimplifyPass());
589 LPM1.addPass(Pass: LoopSimplifyCFGPass());
590
591 // Try to remove as much code from the loop header as possible,
592 // to reduce amount of IR that will have to be duplicated. However,
593 // do not perform speculative hoisting the first time as LICM
594 // will destroy metadata that may not need to be destroyed if run
595 // after loop rotation.
596 // TODO: Investigate promotion cap for O1.
597 LPM1.addPass(Pass: LICMPass(*PTO.LicmMssaOptCap, *PTO.LicmMssaNoAccForPromotionCap,
598 /*AllowSpeculation=*/false));
599
600 LPM1.addPass(
601 Pass: LoopRotatePass(/*EnableHeaderDuplication=*/true, isLTOPreLink(Phase)));
602 // TODO: Investigate promotion cap for O1.
603 LPM1.addPass(Pass: LICMPass(*PTO.LicmMssaOptCap, *PTO.LicmMssaNoAccForPromotionCap,
604 /*AllowSpeculation=*/true));
605 LPM1.addPass(
606 Pass: SimpleLoopUnswitchPass(/* NonTrivial */ Level == OptimizationLevel::O3));
607 if (Opts.enable_loop_flatten)
608 LPM1.addPass(Pass: LoopFlattenPass());
609
610 LPM2.addPass(Pass: LoopIdiomRecognizePass());
611 LPM2.addPass(Pass: IndVarSimplifyPass());
612
613 {
614 ExtraLoopPassManager<ShouldRunExtraSimpleLoopUnswitch> ExtraPasses;
615 ExtraPasses.addPass(Pass: SimpleLoopUnswitchPass(/* NonTrivial */ Level ==
616 OptimizationLevel::O3));
617 LPM2.addPass(Pass: std::move(ExtraPasses));
618 }
619
620 invokeLateLoopOptimizationsEPCallbacks(LPM&: LPM2, Level);
621
622 LPM2.addPass(Pass: LoopDeletionPass());
623
624 // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
625 // because it changes IR to makes profile annotation in back compile
626 // inaccurate. The normal unroller doesn't pay attention to forced full unroll
627 // attributes so we need to make sure and allow the full unroll pass to pay
628 // attention to it.
629 if (!isThinLTOPreLink(Phase) || !PGOOpt ||
630 PGOOpt->Action != PGOOptions::SampleUse)
631 LPM2.addPass(Pass: LoopFullUnrollPass(static_cast<int>(Level),
632 /* OnlyWhenForced= */ !PTO.LoopUnrolling,
633 *PTO.ForgetAllSCEVInLoopUnroll));
634
635 invokeLoopOptimizerEndEPCallbacks(LPM&: LPM2, Level);
636
637 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM1),
638 /*UseMemorySSA=*/true));
639 FPM.addPass(
640 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
641 FPM.addPass(Pass: InstCombinePass());
642 // The loop passes in LPM2 (LoopIdiomRecognizePass, IndVarSimplifyPass,
643 // LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA.
644 // *All* loop passes must preserve it, in order to be able to use it.
645 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM2),
646 /*UseMemorySSA=*/false));
647
648 // Delete small array after loop unroll.
649 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
650
651 // Try vectorization/scalarization transforms that are both improvements
652 // themselves and can allow further folds with GVN and InstCombine.
653 FPM.addPass(Pass: VectorCombinePass(/*TryEarlyFoldsOnly=*/true));
654
655 // Eliminate redundancies.
656 FPM.addPass(Pass: MergedLoadStoreMotionPass());
657 if (Opts.enable_newgvn)
658 FPM.addPass(Pass: NewGVNPass());
659 else
660 FPM.addPass(Pass: GVNPass());
661
662 // Sparse conditional constant propagation.
663 // FIXME: It isn't clear why we do this *after* loop passes rather than
664 // before...
665 FPM.addPass(Pass: SCCPPass());
666
667 // Delete dead bit computations (instcombine runs after to fold away the dead
668 // computations, and then ADCE will run later to exploit any new DCE
669 // opportunities that creates).
670 FPM.addPass(Pass: BDCEPass());
671
672 // Run instcombine after redundancy and dead bit elimination to exploit
673 // opportunities opened up by them.
674 FPM.addPass(Pass: InstCombinePass());
675 invokePeepholeEPCallbacks(FPM, Level);
676
677 // Re-consider control flow based optimizations after redundancy elimination,
678 // redo DCE, etc.
679 if (Opts.enable_dfa_jump_thread)
680 FPM.addPass(Pass: DFAJumpThreadingPass());
681
682 FPM.addPass(Pass: JumpThreadingPass());
683 FPM.addPass(Pass: CorrelatedValuePropagationPass());
684
685 // Finally, do an expensive DCE pass to catch all the dead code exposed by
686 // the simplifications and basic cleanup after all the simplifications.
687 // TODO: Investigate if this is too expensive.
688 FPM.addPass(Pass: ADCEPass());
689
690 // Specially optimize memory movement as it doesn't look like dataflow in SSA.
691 FPM.addPass(Pass: MemCpyOptPass());
692
693 FPM.addPass(Pass: DSEPass());
694 FPM.addPass(Pass: MoveAutoInitPass());
695
696 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(
697 Pass: LICMPass(*PTO.LicmMssaOptCap, *PTO.LicmMssaNoAccForPromotionCap,
698 /*AllowSpeculation=*/true),
699 /*UseMemorySSA=*/true));
700
701 FPM.addPass(Pass: CoroElidePass());
702
703 invokeScalarOptimizerLateEPCallbacks(FPM, Level);
704
705 FPM.addPass(Pass: SimplifyCFGPass(SimplifyCFGOptions()
706 .convertSwitchRangeToICmp(B: true)
707 .convertSwitchToArithmetic(B: true)
708 .hoistCommonInsts(B: true)
709 .sinkCommonInsts(B: true)));
710 FPM.addPass(Pass: InstCombinePass());
711 invokePeepholeEPCallbacks(FPM, Level);
712
713 return FPM;
714}
715
716void PassBuilder::addRequiredLTOPreLinkPasses(ModulePassManager &MPM) {
717 MPM.addPass(Pass: CanonicalizeAliasesPass());
718 MPM.addPass(Pass: NameAnonGlobalPass());
719 MPM.addPass(Pass: AssignGUIDPass());
720}
721
722void PassBuilder::addPreInlinerPasses(ModulePassManager &MPM,
723 OptimizationLevel Level,
724 ThinOrFullLTOPhase LTOPhase,
725 bool IgnoreProfile) {
726 assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");
727 if (Opts.disable_preinline)
728 return;
729 InlineParams IP;
730
731 IP.DefaultThreshold = Opts.preinline_threshold;
732 IP.IgnoreProfile = IgnoreProfile;
733
734 // FIXME: The hint threshold has the same value used by the regular inliner
735 // when not optimzing for size. This should probably be lowered after
736 // performance testing.
737 // FIXME: this comment is cargo culted from the old pass manager, revisit).
738 IP.HintThreshold = 325;
739 IP.OptSizeHintThreshold = Opts.preinline_threshold;
740 ModuleInlinerWrapperPass MIWP(
741 IP, /* MandatoryFirst */ true,
742 InlineContext{.LTOPhase: LTOPhase, .Pass: InlinePass::EarlyInliner});
743 CGSCCPassManager &CGPipeline = MIWP.getPM();
744
745 FunctionPassManager FPM;
746 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
747 FPM.addPass(Pass: EarlyCSEPass()); // Catch trivial redundancies.
748 FPM.addPass(Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(
749 B: true))); // Merge & remove basic blocks.
750 FPM.addPass(Pass: InstCombinePass()); // Combine silly sequences.
751 invokePeepholeEPCallbacks(FPM, Level);
752
753 CGPipeline.addPass(Pass: createCGSCCToFunctionPassAdaptor(
754 Pass: std::move(FPM), EagerlyInvalidate: *PTO.EagerlyInvalidateAnalyses));
755
756 MPM.addPass(Pass: std::move(MIWP));
757
758 // Delete anything that is now dead to make sure that we don't instrument
759 // dead code. Instrumentation can end up keeping dead code around and
760 // dramatically increase code size.
761 MPM.addPass(Pass: GlobalDCEPass());
762}
763
764void PassBuilder::addPostPGOLoopRotation(ModulePassManager &MPM,
765 OptimizationLevel Level) {
766 if (Opts.enable_post_pgo_loop_rotation) {
767 // Disable header duplication in loop rotation at -Oz.
768 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
769 Pass: createFunctionToLoopPassAdaptor(Pass: LoopRotatePass(),
770 /*UseMemorySSA=*/false),
771 EagerlyInvalidate: *PTO.EagerlyInvalidateAnalyses));
772 }
773}
774
775void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM,
776 OptimizationLevel Level, bool RunProfileGen,
777 bool IsCS, bool AtomicCounterUpdate,
778 std::string ProfileFile,
779 std::string ProfileRemappingFile) {
780 assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");
781
782 if (!RunProfileGen) {
783 assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
784 MPM.addPass(
785 Pass: PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS, FS));
786 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
787 // RequireAnalysisPass for PSI before subsequent non-module passes.
788 MPM.addPass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
789 return;
790 }
791
792 // Perform PGO instrumentation.
793 MPM.addPass(Pass: PGOInstrumentationGen(IsCS ? PGOInstrumentationType::CSFDO
794 : PGOInstrumentationType::FDO));
795
796 addPostPGOLoopRotation(MPM, Level);
797 // Add the profile lowering pass.
798 InstrProfOptions Options;
799 if (!ProfileFile.empty())
800 Options.InstrProfileOutput = ProfileFile;
801 // Do counter promotion at Level greater than O0.
802 Options.DoCounterPromotion = true;
803 Options.UseBFIInPromotion = IsCS;
804 if (Opts.enable_sampled_instrumentation) {
805 Options.Sampling = true;
806 // With sampling, there is little beneifit to enable counter promotion.
807 // But note that sampling does work with counter promotion.
808 Options.DoCounterPromotion = false;
809 }
810 Options.Atomic = AtomicCounterUpdate;
811 MPM.addPass(Pass: InstrProfilingLoweringPass(Options, IsCS));
812}
813
814void PassBuilder::addPGOInstrPassesForO0(ModulePassManager &MPM,
815 bool RunProfileGen, bool IsCS,
816 bool AtomicCounterUpdate,
817 std::string ProfileFile,
818 std::string ProfileRemappingFile) {
819 if (!RunProfileGen) {
820 assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
821 MPM.addPass(
822 Pass: PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS, FS));
823 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
824 // RequireAnalysisPass for PSI before subsequent non-module passes.
825 MPM.addPass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
826 return;
827 }
828
829 // Perform PGO instrumentation.
830 MPM.addPass(Pass: PGOInstrumentationGen(IsCS ? PGOInstrumentationType::CSFDO
831 : PGOInstrumentationType::FDO));
832 // Add the profile lowering pass.
833 InstrProfOptions Options;
834 if (!ProfileFile.empty())
835 Options.InstrProfileOutput = ProfileFile;
836 // Do not do counter promotion at O0.
837 Options.DoCounterPromotion = false;
838 Options.UseBFIInPromotion = IsCS;
839 Options.Atomic = AtomicCounterUpdate;
840 MPM.addPass(Pass: InstrProfilingLoweringPass(Options, IsCS));
841}
842
843ModuleInlinerWrapperPass
844PassBuilder::buildInlinerPipeline(OptimizationLevel Level,
845 ThinOrFullLTOPhase Phase) {
846 InlineParams IP;
847 if (PTO.InlinerThreshold == -1)
848 IP = ::getInlineParamsFromOptLevel(Context, Level);
849 else
850 IP = getInlineParams(C: Context, Threshold: PTO.InlinerThreshold);
851 // For PreLinkThinLTO + SamplePGO or PreLinkFullLTO + SamplePGO,
852 // set hot-caller threshold to 0 to disable hot
853 // callsite inline (as much as possible [1]) because it makes
854 // profile annotation in the backend inaccurate.
855 //
856 // [1] Note the cost of a function could be below zero due to erased
857 // prologue / epilogue.
858 if (isLTOPreLink(Phase) && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse)
859 IP.HotCallSiteThreshold = 0;
860
861 if (PGOOpt)
862 IP.EnableDeferral = Opts.enable_npm_pgo_inline_deferral;
863
864 ModuleInlinerWrapperPass MIWP(IP, Opts.mandatory_inlining_first,
865 InlineContext{.LTOPhase: Phase, .Pass: InlinePass::CGSCCInliner},
866 Opts.enable_ml_inliner,
867 *PTO.MaxDevirtIterations);
868
869 // Require the GlobalsAA analysis for the module so we can query it within
870 // the CGSCC pipeline.
871 if (Opts.enable_global_analyses) {
872 MIWP.addModulePass(Pass: RequireAnalysisPass<GlobalsAA, Module>());
873 // Invalidate AAManager so it can be recreated and pick up the newly
874 // available GlobalsAA.
875 MIWP.addModulePass(
876 Pass: createModuleToFunctionPassAdaptor(Pass: InvalidateAnalysisPass<AAManager>()));
877 }
878
879 // Require the ProfileSummaryAnalysis for the module so we can query it within
880 // the inliner pass.
881 MIWP.addModulePass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
882
883 // Now begin the main postorder CGSCC pipeline.
884 // FIXME: The current CGSCC pipeline has its origins in the legacy pass
885 // manager and trying to emulate its precise behavior. Much of this doesn't
886 // make a lot of sense and we should revisit the core CGSCC structure.
887 CGSCCPassManager &MainCGPipeline = MIWP.getPM();
888
889 // Note: historically, the PruneEH pass was run first to deduce nounwind and
890 // generally clean up exception handling overhead. It isn't clear this is
891 // valuable as the inliner doesn't currently care whether it is inlining an
892 // invoke or a call.
893
894 if (Opts.attributor_enable & AttributorRunOption::CGSCC)
895 MainCGPipeline.addPass(Pass: AttributorCGSCCPass());
896
897 // Deduce function attributes. We do another run of this after the function
898 // simplification pipeline, so this only needs to run when it could affect the
899 // function simplification pipeline, which is only the case with recursive
900 // functions.
901 if (Opts.attributor_enable & AttributorRunOption::CGSCC_LIGHT) {
902 MainCGPipeline.addPass(Pass: AttributorLightCGSCCPass());
903 } else {
904 MainCGPipeline.addPass(
905 Pass: PostOrderFunctionAttrsPass(/*SkipNonRecursive*/ true));
906 }
907
908 // When at O3 add argument promotion to the pass pipeline.
909 // FIXME: It isn't at all clear why this should be limited to O3.
910 if (Level == OptimizationLevel::O3)
911 MainCGPipeline.addPass(Pass: ArgumentPromotionPass());
912
913 // Try to perform OpenMP specific optimizations. This is a (quick!) no-op if
914 // there are no OpenMP runtime calls present in the module.
915 if (Level == OptimizationLevel::O2 || Level == OptimizationLevel::O3)
916 MainCGPipeline.addPass(Pass: OpenMPOptCGSCCPass(Phase));
917
918 invokeCGSCCOptimizerLateEPCallbacks(CGPM&: MainCGPipeline, Level);
919
920 // Add the core function simplification pipeline nested inside the
921 // CGSCC walk.
922 MainCGPipeline.addPass(Pass: createCGSCCToFunctionPassAdaptor(
923 Pass: buildFunctionSimplificationPipeline(Level, Phase),
924 EagerlyInvalidate: *PTO.EagerlyInvalidateAnalyses, /*NoRerun=*/true));
925
926 // Finally, deduce any function attributes based on the fully simplified
927 // function.
928 if (Opts.attributor_enable & AttributorRunOption::CGSCC_LIGHT)
929 MainCGPipeline.addPass(Pass: AttributorLightCGSCCPass());
930 else
931 MainCGPipeline.addPass(Pass: PostOrderFunctionAttrsPass());
932
933 // Mark that the function is fully simplified and that it shouldn't be
934 // simplified again if we somehow revisit it due to CGSCC mutations unless
935 // it's been modified since.
936 MainCGPipeline.addPass(Pass: createCGSCCToFunctionPassAdaptor(
937 Pass: RequireAnalysisPass<ShouldNotRunFunctionPassesAnalysis, Function>()));
938
939 if (!isThinLTOPreLink(Phase)) {
940 MainCGPipeline.addPass(Pass: CoroSplitPass(Level != OptimizationLevel::O0));
941 MainCGPipeline.addPass(Pass: CoroAnnotationElidePass());
942 }
943
944 // Make sure we don't affect potential future NoRerun CGSCC adaptors.
945 MIWP.addLateModulePass(Pass: createModuleToFunctionPassAdaptor(
946 Pass: InvalidateAnalysisPass<ShouldNotRunFunctionPassesAnalysis>()));
947
948 return MIWP;
949}
950
951ModulePassManager
952PassBuilder::buildModuleInlinerPipeline(OptimizationLevel Level,
953 ThinOrFullLTOPhase Phase) {
954 ModulePassManager MPM;
955
956 InlineParams IP = ::getInlineParamsFromOptLevel(Context, Level);
957 // For PreLinkThinLTO + SamplePGO or PreLinkFullLTO + SamplePGO,
958 // set hot-caller threshold to 0 to disable hot
959 // callsite inline (as much as possible [1]) because it makes
960 // profile annotation in the backend inaccurate.
961 //
962 // [1] Note the cost of a function could be below zero due to erased
963 // prologue / epilogue.
964 if (isLTOPreLink(Phase) && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse)
965 IP.HotCallSiteThreshold = 0;
966
967 if (PGOOpt)
968 IP.EnableDeferral = Opts.enable_npm_pgo_inline_deferral;
969
970 // The inline deferral logic is used to avoid losing some
971 // inlining chance in future. It is helpful in SCC inliner, in which
972 // inlining is processed in bottom-up order.
973 // While in module inliner, the inlining order is a priority-based order
974 // by default. The inline deferral is unnecessary there. So we disable the
975 // inline deferral logic in module inliner.
976 IP.EnableDeferral = false;
977
978 MPM.addPass(Pass: ModuleInlinerPass(IP, Opts.enable_ml_inliner, Phase));
979 if (!UseCtxProfile.empty() && Phase == ThinOrFullLTOPhase::ThinLTOPostLink) {
980 MPM.addPass(Pass: GlobalOptPass());
981 MPM.addPass(Pass: GlobalDCEPass());
982 MPM.addPass(Pass: AssignGUIDPass());
983 MPM.addPass(Pass: PGOCtxProfFlatteningPass(/*IsPreThinlink=*/false));
984 }
985
986 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
987 Pass: buildFunctionSimplificationPipeline(Level, Phase),
988 EagerlyInvalidate: *PTO.EagerlyInvalidateAnalyses));
989
990 if (!isThinLTOPreLink(Phase)) {
991 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(
992 Pass: CoroSplitPass(Level != OptimizationLevel::O0)));
993 MPM.addPass(
994 Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: CoroAnnotationElidePass()));
995 }
996
997 return MPM;
998}
999
1000ModulePassManager PassBuilder::buildModuleSimplificationPipeline(
1001 OptimizationLevel Level, ThinOrFullLTOPhase Phase,
1002 const ModuleSummaryIndex *ImportSummary) {
1003 assert(Level != OptimizationLevel::O0 &&
1004 "Should not be used for O0 pipeline");
1005
1006 assert(!isFullLTOPostLink(Phase) &&
1007 "FullLTOPostLink shouldn't call buildModuleSimplificationPipeline!");
1008
1009 assert((!ImportSummary || isThinLTOPostLink(Phase)) &&
1010 "ImportSummary is only meaningful in the ThinLTO post link phase");
1011
1012 ModulePassManager MPM;
1013
1014 // Place pseudo probe instrumentation as the first pass of the pipeline to
1015 // minimize the impact of optimization changes.
1016 if (PGOOpt && PGOOpt->PseudoProbeForProfiling && !isThinLTOPostLink(Phase))
1017 MPM.addPass(Pass: SampleProfileProbePass(TM));
1018
1019 bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse);
1020
1021 // In ThinLTO mode, when flattened profile is used, all the available
1022 // profile information will be annotated in PreLink phase so there is
1023 // no need to load the profile again in PostLink.
1024 bool LoadSampleProfile = HasSampleProfile && !(Opts.flattened_profile_used &&
1025 isThinLTOPostLink(Phase));
1026
1027 // During the ThinLTO backend phase we perform early indirect call promotion
1028 // here, before globalopt. Otherwise imported available_externally functions
1029 // look unreferenced and are removed. If we are going to load the sample
1030 // profile then defer until later.
1031 // TODO: See if we can move later and consolidate with the location where
1032 // we perform ICP when we are loading a sample profile.
1033 // TODO: We pass HasSampleProfile (whether there was a sample profile file
1034 // passed to the compile) to the SamplePGO flag of ICP. This is used to
1035 // determine whether the new direct calls are annotated with prof metadata.
1036 // Ideally this should be determined from whether the IR is annotated with
1037 // sample profile, and not whether the a sample profile was provided on the
1038 // command line. E.g. for flattened profiles where we will not be reloading
1039 // the sample profile in the ThinLTO backend, we ideally shouldn't have to
1040 // provide the sample profile file.
1041 if (isThinLTOPostLink(Phase) && !LoadSampleProfile)
1042 MPM.addPass(Pass: PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile,
1043 ImportSummary));
1044
1045 // Create an early function pass manager to cleanup the output of the
1046 // frontend. Not necessary with LTO post link pipelines since the pre link
1047 // pipeline already cleaned up the frontend output.
1048 if (!isThinLTOPostLink(Phase)) {
1049 // Do basic inference of function attributes from known properties of system
1050 // libraries and other oracles.
1051 MPM.addPass(Pass: InferFunctionAttrsPass());
1052 MPM.addPass(Pass: CoroEarlyPass());
1053
1054 FunctionPassManager EarlyFPM;
1055 EarlyFPM.addPass(Pass: EntryExitInstrumenterPass(/*PostInlining=*/false));
1056 // Lower llvm.expect to metadata before attempting transforms.
1057 // Compare/branch metadata may alter the behavior of passes like
1058 // SimplifyCFG.
1059 EarlyFPM.addPass(Pass: LowerExpectIntrinsicPass());
1060 EarlyFPM.addPass(Pass: SimplifyCFGPass());
1061 EarlyFPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
1062 EarlyFPM.addPass(Pass: EarlyCSEPass());
1063 if (Level == OptimizationLevel::O3)
1064 EarlyFPM.addPass(Pass: CallSiteSplittingPass());
1065 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
1066 Pass: std::move(EarlyFPM), EagerlyInvalidate: *PTO.EagerlyInvalidateAnalyses));
1067 }
1068
1069 if (LoadSampleProfile) {
1070 // Annotate sample profile right after early FPM to ensure freshness of
1071 // the debug info.
1072 MPM.addPass(Pass: SampleProfileLoaderPass(
1073 PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile, Phase, FS));
1074 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1075 // RequireAnalysisPass for PSI before subsequent non-module passes.
1076 MPM.addPass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
1077 // Do not invoke ICP in the LTOPrelink phase as it makes it hard
1078 // for the profile annotation to be accurate in the LTO backend.
1079 if (!isLTOPreLink(Phase))
1080 // We perform early indirect call promotion here, before globalopt.
1081 // This is important for the ThinLTO backend phase because otherwise
1082 // imported available_externally functions look unreferenced and are
1083 // removed.
1084 MPM.addPass(Pass: PGOIndirectCallPromotion(
1085 true /* IsInLTO */, true /* SamplePGO */, ImportSummary));
1086 }
1087
1088 // Try to perform OpenMP specific optimizations on the module. This is a
1089 // (quick!) no-op if there are no OpenMP runtime calls present in the module.
1090 MPM.addPass(Pass: OpenMPOptPass(Phase));
1091
1092 if (Opts.attributor_enable & AttributorRunOption::MODULE)
1093 MPM.addPass(Pass: AttributorPass());
1094 else if (Opts.attributor_enable & AttributorRunOption::MODULE_LIGHT)
1095 MPM.addPass(Pass: AttributorLightPass());
1096
1097 // Lower type metadata and the type.test intrinsic in the ThinLTO post link
1098 // pipeline after ICP. This is to enable usage of the type tests in ICP
1099 // sequences. It also lets ICP (including the ICP performed by the sample
1100 // profile loader) see the original names of the CFI functions it wants to
1101 // promote to, which LowerTypeTests renames to <name>.cfi.
1102 if (isThinLTOPostLink(Phase)) {
1103 if (ImportSummary)
1104 MPM.addPass(Pass: LowerTypeTestsPass(nullptr, ImportSummary));
1105 MPM.addPass(Pass: DropTypeTestsPass());
1106 }
1107
1108 invokePipelineEarlySimplificationEPCallbacks(MPM, Level, Phase);
1109
1110 // Interprocedural constant propagation now that basic cleanup has occurred
1111 // and prior to optimizing globals.
1112 // FIXME: This position in the pipeline hasn't been carefully considered in
1113 // years, it should be re-analyzed.
1114 MPM.addPass(
1115 Pass: IPSCCPPass(IPSCCPOptions(/*AllowFuncSpec=*/!isLTOPreLink(Phase))));
1116
1117 // Attach metadata to indirect call sites indicating the set of functions
1118 // they may target at run-time. This should follow IPSCCP.
1119 MPM.addPass(Pass: CalledValuePropagationPass());
1120
1121 // Optimize globals to try and fold them into constants.
1122 MPM.addPass(Pass: GlobalOptPass());
1123
1124 // Create a small function pass pipeline to cleanup after all the global
1125 // optimizations.
1126 FunctionPassManager GlobalCleanupPM;
1127 // FIXME: Should this instead by a run of SROA?
1128 GlobalCleanupPM.addPass(Pass: PromotePass());
1129 GlobalCleanupPM.addPass(Pass: InstCombinePass());
1130 invokePeepholeEPCallbacks(FPM&: GlobalCleanupPM, Level);
1131 GlobalCleanupPM.addPass(
1132 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
1133 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
1134 Pass: std::move(GlobalCleanupPM), EagerlyInvalidate: *PTO.EagerlyInvalidateAnalyses));
1135
1136 // We already asserted this happens in non-FullLTOPostLink earlier.
1137 const bool IsPreLink = !isThinLTOPostLink(Phase);
1138 // Enable contextual profiling instrumentation.
1139 const bool IsCtxProfGen =
1140 IsPreLink && PGOCtxProfLoweringPass::isCtxIRPGOInstrEnabled(
1141 LibraryOptions: Context.getLibraryOptions());
1142 const bool IsPGOPreLink = !IsCtxProfGen && PGOOpt && IsPreLink;
1143 const bool IsPGOInstrGen =
1144 IsPGOPreLink && PGOOpt->Action == PGOOptions::IRInstr;
1145 const bool IsPGOInstrUse =
1146 IsPGOPreLink && PGOOpt->Action == PGOOptions::IRUse;
1147 const bool IsMemprofUse = IsPGOPreLink && !PGOOpt->MemoryProfile.empty();
1148 // We don't want to mix pgo ctx gen and pgo gen; we also don't currently
1149 // enable ctx profiling from the frontend.
1150 assert(!(IsPGOInstrGen && PGOCtxProfLoweringPass::isCtxIRPGOInstrEnabled(
1151 Context.getLibraryOptions())) &&
1152 "Enabling both instrumented PGO and contextual instrumentation is not "
1153 "supported.");
1154 const bool IsCtxProfUse = !UseCtxProfile.empty() && isThinLTOPreLink(Phase);
1155
1156 assert((Opts.instrument_cold_function_only_path.empty() ||
1157 isPGOInstrumentColdFunctionOnly(Context.getLibraryOptions())) &&
1158 "--instrument-cold-function-only-path is provided but "
1159 "--pgo-instrument-cold-function-only is not enabled");
1160 const bool IsColdFuncOnlyInstrGen =
1161 isPGOInstrumentColdFunctionOnly(LibraryOptions: Context.getLibraryOptions()) &&
1162 IsPGOPreLink && !Opts.instrument_cold_function_only_path.empty();
1163
1164 // When a memory profile is used together with a sample profile, the
1165 // pre-inliner runs after the sample profile has been loaded. By default make
1166 // it ignore the sample profile, so that the inlining seen by MemProfUsePass
1167 // doesn't depend on the sample profile. Otherwise, a different sample profile
1168 // than the one used to build the memory-profiled binary can cause additional
1169 // inlining (e.g. of newly hot callees) that leads to memprof matching
1170 // failures. This matches the IR PGO pipeline, where the pre-inliner runs
1171 // before the profile is annotated.
1172 const bool PreInlinerIgnoreProfile =
1173 IsMemprofUse && HasSampleProfile && Opts.memprof_preinline_ignore_profile;
1174 if (IsPGOInstrGen || IsPGOInstrUse || IsMemprofUse || IsCtxProfGen ||
1175 IsCtxProfUse || IsColdFuncOnlyInstrGen)
1176 addPreInlinerPasses(MPM, Level, LTOPhase: Phase, IgnoreProfile: PreInlinerIgnoreProfile);
1177
1178 // Add all the requested passes for instrumentation PGO, if requested.
1179 if (IsPGOInstrGen || IsPGOInstrUse) {
1180 addPGOInstrPasses(MPM, Level,
1181 /*RunProfileGen=*/IsPGOInstrGen,
1182 /*IsCS=*/false, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate,
1183 ProfileFile: PGOOpt->ProfileFile, ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
1184 } else if (IsCtxProfGen || IsCtxProfUse) {
1185 MPM.addPass(Pass: PGOInstrumentationGen(PGOInstrumentationType::CTXPROF));
1186 // In pre-link, we just want the instrumented IR. We use the contextual
1187 // profile in the post-thinlink phase.
1188 // The instrumentation will be removed in post-thinlink after IPO.
1189 if (IsCtxProfUse) {
1190 MPM.addPass(Pass: AssignGUIDPass());
1191 MPM.addPass(Pass: PGOCtxProfFlatteningPass(/*IsPreThinlink=*/true));
1192 return MPM;
1193 }
1194 // Block further inlining in the instrumented ctxprof case. This avoids
1195 // confusingly collecting profiles for the same GUID corresponding to
1196 // different variants of the function. We could do like PGO and identify
1197 // functions by a (GUID, Hash) tuple, but since the ctxprof "use" waits for
1198 // thinlto to happen before performing any further optimizations, it's
1199 // unnecessary to collect profiles for non-prevailing copies.
1200 MPM.addPass(Pass: NoinlineNonPrevailing());
1201 addPostPGOLoopRotation(MPM, Level);
1202 MPM.addPass(Pass: AssignGUIDPass());
1203 MPM.addPass(Pass: PGOCtxProfLoweringPass());
1204 } else if (IsColdFuncOnlyInstrGen) {
1205 addPGOInstrPasses(MPM, Level, /* RunProfileGen */ true, /* IsCS */ false,
1206 /* AtomicCounterUpdate */ false,
1207 ProfileFile: Opts.instrument_cold_function_only_path.str(),
1208 /* ProfileRemappingFile */ "");
1209 }
1210
1211 if (IsPGOInstrGen || IsPGOInstrUse || IsCtxProfGen)
1212 MPM.addPass(Pass: PGOIndirectCallPromotion(false, false));
1213
1214 if (IsPGOPreLink && PGOOpt->CSAction == PGOOptions::CSIRInstr)
1215 MPM.addPass(Pass: PGOInstrumentationGenCreateVar(
1216 PGOOpt->CSProfileGenFile, Opts.enable_sampled_instrumentation));
1217
1218 if (IsMemprofUse)
1219 MPM.addPass(Pass: MemProfUsePass(PGOOpt->MemoryProfile, FS));
1220
1221 if (PGOOpt && (PGOOpt->Action == PGOOptions::IRUse ||
1222 PGOOpt->Action == PGOOptions::SampleUse))
1223 MPM.addPass(Pass: PGOForceFunctionAttrsPass(PGOOpt->ColdOptType));
1224
1225 MPM.addPass(Pass: AlwaysInlinerPass(/*InsertLifetimeIntrinsics=*/true));
1226
1227 if (Opts.enable_module_inliner)
1228 MPM.addPass(Pass: buildModuleInlinerPipeline(Level, Phase));
1229 else
1230 MPM.addPass(Pass: buildInlinerPipeline(Level, Phase));
1231
1232 // Remove any dead arguments exposed by cleanups, constant folding globals,
1233 // and argument promotion.
1234 MPM.addPass(Pass: DeadArgumentEliminationPass());
1235
1236 if (isThinLTOPostLink(Phase))
1237 MPM.addPass(Pass: SimplifyTypeTestsPass());
1238
1239 if (!isThinLTOPreLink(Phase))
1240 MPM.addPass(Pass: CoroCleanupPass());
1241
1242 // Optimize globals now that functions are fully simplified.
1243 MPM.addPass(Pass: GlobalOptPass());
1244 MPM.addPass(Pass: GlobalDCEPass());
1245
1246 return MPM;
1247}
1248
1249/// TODO: Should LTO cause any differences to this set of passes?
1250void PassBuilder::addVectorPasses(OptimizationLevel Level,
1251 FunctionPassManager &FPM,
1252 ThinOrFullLTOPhase LTOPhase) {
1253 FPM.addPass(Pass: LoopVectorizePass(
1254 LoopVectorizeOptions(!PTO.LoopInterleaving, !PTO.LoopVectorization)));
1255
1256 // Drop dereferenceable assumes after vectorization, as they are no longer
1257 // needed and can inhibit further optimization.
1258 if (!isLTOPreLink(Phase: LTOPhase))
1259 FPM.addPass(Pass: DropUnnecessaryAssumesPass(/*DropDereferenceable=*/true));
1260
1261 FPM.addPass(Pass: InferAlignmentPass());
1262 if (isFullLTOPostLink(Phase: LTOPhase)) {
1263 // The vectorizer may have significantly shortened a loop body; unroll
1264 // again. Unroll small loops to hide loop backedge latency and saturate any
1265 // parallel execution resources of an out-of-order processor. We also then
1266 // need to clean up redundancies and loop invariant code.
1267 // FIXME: It would be really good to use a loop-integrated instruction
1268 // combiner for cleanup here so that the unrolling and LICM can be pipelined
1269 // across the loop nests.
1270 // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1271 if (Opts.enable_unroll_and_jam && PTO.LoopUnrolling)
1272 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(
1273 Pass: LoopUnrollAndJamPass(static_cast<int>(Level))));
1274 FPM.addPass(Pass: LoopUnrollPass(LoopUnrollOptions(
1275 static_cast<int>(Level), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1276 *PTO.ForgetAllSCEVInLoopUnroll)));
1277 FPM.addPass(Pass: WarnMissedTransformationsPass());
1278 // Now that we are done with loop unrolling, be it either by LoopVectorizer,
1279 // or LoopUnroll passes, some variable-offset GEP's into alloca's could have
1280 // become constant-offset, thus enabling SROA and alloca promotion. Do so.
1281 // NOTE: we are very late in the pipeline, and we don't have any LICM
1282 // or SimplifyCFG passes scheduled after us, that would cleanup
1283 // the CFG mess this may created if allowed to modify CFG, so forbid that.
1284
1285 // We also turn on struct to vector canonicalization here, which allows
1286 // converting allocas of homogeneous structs into vector allocas when the
1287 // allocas' users are all memory intrinsics. This allows promotion in some
1288 // cases because structs cannot promote to SSA values, but vectors can. We
1289 // only turn this on after memcpyopt runs because this might hinder
1290 // memcpyopt's optimizations if done before. Look at the documentation for
1291 // `tryCanonicalizeStructToVector` in SROA.cpp to see why.
1292 FPM.addPass(Pass: SROAPass(SROAOptions(SROAOptions::PreserveCFG,
1293 /*AggregateToVector=*/true)));
1294 }
1295
1296 if (!isFullLTOPostLink(Phase: LTOPhase)) {
1297 // Eliminate loads by forwarding stores from the previous iteration to loads
1298 // of the current iteration.
1299 FPM.addPass(Pass: LoopLoadEliminationPass());
1300 }
1301 // Cleanup after the loop optimization passes.
1302 FPM.addPass(Pass: InstCombinePass());
1303
1304 if (Level > OptimizationLevel::O1 && Opts.extra_vectorizer_passes) {
1305 ExtraFunctionPassManager<ShouldRunExtraVectorPasses> ExtraPasses;
1306 // At higher optimization levels, try to clean up any runtime overlap and
1307 // alignment checks inserted by the vectorizer. We want to track correlated
1308 // runtime checks for two inner loops in the same outer loop, fold any
1309 // common computations, hoist loop-invariant aspects out of any outer loop,
1310 // and unswitch the runtime checks if possible. Once hoisted, we may have
1311 // dead (or speculatable) control flows or more combining opportunities.
1312 ExtraPasses.addPass(Pass: EarlyCSEPass());
1313 ExtraPasses.addPass(Pass: CorrelatedValuePropagationPass());
1314 ExtraPasses.addPass(Pass: InstCombinePass());
1315 LoopPassManager LPM;
1316 LPM.addPass(Pass: LICMPass(*PTO.LicmMssaOptCap, *PTO.LicmMssaNoAccForPromotionCap,
1317 /*AllowSpeculation=*/true));
1318 LPM.addPass(Pass: SimpleLoopUnswitchPass(/* NonTrivial */ Level ==
1319 OptimizationLevel::O3));
1320 ExtraPasses.addPass(
1321 Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM), /*UseMemorySSA=*/true));
1322 ExtraPasses.addPass(
1323 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
1324 ExtraPasses.addPass(Pass: InstCombinePass());
1325 FPM.addPass(Pass: std::move(ExtraPasses));
1326 }
1327
1328 // Now that we've formed fast to execute loop structures, we do further
1329 // optimizations. These are run afterward as they might block doing complex
1330 // analyses and transforms such as what are needed for loop vectorization.
1331
1332 // Cleanup after loop vectorization, etc. Simplification passes like CVP and
1333 // GVN, loop transforms, and others have already run, so it's now better to
1334 // convert to more optimized IR using more aggressive simplify CFG options.
1335 // The extra sinking transform can create larger basic blocks, so do this
1336 // before SLP vectorization.
1337 FPM.addPass(Pass: SimplifyCFGPass(SimplifyCFGOptions()
1338 .forwardSwitchCondToPhi(B: true)
1339 .convertSwitchRangeToICmp(B: true)
1340 .convertSwitchToArithmetic(B: true)
1341 .convertSwitchToLookupTable(B: true)
1342 .needCanonicalLoops(B: false)
1343 .hoistCommonInsts(B: true)
1344 .sinkCommonInsts(B: true)));
1345
1346 if (isFullLTOPostLink(Phase: LTOPhase)) {
1347 FPM.addPass(Pass: SCCPPass());
1348 FPM.addPass(Pass: InstCombinePass());
1349 FPM.addPass(Pass: BDCEPass());
1350 }
1351
1352 // Optimize parallel scalar instruction chains into SIMD instructions.
1353 if (PTO.SLPVectorization) {
1354 FPM.addPass(Pass: SLPVectorizerPass());
1355 if (Level >= OptimizationLevel::O2 && Opts.extra_vectorizer_passes) {
1356 FPM.addPass(Pass: EarlyCSEPass());
1357 }
1358 }
1359 // Enhance/cleanup vector code.
1360 FPM.addPass(Pass: VectorCombinePass());
1361
1362 if (!isFullLTOPostLink(Phase: LTOPhase)) {
1363 FPM.addPass(Pass: InstCombinePass());
1364 // Unroll small loops to hide loop backedge latency and saturate any
1365 // parallel execution resources of an out-of-order processor. We also then
1366 // need to clean up redundancies and loop invariant code.
1367 // FIXME: It would be really good to use a loop-integrated instruction
1368 // combiner for cleanup here so that the unrolling and LICM can be pipelined
1369 // across the loop nests.
1370 // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1371 if (Opts.enable_unroll_and_jam && PTO.LoopUnrolling) {
1372 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(
1373 Pass: LoopUnrollAndJamPass(static_cast<int>(Level))));
1374 }
1375 FPM.addPass(Pass: LoopUnrollPass(LoopUnrollOptions(
1376 static_cast<int>(Level), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1377 *PTO.ForgetAllSCEVInLoopUnroll)));
1378 FPM.addPass(Pass: WarnMissedTransformationsPass());
1379 // Now that we are done with loop unrolling, be it either by LoopVectorizer,
1380 // or LoopUnroll passes, some variable-offset GEP's into alloca's could have
1381 // become constant-offset, thus enabling SROA and alloca promotion. Do so.
1382 // NOTE: we are very late in the pipeline, and we don't have any LICM
1383 // or SimplifyCFG passes scheduled after us, that would cleanup
1384 // the CFG mess this may created if allowed to modify CFG, so forbid that.
1385
1386 // We also turn on struct to vector canonicalization here, which allows
1387 // converting allocas of homogeneous structs into vector allocas when the
1388 // allocas' users are all memory intrinsics. This allows promotion in some
1389 // cases because structs cannot promote to SSA values, but vectors can. We
1390 // only turn this on after memcpyopt runs because this might hinder
1391 // memcpyopt's optimizations if done before. Look at the documentation for
1392 // `tryCanonicalizeStructToVector` in SROA.cpp to see why.
1393 FPM.addPass(Pass: SROAPass(SROAOptions(SROAOptions::PreserveCFG,
1394 /*AggregateToVector=*/true)));
1395 }
1396
1397 FPM.addPass(Pass: InferAlignmentPass());
1398 FPM.addPass(Pass: InstCombinePass());
1399
1400 // This is needed for two reasons:
1401 // 1. It works around problems that instcombine introduces, such as sinking
1402 // expensive FP divides into loops containing multiplications using the
1403 // divide result.
1404 // 2. It helps to clean up some loop-invariant code created by the loop
1405 // unroll pass when IsFullLTO=false.
1406 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(
1407 Pass: LICMPass(*PTO.LicmMssaOptCap, *PTO.LicmMssaNoAccForPromotionCap,
1408 /*AllowSpeculation=*/true),
1409 /*UseMemorySSA=*/true));
1410
1411 // Now that we've vectorized and unrolled loops, we may have more refined
1412 // alignment information, try to re-derive it here.
1413 FPM.addPass(Pass: AlignmentFromAssumptionsPass());
1414}
1415
1416ModulePassManager
1417PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
1418 ThinOrFullLTOPhase LTOPhase) {
1419 ModulePassManager MPM;
1420
1421 // Run partial inlining pass to partially inline functions that have
1422 // large bodies.
1423 if (Opts.enable_partial_inlining)
1424 MPM.addPass(Pass: PartialInlinerPass());
1425
1426 // Remove avail extern fns and globals definitions since we aren't compiling
1427 // an object file for later LTO. For LTO we want to preserve these so they
1428 // are eligible for inlining at link-time. Note if they are unreferenced they
1429 // will be removed by GlobalDCE later, so this only impacts referenced
1430 // available externally globals. Eventually they will be suppressed during
1431 // codegen, but eliminating here enables more opportunity for GlobalDCE as it
1432 // may make globals referenced by available external functions dead and saves
1433 // running remaining passes on the eliminated functions. These should be
1434 // preserved during prelinking for link-time inlining decisions.
1435 if (!isLTOPreLink(Phase: LTOPhase))
1436 MPM.addPass(Pass: EliminateAvailableExternallyPass());
1437
1438 // Do RPO function attribute inference across the module to forward-propagate
1439 // attributes where applicable.
1440 // FIXME: Is this really an optimization rather than a canonicalization?
1441 if (Opts.attributor_enable & AttributorRunOption::MODULE_LIGHT)
1442 MPM.addPass(Pass: AttributorLightPass());
1443 else
1444 MPM.addPass(Pass: ReversePostOrderFunctionAttrsPass());
1445
1446 // Do a post inline PGO instrumentation and use pass. This is a context
1447 // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as
1448 // cross-module inline has not been done yet. The context sensitive
1449 // instrumentation is after all the inlines are done.
1450 if (!isLTOPreLink(Phase: LTOPhase) && PGOOpt) {
1451 if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1452 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/true,
1453 /*IsCS=*/true, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate,
1454 ProfileFile: PGOOpt->CSProfileGenFile, ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
1455 else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1456 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/false,
1457 /*IsCS=*/true, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate,
1458 ProfileFile: PGOOpt->ProfileFile, ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
1459 }
1460
1461 // Re-compute GlobalsAA here prior to function passes. This is particularly
1462 // useful as the above will have inlined, DCE'ed, and function-attr
1463 // propagated everything. We should at this point have a reasonably minimal
1464 // and richly annotated call graph. By computing aliasing and mod/ref
1465 // information for all local globals here, the late loop passes and notably
1466 // the vectorizer will be able to use them to help recognize vectorizable
1467 // memory operations.
1468 if (Opts.enable_global_analyses)
1469 MPM.addPass(Pass: RecomputeGlobalsAAPass());
1470
1471 invokeOptimizerEarlyEPCallbacks(MPM, Level, Phase: LTOPhase);
1472
1473 FunctionPassManager OptimizePM;
1474
1475 // Only drop unnecessary assumes post-inline and post-link, as otherwise
1476 // additional uses of the affected value may be introduced through inlining
1477 // and CSE.
1478 if (!isLTOPreLink(Phase: LTOPhase))
1479 OptimizePM.addPass(Pass: DropUnnecessaryAssumesPass());
1480
1481 // Scheduling LoopVersioningLICM when inlining is over, because after that
1482 // we may see more accurate aliasing. Reason to run this late is that too
1483 // early versioning may prevent further inlining due to increase of code
1484 // size. Other optimizations which runs later might get benefit of no-alias
1485 // assumption in clone loop.
1486 if (Opts.enable_loop_versioning_licm) {
1487 OptimizePM.addPass(
1488 Pass: createFunctionToLoopPassAdaptor(Pass: LoopVersioningLICMPass()));
1489 // LoopVersioningLICM pass might increase new LICM opportunities.
1490 OptimizePM.addPass(Pass: createFunctionToLoopPassAdaptor(
1491 Pass: LICMPass(*PTO.LicmMssaOptCap, *PTO.LicmMssaNoAccForPromotionCap,
1492 /*AllowSpeculation=*/true),
1493 /*USeMemorySSA=*/UseMemorySSA: true));
1494 }
1495
1496 OptimizePM.addPass(Pass: Float2IntPass());
1497 // Defer until LTO post-link where some constants may become known.
1498 if (!isLTOPreLink(Phase: LTOPhase))
1499 OptimizePM.addPass(Pass: LowerConstantIntrinsicsPass());
1500
1501 if (Opts.enable_matrix) {
1502 OptimizePM.addPass(Pass: LowerMatrixIntrinsicsPass());
1503 OptimizePM.addPass(Pass: EarlyCSEPass());
1504 }
1505
1506 // CHR pass should only be applied with the profile information.
1507 // The check is to check the profile summary information in CHR.
1508 if (Opts.enable_chr && Level == OptimizationLevel::O3)
1509 OptimizePM.addPass(Pass: ControlHeightReductionPass());
1510
1511 // FIXME: We need to run some loop optimizations to re-rotate loops after
1512 // simplifycfg and others undo their rotation.
1513
1514 // Optimize the loop execution. These passes operate on entire loop nests
1515 // rather than on each loop in an inside-out manner, and so they are actually
1516 // function passes.
1517
1518 invokeVectorizerStartEPCallbacks(FPM&: OptimizePM, Level);
1519
1520 LoopPassManager LPM;
1521 // First rotate loops that may have been un-rotated by prior passes.
1522 // Disable header duplication at -Oz.
1523 LPM.addPass(Pass: LoopRotatePass(/*EnableLoopHeaderDuplication=*/true,
1524 isLTOPreLink(Phase: LTOPhase),
1525 /*CheckExitCount=*/true));
1526 // Some loops may have become dead by now. Try to delete them.
1527 // FIXME: see discussion in https://reviews.llvm.org/D112851,
1528 // this may need to be revisited once we run GVN before loop deletion
1529 // in the simplification pipeline.
1530 LPM.addPass(Pass: LoopDeletionPass());
1531
1532 if (PTO.LoopInterchange)
1533 LPM.addPass(Pass: LoopInterchangePass());
1534
1535 OptimizePM.addPass(
1536 Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM), /*UseMemorySSA=*/false));
1537
1538 // FIXME: This may not be the right place in the pipeline.
1539 // We need to have the data to support the right place.
1540 if (PTO.LoopFusion)
1541 OptimizePM.addPass(Pass: LoopFusePass());
1542
1543 // Distribute loops to allow partial vectorization. I.e. isolate dependences
1544 // into separate loop that would otherwise inhibit vectorization. This is
1545 // currently only performed for loops marked with the metadata
1546 // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
1547 OptimizePM.addPass(Pass: LoopDistributePass());
1548
1549 // Populates the VFABI attribute with the scalar-to-vector mappings
1550 // from the TargetLibraryInfo.
1551 OptimizePM.addPass(Pass: InjectTLIMappings());
1552
1553 addVectorPasses(Level, FPM&: OptimizePM, LTOPhase);
1554
1555 invokeVectorizerEndEPCallbacks(FPM&: OptimizePM, Level);
1556
1557 // LoopSink pass sinks instructions hoisted by LICM, which serves as a
1558 // canonicalization pass that enables other optimizations. As a result,
1559 // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
1560 // result too early.
1561 OptimizePM.addPass(Pass: LoopSinkPass());
1562
1563 // And finally clean up LCSSA form before generating code.
1564 OptimizePM.addPass(Pass: InstSimplifyPass());
1565
1566 // This hoists/decomposes div/rem ops. It should run after other sink/hoist
1567 // passes to avoid re-sinking, but before SimplifyCFG because it can allow
1568 // flattening of blocks.
1569 OptimizePM.addPass(Pass: DivRemPairsPass());
1570
1571 // Merge adjacent icmps into memcmp, then expand memcmp to loads/compares.
1572 // TODO: move this further up so that it can be optimized by GVN, etc.
1573 if (Opts.enable_mergeicmps)
1574 OptimizePM.addPass(Pass: MergeICmpsPass());
1575 OptimizePM.addPass(Pass: ExpandMemCmpPass());
1576
1577 // Try to annotate calls that were created during optimization.
1578 OptimizePM.addPass(
1579 Pass: TailCallElimPass(/*UpdateFunctionEntryCount=*/isInstrumentedPGOUse()));
1580
1581 // LoopSink (and other loop passes since the last simplifyCFG) might have
1582 // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
1583 OptimizePM.addPass(
1584 Pass: SimplifyCFGPass(SimplifyCFGOptions()
1585 .convertSwitchRangeToICmp(B: true)
1586 .convertSwitchToArithmetic(B: true)
1587 .speculateUnpredictables(B: true)
1588 .hoistLoadsStoresWithCondFaulting(B: true)));
1589
1590 // Add the core optimizing pipeline.
1591 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
1592 Pass: std::move(OptimizePM), EagerlyInvalidate: *PTO.EagerlyInvalidateAnalyses));
1593
1594 // AllocToken transforms heap allocation calls; this needs to run late after
1595 // other allocation call transformations (such as those in InstCombine).
1596 if (!isLTOPreLink(Phase: LTOPhase))
1597 MPM.addPass(Pass: AllocTokenPass());
1598
1599 invokeOptimizerLastEPCallbacks(MPM, Level, Phase: LTOPhase);
1600
1601 // Run the Instrumentor pass late.
1602 if (Opts.enable_instrumentor)
1603 MPM.addPass(Pass: InstrumentorPass(FS));
1604
1605 // Split out cold code. Splitting is done late to avoid hiding context from
1606 // other optimizations and inadvertently regressing performance. The tradeoff
1607 // is that this has a higher code size cost than splitting early.
1608 if (Opts.hot_cold_split && !isLTOPreLink(Phase: LTOPhase))
1609 MPM.addPass(Pass: HotColdSplittingPass());
1610
1611 // Now we need to do some global optimization transforms.
1612 // FIXME: It would seem like these should come first in the optimization
1613 // pipeline and maybe be the bottom of the canonicalization pipeline? Weird
1614 // ordering here.
1615 MPM.addPass(Pass: GlobalDCEPass());
1616 MPM.addPass(Pass: ConstantMergePass());
1617
1618 // Merge functions if requested. It has a better chance to merge functions
1619 // after ConstantMerge folded jump tables.
1620 if (*PTO.MergeFunctions)
1621 MPM.addPass(Pass: MergeFunctionsPass());
1622
1623 if (PTO.CallGraphProfile && !isLTOPreLink(Phase: LTOPhase))
1624 MPM.addPass(Pass: CGProfilePass(isLTOPostLink(Phase: LTOPhase)));
1625
1626 // RelLookupTableConverterPass runs later in LTO post-link pipeline.
1627 if (!isLTOPreLink(Phase: LTOPhase))
1628 MPM.addPass(Pass: RelLookupTableConverterPass());
1629
1630 // Add devirtualization pass only when LTO is not enabled, as otherwise
1631 // the pass is already enabled in the LTO pipeline.
1632 if (*PTO.DevirtualizeSpeculatively && LTOPhase == ThinOrFullLTOPhase::None) {
1633 // TODO: explore a better pipeline configuration that can improve
1634 // compilation time overhead.
1635 // FIXME: move this earlier (lots of pass ordering tests will need fixing)
1636 MPM.addPass(Pass: NameAnonGlobalPass());
1637 MPM.addPass(Pass: AssignGUIDPass());
1638 MPM.addPass(Pass: WholeProgramDevirtPass(
1639 /*ExportSummary*/ nullptr,
1640 /*ImportSummary*/ nullptr,
1641 /*DevirtSpeculatively*/ *PTO.DevirtualizeSpeculatively));
1642 MPM.addPass(Pass: DropTypeTestsPass());
1643 // Given that the devirtualization creates more opportunities for inlining,
1644 // we run the Inliner again here to maximize the optimization gain we
1645 // get from devirtualization.
1646 // Also, we can't run devirtualization before inlining because the
1647 // devirtualization depends on the passes optimizing/eliminating vtable GVs
1648 // and those passes are only effective after inlining.
1649 addModuleInlinerPass(MPM, Context, Level, Phase: ThinOrFullLTOPhase::None);
1650 }
1651
1652 // Attach !implicit.ref metadata from all functions to copyright strings.
1653 MPM.addPass(Pass: LowerCommentStringPass());
1654
1655 return MPM;
1656}
1657
1658ModulePassManager
1659PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level,
1660 ThinOrFullLTOPhase Phase) {
1661 if (Level == OptimizationLevel::O0)
1662 return buildO0DefaultPipeline(Level, Phase);
1663
1664 ModulePassManager MPM;
1665 addPipelineStartPasses(MPM, Opts);
1666 // Currently this pipeline is only invoked in an LTO pre link pass or when we
1667 // are not running LTO. If that changes the below checks may need updating.
1668 assert(isLTOPreLink(Phase) || Phase == ThinOrFullLTOPhase::None);
1669
1670 // If we are invoking this in non-LTO mode, remove any MemProf related
1671 // attributes and metadata, as we don't know whether we are linking with
1672 // a library containing the necessary interfaces.
1673 if (Phase == ThinOrFullLTOPhase::None)
1674 MPM.addPass(Pass: MemProfRemoveInfo());
1675
1676 // Convert @llvm.global.annotations to !annotation metadata.
1677 MPM.addPass(Pass: Annotation2MetadataPass());
1678
1679 // Force any function attributes we want the rest of the pipeline to observe.
1680 MPM.addPass(Pass: ForceFunctionAttrsPass());
1681
1682 if (Opts.opt_pipeline_trigger_crash)
1683 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: TriggerCrashFunctionPass()));
1684
1685 if (PGOOpt && PGOOpt->DebugInfoForProfiling)
1686 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: AddDiscriminatorsPass()));
1687
1688 // Apply module pipeline start EP callback.
1689 invokePipelineStartEPCallbacks(MPM, Level);
1690
1691 // Add the core simplification pipeline.
1692 MPM.addPass(Pass: buildModuleSimplificationPipeline(Level, Phase));
1693
1694 // Now add the optimization pipeline.
1695 MPM.addPass(Pass: buildModuleOptimizationPipeline(Level, LTOPhase: Phase));
1696
1697 if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&
1698 PGOOpt->Action == PGOOptions::SampleUse)
1699 MPM.addPass(Pass: PseudoProbeUpdatePass());
1700
1701 // Emit annotation remarks.
1702 addAnnotationRemarksPass(MPM);
1703
1704 if (isLTOPreLink(Phase))
1705 addRequiredLTOPreLinkPasses(MPM);
1706
1707 addPipelineEndPasses(MPM, Opts);
1708 return MPM;
1709}
1710
1711ModulePassManager
1712PassBuilder::buildFatLTODefaultPipeline(OptimizationLevel Level, bool ThinLTO,
1713 bool EmitSummary, bool Verify) {
1714 ModulePassManager MPM;
1715
1716 addPipelineStartPasses(MPM, Opts);
1717
1718 if (ThinLTO)
1719 MPM.addPass(Pass: buildThinLTOPreLinkDefaultPipeline(Level));
1720 else
1721 MPM.addPass(Pass: buildLTOPreLinkDefaultPipeline(Level));
1722 // AssignGUIDPass attaches !guid metadata (MD_unique_id) to global objects,
1723 // triggering the bitcode writer to emit a METADATA_KIND_BLOCK. Standard LTO
1724 // bitcode emission runs VerifierPass by default, which registers metadata
1725 // kind IDs in LLVMContext. Running VerifierPass here before EmbedBitcodePass
1726 // to get the same behavior.
1727 if (Verify)
1728 MPM.addPass(Pass: VerifierPass());
1729 MPM.addPass(Pass: EmbedBitcodePass(ThinLTO, EmitSummary));
1730
1731 // Perform any cleanups to the IR that aren't suitable for per TU compilation,
1732 // like removing CFI/WPD related instructions. Note, we reuse
1733 // DropTypeTestsPass to clean up type tests rather than duplicate that logic
1734 // in FatLtoCleanup; checked-load cleanup is shared the other way, with
1735 // FatLtoCleanup calling lowertypetests::cleanUpTypeCheckedLoad.
1736 MPM.addPass(Pass: FatLtoCleanup());
1737
1738 // If we're doing FatLTO w/ CFI enabled, we don't want the type tests in the
1739 // object code, only in the bitcode section, so drop it before we run
1740 // module optimization and generate machine code. If llvm.type.test() isn't in
1741 // the IR, this won't do anything.
1742 MPM.addPass(Pass: DropTypeTestsPass(lowertypetests::DropTestKind::All));
1743
1744 // Use the ThinLTO post-link pipeline with sample profiling
1745 if (ThinLTO && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse)
1746 MPM.addPass(Pass: buildThinLTODefaultPipeline(Level, /*ImportSummary=*/nullptr));
1747 else {
1748 // ModuleSimplification does not run the coroutine passes for
1749 // ThinLTOPreLink, so we need the coroutine passes to run for ThinLTO
1750 // builds, otherwise they will miscompile.
1751 if (ThinLTO) {
1752 // TODO: replace w/ buildCoroWrapper() when it takes phase and level into
1753 // consideration.
1754 CGSCCPassManager CGPM;
1755 CGPM.addPass(Pass: CoroSplitPass(Level != OptimizationLevel::O0));
1756 CGPM.addPass(Pass: CoroAnnotationElidePass());
1757 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: std::move(CGPM)));
1758 MPM.addPass(Pass: CoroCleanupPass());
1759 }
1760
1761 // otherwise, just use module optimization
1762 MPM.addPass(
1763 Pass: buildModuleOptimizationPipeline(Level, LTOPhase: ThinOrFullLTOPhase::None));
1764 // Emit annotation remarks.
1765 addAnnotationRemarksPass(MPM);
1766 }
1767
1768 addPipelineEndPasses(MPM, Opts);
1769
1770 return MPM;
1771}
1772
1773ModulePassManager
1774PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level) {
1775 if (Level == OptimizationLevel::O0)
1776 return buildO0DefaultPipeline(Level, Phase: ThinOrFullLTOPhase::ThinLTOPreLink);
1777
1778 ModulePassManager MPM;
1779
1780 addPipelineStartPasses(MPM, Opts);
1781
1782 // Convert @llvm.global.annotations to !annotation metadata.
1783 MPM.addPass(Pass: Annotation2MetadataPass());
1784
1785 // Force any function attributes we want the rest of the pipeline to observe.
1786 MPM.addPass(Pass: ForceFunctionAttrsPass());
1787
1788 if (PGOOpt && PGOOpt->DebugInfoForProfiling)
1789 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: AddDiscriminatorsPass()));
1790
1791 // Apply module pipeline start EP callback.
1792 invokePipelineStartEPCallbacks(MPM, Level);
1793
1794 // If we are planning to perform ThinLTO later, we don't bloat the code with
1795 // unrolling/vectorization/... now. Just simplify the module as much as we
1796 // can.
1797 MPM.addPass(Pass: buildModuleSimplificationPipeline(
1798 Level, Phase: ThinOrFullLTOPhase::ThinLTOPreLink));
1799 // In pre-link, for ctx prof use, we stop here with an instrumented IR. We let
1800 // thinlto use the contextual info to perform imports; then use the contextual
1801 // profile in the post-thinlink phase.
1802 if (!UseCtxProfile.empty()) {
1803 addRequiredLTOPreLinkPasses(MPM);
1804 return MPM;
1805 }
1806
1807 // Run partial inlining pass to partially inline functions that have
1808 // large bodies.
1809 // FIXME: It isn't clear whether this is really the right place to run this
1810 // in ThinLTO. Because there is another canonicalization and simplification
1811 // phase that will run after the thin link, running this here ends up with
1812 // less information than will be available later and it may grow functions in
1813 // ways that aren't beneficial.
1814 if (Opts.enable_partial_inlining)
1815 MPM.addPass(Pass: PartialInlinerPass());
1816
1817 if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&
1818 PGOOpt->Action == PGOOptions::SampleUse)
1819 MPM.addPass(Pass: PseudoProbeUpdatePass());
1820
1821 // Handle Optimizer{Early,Last}EPCallbacks added by clang on PreLink. Actual
1822 // optimization is going to be done in PostLink stage, but clang can't add
1823 // callbacks there in case of in-process ThinLTO called by linker.
1824 invokeOptimizerEarlyEPCallbacks(MPM, Level,
1825 /*Phase=*/ThinOrFullLTOPhase::ThinLTOPreLink);
1826 invokeOptimizerLastEPCallbacks(MPM, Level,
1827 /*Phase=*/ThinOrFullLTOPhase::ThinLTOPreLink);
1828
1829 // Emit annotation remarks.
1830 addAnnotationRemarksPass(MPM);
1831
1832 // Attach !implicit.ref metadata from all functions to copyright strings.
1833 MPM.addPass(Pass: LowerCommentStringPass());
1834
1835 addRequiredLTOPreLinkPasses(MPM);
1836
1837 addPipelineEndPasses(MPM, Opts);
1838
1839 return MPM;
1840}
1841
1842ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
1843 OptimizationLevel Level, const ModuleSummaryIndex *ImportSummary) {
1844 ModulePassManager MPM;
1845
1846 addPipelineStartPasses(MPM, Opts);
1847
1848 invokeThinLinkTimeOptimizationEarlyEPCallbacks(MPM, Level);
1849
1850 // If we are invoking this without a summary index noting that we are linking
1851 // with a library containing the necessary APIs, remove any MemProf related
1852 // attributes and metadata.
1853 if (!ImportSummary || !ImportSummary->withSupportsHotColdNew())
1854 MPM.addPass(Pass: MemProfRemoveInfo());
1855
1856 if (ImportSummary) {
1857 // For ThinLTO we must apply the context disambiguation decisions early, to
1858 // ensure we can correctly match the callsites to summary data.
1859 if (EnableMemProfContextDisambiguation)
1860 MPM.addPass(Pass: MemProfContextDisambiguation(
1861 ImportSummary, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
1862
1863 // WholeProgramDevirt imports type identifier resolutions for whole-program
1864 // devirtualization. It must run early because later passes may disturb the
1865 // specific instruction patterns that it looks for, creating dependencies on
1866 // resolutions that may not appear in the summary.
1867 //
1868 // For example, GVN may transform the pattern assume(type.test) appearing in
1869 // two basic blocks into assume(phi(type.test, type.test)), which would
1870 // transform a dependency on a WPD resolution into a dependency on a type
1871 // identifier resolution for CFI.
1872 //
1873 // Also, WPD has access to more precise information than ICP and can
1874 // devirtualize more effectively, so it should operate on the IR first.
1875 //
1876 // It also needs to run at -O0 to lower type metadata and intrinsics.
1877 MPM.addPass(Pass: WholeProgramDevirtPass(nullptr, ImportSummary));
1878 }
1879
1880 // LowerTypeTests renames CFI functions to <name>.cfi, so with optimization
1881 // it runs in buildModuleSimplificationPipeline after ICP. Otherwise, lower
1882 // type tests here.
1883 if (Level == OptimizationLevel::O0) {
1884 if (ImportSummary)
1885 MPM.addPass(Pass: LowerTypeTestsPass(nullptr, ImportSummary));
1886 // Run a second time to clean up any type tests left behind by WPD for use
1887 // in ICP.
1888 MPM.addPass(Pass: DropTypeTestsPass());
1889 MPM.addPass(Pass: buildCoroWrapper(Phase: ThinOrFullLTOPhase::ThinLTOPostLink));
1890
1891 // AllocToken transforms heap allocation calls; this needs to run late after
1892 // other allocation call transformations (such as those in InstCombine).
1893 MPM.addPass(Pass: AllocTokenPass());
1894
1895 // Drop available_externally and unreferenced globals. This is necessary
1896 // with ThinLTO in order to avoid leaving undefined references to dead
1897 // globals in the object file.
1898 MPM.addPass(Pass: EliminateAvailableExternallyPass());
1899 MPM.addPass(Pass: GlobalDCEPass());
1900
1901 invokeThinLinkTimeOptimizationLastEPCallbacks(MPM, Level);
1902
1903 return MPM;
1904 }
1905 if (!UseCtxProfile.empty()) {
1906 // FIXME: Move LowerTypeTestsPass (and DropTypeTestsPass) into
1907 // buildModuleInlinerPipeline after ModuleInlinerPass, which performs ICP.
1908 if (ImportSummary)
1909 MPM.addPass(Pass: LowerTypeTestsPass(nullptr, ImportSummary));
1910 MPM.addPass(
1911 Pass: buildModuleInlinerPipeline(Level, Phase: ThinOrFullLTOPhase::ThinLTOPostLink));
1912 } else {
1913 // Add the core simplification pipeline.
1914 MPM.addPass(Pass: buildModuleSimplificationPipeline(
1915 Level, Phase: ThinOrFullLTOPhase::ThinLTOPostLink, ImportSummary));
1916 }
1917 // Now add the optimization pipeline.
1918 MPM.addPass(Pass: buildModuleOptimizationPipeline(
1919 Level, LTOPhase: ThinOrFullLTOPhase::ThinLTOPostLink));
1920
1921 invokeThinLinkTimeOptimizationLastEPCallbacks(MPM, Level);
1922
1923 // Emit annotation remarks.
1924 addAnnotationRemarksPass(MPM);
1925
1926 addPipelineEndPasses(MPM, Opts);
1927
1928 return MPM;
1929}
1930
1931ModulePassManager
1932PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level) {
1933 // FIXME: We should use a customized pre-link pipeline!
1934 return buildPerModuleDefaultPipeline(Level,
1935 Phase: ThinOrFullLTOPhase::FullLTOPreLink);
1936}
1937
1938ModulePassManager
1939PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
1940 ModuleSummaryIndex *ExportSummary) {
1941 ModulePassManager MPM;
1942
1943 addPipelineStartPasses(MPM, Opts);
1944
1945 invokeFullLinkTimeOptimizationEarlyEPCallbacks(MPM, Level);
1946
1947 // If we are invoking this without a summary index noting that we are linking
1948 // with a library containing the necessary APIs, remove any MemProf related
1949 // attributes and metadata.
1950 if (!ExportSummary || !ExportSummary->withSupportsHotColdNew())
1951 MPM.addPass(Pass: MemProfRemoveInfo());
1952
1953 // Create a function that performs CFI checks for cross-DSO calls with targets
1954 // in the current module.
1955 MPM.addPass(Pass: CrossDSOCFIPass());
1956
1957 if (Level == OptimizationLevel::O0) {
1958 // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1959 // metadata and intrinsics.
1960 MPM.addPass(Pass: WholeProgramDevirtPass(ExportSummary, nullptr));
1961 MPM.addPass(Pass: LowerTypeTestsPass(ExportSummary, nullptr));
1962 // Run a second time to clean up any type tests left behind by WPD for use
1963 // in ICP.
1964 MPM.addPass(Pass: DropTypeTestsPass());
1965
1966 MPM.addPass(Pass: buildCoroWrapper(Phase: ThinOrFullLTOPhase::FullLTOPostLink));
1967
1968 // AllocToken transforms heap allocation calls; this needs to run late after
1969 // other allocation call transformations (such as those in InstCombine).
1970 MPM.addPass(Pass: AllocTokenPass());
1971
1972 invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);
1973
1974 // Emit annotation remarks.
1975 addAnnotationRemarksPass(MPM);
1976
1977 return MPM;
1978 }
1979
1980 if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
1981 // Load sample profile before running the LTO optimization pipeline.
1982 MPM.addPass(Pass: SampleProfileLoaderPass(PGOOpt->ProfileFile,
1983 PGOOpt->ProfileRemappingFile,
1984 ThinOrFullLTOPhase::FullLTOPostLink));
1985 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1986 // RequireAnalysisPass for PSI before subsequent non-module passes.
1987 MPM.addPass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
1988 }
1989
1990 // Try to run OpenMP optimizations, quick no-op if no OpenMP metadata present.
1991 MPM.addPass(Pass: OpenMPOptPass(ThinOrFullLTOPhase::FullLTOPostLink));
1992
1993 // Remove unused virtual tables to improve the quality of code generated by
1994 // whole-program devirtualization and bitset lowering.
1995 MPM.addPass(Pass: GlobalDCEPass(/*InLTOPostLink=*/true));
1996
1997 // Do basic inference of function attributes from known properties of system
1998 // libraries and other oracles.
1999 MPM.addPass(Pass: InferFunctionAttrsPass());
2000
2001 if (Level >= OptimizationLevel::O2) {
2002 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
2003 Pass: CallSiteSplittingPass(), EagerlyInvalidate: *PTO.EagerlyInvalidateAnalyses));
2004
2005 // Indirect call promotion. This should promote all the targets that are
2006 // left by the earlier promotion pass that promotes intra-module targets.
2007 // This two-step promotion is to save the compile time. For LTO, it should
2008 // produce the same result as if we only do promotion here.
2009 MPM.addPass(Pass: PGOIndirectCallPromotion(
2010 true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
2011
2012 // Promoting by-reference arguments to by-value exposes more constants to
2013 // IPSCCP.
2014 CGSCCPassManager CGPM;
2015 if (Opts.attributor_enable & AttributorRunOption::CGSCC_LIGHT)
2016 CGPM.addPass(Pass: AttributorLightCGSCCPass());
2017 else
2018 CGPM.addPass(Pass: PostOrderFunctionAttrsPass());
2019 CGPM.addPass(Pass: ArgumentPromotionPass());
2020 CGPM.addPass(
2021 Pass: createCGSCCToFunctionPassAdaptor(Pass: SROAPass(SROAOptions::ModifyCFG)));
2022 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: std::move(CGPM)));
2023
2024 // Propagate constants at call sites into the functions they call. This
2025 // opens opportunities for globalopt (and inlining) by substituting function
2026 // pointers passed as arguments to direct uses of functions.
2027 MPM.addPass(Pass: IPSCCPPass(IPSCCPOptions(/*AllowFuncSpec=*/true)));
2028
2029 // Attach metadata to indirect call sites indicating the set of functions
2030 // they may target at run-time. This should follow IPSCCP.
2031 MPM.addPass(Pass: CalledValuePropagationPass());
2032 }
2033
2034 // Do RPO function attribute inference across the module to forward-propagate
2035 // attributes where applicable.
2036 // FIXME: Is this really an optimization rather than a canonicalization?
2037 if (Opts.attributor_enable & AttributorRunOption::MODULE_LIGHT)
2038 MPM.addPass(Pass: AttributorLightPass());
2039 else
2040 MPM.addPass(Pass: ReversePostOrderFunctionAttrsPass());
2041
2042 // Use in-range annotations on GEP indices to split globals where beneficial.
2043 MPM.addPass(Pass: GlobalSplitPass());
2044
2045 // Run whole program optimization of virtual call when the list of callees
2046 // is fixed.
2047 MPM.addPass(Pass: WholeProgramDevirtPass(ExportSummary, nullptr));
2048
2049 MPM.addPass(Pass: NoRecurseLTOInferencePass());
2050 // Stop here at -O1.
2051 if (Level == OptimizationLevel::O1) {
2052 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
2053 Pass: LowerConstantIntrinsicsPass(), EagerlyInvalidate: *PTO.EagerlyInvalidateAnalyses));
2054
2055 // The LowerTypeTestsPass needs to run to lower type metadata and the
2056 // type.test intrinsics. The pass does nothing if CFI is disabled.
2057 MPM.addPass(Pass: LowerTypeTestsPass(ExportSummary, nullptr));
2058 // Run a second time to clean up any type tests left behind by WPD for use
2059 // in ICP (which is performed earlier than this in the regular LTO
2060 // pipeline).
2061 MPM.addPass(Pass: DropTypeTestsPass());
2062
2063 MPM.addPass(Pass: buildCoroWrapper(Phase: ThinOrFullLTOPhase::FullLTOPostLink));
2064
2065 // AllocToken transforms heap allocation calls; this needs to run late after
2066 // other allocation call transformations (such as those in InstCombine).
2067 MPM.addPass(Pass: AllocTokenPass());
2068
2069 invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);
2070
2071 // Emit annotation remarks.
2072 addAnnotationRemarksPass(MPM);
2073
2074 addPipelineEndPasses(MPM, Opts);
2075
2076 return MPM;
2077 }
2078
2079 // TODO: Skip to match buildCoroWrapper.
2080 MPM.addPass(Pass: CoroEarlyPass());
2081
2082 // Optimize globals to try and fold them into constants.
2083 MPM.addPass(Pass: GlobalOptPass());
2084
2085 // Promote any localized globals to SSA registers.
2086 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: PromotePass()));
2087
2088 // Linking modules together can lead to duplicate global constant, only
2089 // keep one copy of each constant.
2090 MPM.addPass(Pass: ConstantMergePass());
2091
2092 // Remove unused arguments from functions.
2093 MPM.addPass(Pass: DeadArgumentEliminationPass());
2094
2095 // Reduce the code after globalopt and ipsccp. Both can open up significant
2096 // simplification opportunities, and both can propagate functions through
2097 // function pointers. When this happens, we often have to resolve varargs
2098 // calls, etc, so let instcombine do this.
2099 FunctionPassManager PeepholeFPM;
2100 PeepholeFPM.addPass(Pass: InstCombinePass());
2101 if (Level >= OptimizationLevel::O2)
2102 PeepholeFPM.addPass(Pass: AggressiveInstCombinePass());
2103 invokePeepholeEPCallbacks(FPM&: PeepholeFPM, Level);
2104
2105 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
2106 Pass: std::move(PeepholeFPM), EagerlyInvalidate: *PTO.EagerlyInvalidateAnalyses));
2107
2108 // Lower variadic functions for supported targets prior to inlining.
2109 MPM.addPass(Pass: ExpandVariadicsPass(ExpandVariadicsMode::Optimize));
2110
2111 // Note: historically, the PruneEH pass was run first to deduce nounwind and
2112 // generally clean up exception handling overhead. It isn't clear this is
2113 // valuable as the inliner doesn't currently care whether it is inlining an
2114 // invoke or a call.
2115 // Run the inliner now.
2116 addModuleInlinerPass(MPM, Context, Level,
2117 Phase: ThinOrFullLTOPhase::FullLTOPostLink);
2118
2119 // Perform context disambiguation after inlining, since that would reduce the
2120 // amount of additional cloning required to distinguish the allocation
2121 // contexts.
2122 if (EnableMemProfContextDisambiguation)
2123 MPM.addPass(Pass: MemProfContextDisambiguation(
2124 /*Summary=*/nullptr,
2125 PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
2126
2127 // Optimize globals again after we ran the inliner.
2128 MPM.addPass(Pass: GlobalOptPass());
2129
2130 // Run the OpenMPOpt pass again after global optimizations.
2131 MPM.addPass(Pass: OpenMPOptPass(ThinOrFullLTOPhase::FullLTOPostLink));
2132
2133 // Garbage collect dead functions.
2134 MPM.addPass(Pass: GlobalDCEPass(/*InLTOPostLink=*/true));
2135
2136 // If we didn't decide to inline a function, check to see if we can
2137 // transform it to pass arguments by value instead of by reference.
2138 CGSCCPassManager CGPM;
2139 CGPM.addPass(Pass: ArgumentPromotionPass());
2140 CGPM.addPass(Pass: CoroSplitPass(Level != OptimizationLevel::O0));
2141 CGPM.addPass(Pass: CoroAnnotationElidePass());
2142 invokeCGSCCOptimizerLateEPCallbacks(CGPM, Level);
2143 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: std::move(CGPM)));
2144
2145 FunctionPassManager FPM;
2146 // The IPO Passes may leave cruft around. Clean up after them.
2147 FPM.addPass(Pass: InstCombinePass());
2148 invokePeepholeEPCallbacks(FPM, Level);
2149
2150 if (Opts.enable_constraint_elimination)
2151 FPM.addPass(Pass: ConstraintEliminationPass());
2152
2153 FPM.addPass(Pass: JumpThreadingPass());
2154
2155 // Do a post inline PGO instrumentation and use pass. This is a context
2156 // sensitive PGO pass.
2157 if (PGOOpt) {
2158 if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
2159 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/true,
2160 /*IsCS=*/true, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate,
2161 ProfileFile: PGOOpt->CSProfileGenFile, ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
2162 else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
2163 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/false,
2164 /*IsCS=*/true, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate,
2165 ProfileFile: PGOOpt->ProfileFile, ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
2166 }
2167
2168 // Break up allocas
2169 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
2170
2171 // LTO provides additional opportunities for tailcall elimination due to
2172 // link-time inlining, and visibility of nocapture attribute.
2173 FPM.addPass(
2174 Pass: TailCallElimPass(/*UpdateFunctionEntryCount=*/isInstrumentedPGOUse()));
2175
2176 // Run a few AA driver optimizations here and now to cleanup the code.
2177 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
2178 Pass: std::move(FPM), EagerlyInvalidate: *PTO.EagerlyInvalidateAnalyses));
2179
2180 if (Opts.attributor_enable & AttributorRunOption::CGSCC_LIGHT) {
2181 MPM.addPass(
2182 Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: AttributorLightCGSCCPass()));
2183 } else {
2184 MPM.addPass(
2185 Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: PostOrderFunctionAttrsPass()));
2186 }
2187
2188 // Require the GlobalsAA analysis for the module so we can query it within
2189 // MainFPM.
2190 if (Opts.enable_global_analyses) {
2191 MPM.addPass(Pass: RequireAnalysisPass<GlobalsAA, Module>());
2192 // Invalidate AAManager so it can be recreated and pick up the newly
2193 // available GlobalsAA.
2194 MPM.addPass(
2195 Pass: createModuleToFunctionPassAdaptor(Pass: InvalidateAnalysisPass<AAManager>()));
2196 }
2197
2198 FunctionPassManager MainFPM;
2199 MainFPM.addPass(Pass: createFunctionToLoopPassAdaptor(
2200 Pass: LICMPass(*PTO.LicmMssaOptCap, *PTO.LicmMssaNoAccForPromotionCap,
2201 /*AllowSpeculation=*/true),
2202 /*USeMemorySSA=*/UseMemorySSA: true));
2203
2204 if (Opts.enable_newgvn)
2205 MainFPM.addPass(Pass: NewGVNPass());
2206 else
2207 MainFPM.addPass(Pass: GVNPass());
2208
2209 // Remove dead memcpy()'s.
2210 MainFPM.addPass(Pass: MemCpyOptPass());
2211
2212 // Nuke dead stores.
2213 MainFPM.addPass(Pass: DSEPass());
2214 MainFPM.addPass(Pass: MoveAutoInitPass());
2215 MainFPM.addPass(Pass: MergedLoadStoreMotionPass());
2216
2217 MainFPM.addPass(Pass: LowerConstantIntrinsicsPass());
2218
2219 invokeVectorizerStartEPCallbacks(FPM&: MainFPM, Level);
2220
2221 LoopPassManager LPM;
2222 if (Opts.enable_loop_flatten && Level >= OptimizationLevel::O2)
2223 LPM.addPass(Pass: LoopFlattenPass());
2224 LPM.addPass(Pass: IndVarSimplifyPass());
2225 LPM.addPass(Pass: LoopDeletionPass());
2226 // FIXME: Add loop interchange.
2227
2228 // The loop passes in LPM (LoopFullUnrollPass) do not preserve MemorySSA.
2229 // *All* loop passes must preserve it, in order to be able to use it.
2230 // TODO: Explore enabling MemorySSA for loop passes after we remove the
2231 // LoopFullUnrollPass.
2232 MainFPM.addPass(
2233 Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM), /*UseMemorySSA=*/false));
2234
2235 MainFPM.addPass(Pass: LoopDistributePass());
2236
2237 addVectorPasses(Level, FPM&: MainFPM, LTOPhase: ThinOrFullLTOPhase::FullLTOPostLink);
2238
2239 invokeVectorizerEndEPCallbacks(FPM&: MainFPM, Level);
2240
2241 // Run the OpenMPOpt CGSCC pass again late.
2242 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(
2243 Pass: OpenMPOptCGSCCPass(ThinOrFullLTOPhase::FullLTOPostLink)));
2244
2245 invokePeepholeEPCallbacks(FPM&: MainFPM, Level);
2246 MainFPM.addPass(Pass: JumpThreadingPass());
2247 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
2248 Pass: std::move(MainFPM), EagerlyInvalidate: *PTO.EagerlyInvalidateAnalyses));
2249
2250 // Lower type metadata and the type.test intrinsic. This pass supports
2251 // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
2252 // to be run at link time if CFI is enabled. This pass does nothing if
2253 // CFI is disabled.
2254 MPM.addPass(Pass: LowerTypeTestsPass(ExportSummary, nullptr));
2255 // Run a second time to clean up any type tests left behind by WPD for use
2256 // in ICP (which is performed earlier than this in the regular LTO pipeline).
2257 MPM.addPass(Pass: DropTypeTestsPass());
2258
2259 // Enable splitting late in the FullLTO post-link pipeline.
2260 if (Opts.hot_cold_split)
2261 MPM.addPass(Pass: HotColdSplittingPass());
2262
2263 // Add late LTO optimization passes.
2264 FunctionPassManager LateFPM;
2265
2266 // LoopSink pass sinks instructions hoisted by LICM, which serves as a
2267 // canonicalization pass that enables other optimizations. As a result,
2268 // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
2269 // result too early.
2270 LateFPM.addPass(Pass: LoopSinkPass());
2271
2272 // This hoists/decomposes div/rem ops. It should run after other sink/hoist
2273 // passes to avoid re-sinking, but before SimplifyCFG because it can allow
2274 // flattening of blocks.
2275 LateFPM.addPass(Pass: DivRemPairsPass());
2276
2277 // Merge adjacent icmps into memcmp, then expand memcmp to loads/compares.
2278 // TODO: move this further up so that it can be optimized by GVN, etc.
2279 if (Opts.enable_mergeicmps)
2280 LateFPM.addPass(Pass: MergeICmpsPass());
2281 LateFPM.addPass(Pass: ExpandMemCmpPass());
2282
2283 // Delete basic blocks, which optimization passes may have killed.
2284 LateFPM.addPass(Pass: SimplifyCFGPass(SimplifyCFGOptions()
2285 .convertSwitchRangeToICmp(B: true)
2286 .convertSwitchToArithmetic(B: true)
2287 .hoistCommonInsts(B: true)
2288 .speculateUnpredictables(B: true)));
2289 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(LateFPM)));
2290
2291 // Drop bodies of available eternally objects to improve GlobalDCE.
2292 MPM.addPass(Pass: EliminateAvailableExternallyPass());
2293
2294 // Now that we have optimized the program, discard unreachable functions.
2295 MPM.addPass(Pass: GlobalDCEPass(/*InLTOPostLink=*/true));
2296
2297 if (*PTO.MergeFunctions)
2298 MPM.addPass(Pass: MergeFunctionsPass());
2299
2300 MPM.addPass(Pass: RelLookupTableConverterPass());
2301
2302 if (PTO.CallGraphProfile)
2303 MPM.addPass(Pass: CGProfilePass(/*InLTOPostLink=*/true));
2304
2305 MPM.addPass(Pass: CoroCleanupPass());
2306
2307 // AllocToken transforms heap allocation calls; this needs to run late after
2308 // other allocation call transformations (such as those in InstCombine).
2309 MPM.addPass(Pass: AllocTokenPass());
2310
2311 invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);
2312
2313 // Emit annotation remarks.
2314 addAnnotationRemarksPass(MPM);
2315
2316 addPipelineEndPasses(MPM, Opts);
2317
2318 return MPM;
2319}
2320
2321ModulePassManager
2322PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level,
2323 ThinOrFullLTOPhase Phase) {
2324 assert(Level == OptimizationLevel::O0 &&
2325 "buildO0DefaultPipeline should only be used with O0");
2326
2327 ModulePassManager MPM;
2328
2329 addPipelineStartPasses(MPM, Opts);
2330
2331 // Perform pseudo probe instrumentation in O0 mode. This is for the
2332 // consistency between different build modes. For example, a LTO build can be
2333 // mixed with an O0 prelink and an O2 postlink. Loading a sample profile in
2334 // the postlink will require pseudo probe instrumentation in the prelink.
2335 if (PGOOpt && PGOOpt->PseudoProbeForProfiling)
2336 MPM.addPass(Pass: SampleProfileProbePass(TM));
2337
2338 if (PGOOpt && (PGOOpt->Action == PGOOptions::IRInstr ||
2339 PGOOpt->Action == PGOOptions::IRUse))
2340 addPGOInstrPassesForO0(
2341 MPM,
2342 /*RunProfileGen=*/(PGOOpt->Action == PGOOptions::IRInstr),
2343 /*IsCS=*/false, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate, ProfileFile: PGOOpt->ProfileFile,
2344 ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
2345
2346 // Instrument function entry and exit before all inlining.
2347 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
2348 Pass: EntryExitInstrumenterPass(/*PostInlining=*/false)));
2349
2350 invokePipelineStartEPCallbacks(MPM, Level);
2351
2352 if (PGOOpt && PGOOpt->DebugInfoForProfiling)
2353 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: AddDiscriminatorsPass()));
2354
2355 if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
2356 // Explicitly disable sample loader inlining and use flattened profile in O0
2357 // pipeline.
2358 MPM.addPass(Pass: SampleProfileLoaderPass(PGOOpt->ProfileFile,
2359 PGOOpt->ProfileRemappingFile,
2360 ThinOrFullLTOPhase::None, FS,
2361 /*DisableSampleProfileInlining=*/true,
2362 /*UseFlattenedProfile=*/true));
2363 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
2364 // RequireAnalysisPass for PSI before subsequent non-module passes.
2365 MPM.addPass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
2366 }
2367
2368 invokePipelineEarlySimplificationEPCallbacks(MPM, Level, Phase);
2369
2370 // Build a minimal pipeline based on the semantics required by LLVM,
2371 // which is just that always inlining occurs. Further, disable generating
2372 // lifetime intrinsics to avoid enabling further optimizations during
2373 // code generation.
2374 MPM.addPass(Pass: AlwaysInlinerPass(
2375 /*InsertLifetimeIntrinsics=*/false));
2376
2377 if (*PTO.MergeFunctions)
2378 MPM.addPass(Pass: MergeFunctionsPass());
2379
2380 if (Opts.enable_matrix)
2381 MPM.addPass(
2382 Pass: createModuleToFunctionPassAdaptor(Pass: LowerMatrixIntrinsicsPass(true)));
2383
2384 if (!CGSCCOptimizerLateEPCallbacks.empty()) {
2385 CGSCCPassManager CGPM;
2386 invokeCGSCCOptimizerLateEPCallbacks(CGPM, Level);
2387 if (!CGPM.isEmpty())
2388 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: std::move(CGPM)));
2389 }
2390 if (!LateLoopOptimizationsEPCallbacks.empty()) {
2391 LoopPassManager LPM;
2392 invokeLateLoopOptimizationsEPCallbacks(LPM, Level);
2393 if (!LPM.isEmpty()) {
2394 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
2395 Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM))));
2396 }
2397 }
2398 if (!LoopOptimizerEndEPCallbacks.empty()) {
2399 LoopPassManager LPM;
2400 invokeLoopOptimizerEndEPCallbacks(LPM, Level);
2401 if (!LPM.isEmpty()) {
2402 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
2403 Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM))));
2404 }
2405 }
2406 if (!ScalarOptimizerLateEPCallbacks.empty()) {
2407 FunctionPassManager FPM;
2408 invokeScalarOptimizerLateEPCallbacks(FPM, Level);
2409 if (!FPM.isEmpty())
2410 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
2411 }
2412
2413 invokeOptimizerEarlyEPCallbacks(MPM, Level, Phase);
2414
2415 if (!VectorizerStartEPCallbacks.empty()) {
2416 FunctionPassManager FPM;
2417 invokeVectorizerStartEPCallbacks(FPM, Level);
2418 if (!FPM.isEmpty())
2419 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
2420 }
2421
2422 if (!VectorizerEndEPCallbacks.empty()) {
2423 FunctionPassManager FPM;
2424 invokeVectorizerEndEPCallbacks(FPM, Level);
2425 if (!FPM.isEmpty())
2426 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
2427 }
2428
2429 MPM.addPass(Pass: buildCoroWrapper(Phase));
2430
2431 // AllocToken transforms heap allocation calls; this needs to run late after
2432 // other allocation call transformations (such as those in InstCombine).
2433 if (!isLTOPreLink(Phase))
2434 MPM.addPass(Pass: AllocTokenPass());
2435
2436 invokeOptimizerLastEPCallbacks(MPM, Level, Phase);
2437
2438 if (Opts.enable_instrumentor)
2439 MPM.addPass(Pass: InstrumentorPass(FS));
2440
2441 // Attach !implicit.ref metadata from all functions to copyright strings.
2442 MPM.addPass(Pass: LowerCommentStringPass());
2443
2444 if (isLTOPreLink(Phase))
2445 addRequiredLTOPreLinkPasses(MPM);
2446
2447 // Emit annotation remarks.
2448 addAnnotationRemarksPass(MPM);
2449
2450 addPipelineEndPasses(MPM, Opts);
2451
2452 return MPM;
2453}
2454
2455AAManager PassBuilder::buildDefaultAAPipeline() {
2456 AAManager AA;
2457
2458 // The order in which these are registered determines their priority when
2459 // being queried.
2460
2461 // Add any target-specific alias analyses that should be run early.
2462 if (TM)
2463 TM->registerEarlyDefaultAliasAnalyses(AA);
2464
2465 // First we register the basic alias analysis that provides the majority of
2466 // per-function local AA logic. This is a stateless, on-demand local set of
2467 // AA techniques.
2468 AA.registerFunctionAnalysis<BasicAA>();
2469
2470 // Next we query fast, specialized alias analyses that wrap IR-embedded
2471 // information about aliasing.
2472 AA.registerFunctionAnalysis<ScopedNoAliasAA>();
2473 AA.registerFunctionAnalysis<TypeBasedAA>();
2474
2475 // Add support for querying global aliasing information when available.
2476 // Because the `AAManager` is a function analysis and `GlobalsAA` is a module
2477 // analysis, all that the `AAManager` can do is query for any *cached*
2478 // results from `GlobalsAA` through a readonly proxy.
2479 if (Opts.enable_global_analyses)
2480 AA.registerModuleAnalysis<GlobalsAA>();
2481
2482 // Add target-specific alias analyses.
2483 if (TM)
2484 TM->registerDefaultAliasAnalyses(AA);
2485
2486 return AA;
2487}
2488
2489bool PassBuilder::isInstrumentedPGOUse() const {
2490 return (PGOOpt && PGOOpt->Action == PGOOptions::IRUse) ||
2491 !UseCtxProfile.empty();
2492}
2493