1//===- Construction of pass pipelines -------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9///
10/// This file provides the implementation of the PassBuilder based on our
11/// static pass registry as well as related functionality. It also provides
12/// helpers to aid in analyzing, debugging, and testing passes and pass
13/// pipelines.
14///
15//===----------------------------------------------------------------------===//
16
17#include "PassesOptions.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/Analysis/AliasAnalysis.h"
20#include "llvm/Analysis/BasicAliasAnalysis.h"
21#include "llvm/Analysis/CGSCCPassManager.h"
22#include "llvm/Analysis/FunctionPropertiesAnalysis.h"
23#include "llvm/Analysis/GlobalsModRef.h"
24#include "llvm/Analysis/InlineAdvisor.h"
25#include "llvm/Analysis/InstCount.h"
26#include "llvm/Analysis/ProfileSummaryInfo.h"
27#include "llvm/Analysis/ScopedNoAliasAA.h"
28#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
29#include "llvm/IR/PassManager.h"
30#include "llvm/IR/Verifier.h"
31#include "llvm/Pass.h"
32#include "llvm/Passes/OptimizationLevel.h"
33#include "llvm/Passes/PassBuilder.h"
34#include "llvm/Passes/TriggerCrashPasses.h"
35#include "llvm/Support/CommandLine.h"
36#include "llvm/Support/ErrorHandling.h"
37#include "llvm/Support/PGOOptions.h"
38#include "llvm/Target/TargetMachine.h"
39#include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
40#include "llvm/Transforms/Coroutines/CoroAnnotationElide.h"
41#include "llvm/Transforms/Coroutines/CoroCleanup.h"
42#include "llvm/Transforms/Coroutines/CoroConditionalWrapper.h"
43#include "llvm/Transforms/Coroutines/CoroEarly.h"
44#include "llvm/Transforms/Coroutines/CoroElide.h"
45#include "llvm/Transforms/Coroutines/CoroSplit.h"
46#include "llvm/Transforms/HipStdPar/HipStdPar.h"
47#include "llvm/Transforms/IPO/AlwaysInliner.h"
48#include "llvm/Transforms/IPO/Annotation2Metadata.h"
49#include "llvm/Transforms/IPO/ArgumentPromotion.h"
50#include "llvm/Transforms/IPO/Attributor.h"
51#include "llvm/Transforms/IPO/CalledValuePropagation.h"
52#include "llvm/Transforms/IPO/ConstantMerge.h"
53#include "llvm/Transforms/IPO/CrossDSOCFI.h"
54#include "llvm/Transforms/IPO/DeadArgumentElimination.h"
55#include "llvm/Transforms/IPO/ElimAvailExtern.h"
56#include "llvm/Transforms/IPO/EmbedBitcodePass.h"
57#include "llvm/Transforms/IPO/ExpandVariadics.h"
58#include "llvm/Transforms/IPO/FatLTOCleanup.h"
59#include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
60#include "llvm/Transforms/IPO/FunctionAttrs.h"
61#include "llvm/Transforms/IPO/GlobalDCE.h"
62#include "llvm/Transforms/IPO/GlobalOpt.h"
63#include "llvm/Transforms/IPO/GlobalSplit.h"
64#include "llvm/Transforms/IPO/HotColdSplitting.h"
65#include "llvm/Transforms/IPO/InferFunctionAttrs.h"
66#include "llvm/Transforms/IPO/Inliner.h"
67#include "llvm/Transforms/IPO/Instrumentor.h"
68#include "llvm/Transforms/IPO/LowerTypeTests.h"
69#include "llvm/Transforms/IPO/MemProfContextDisambiguation.h"
70#include "llvm/Transforms/IPO/MergeFunctions.h"
71#include "llvm/Transforms/IPO/ModuleInliner.h"
72#include "llvm/Transforms/IPO/OpenMPOpt.h"
73#include "llvm/Transforms/IPO/PartialInlining.h"
74#include "llvm/Transforms/IPO/SCCP.h"
75#include "llvm/Transforms/IPO/SampleProfile.h"
76#include "llvm/Transforms/IPO/SampleProfileProbe.h"
77#include "llvm/Transforms/IPO/WholeProgramDevirt.h"
78#include "llvm/Transforms/InstCombine/InstCombine.h"
79#include "llvm/Transforms/Instrumentation/AllocToken.h"
80#include "llvm/Transforms/Instrumentation/CGProfile.h"
81#include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
82#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
83#include "llvm/Transforms/Instrumentation/MemProfInstrumentation.h"
84#include "llvm/Transforms/Instrumentation/MemProfUse.h"
85#include "llvm/Transforms/Instrumentation/PGOCtxProfFlattening.h"
86#include "llvm/Transforms/Instrumentation/PGOCtxProfLowering.h"
87#include "llvm/Transforms/Instrumentation/PGOForceFunctionAttrs.h"
88#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
89#include "llvm/Transforms/Scalar/ADCE.h"
90#include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
91#include "llvm/Transforms/Scalar/AnnotationRemarks.h"
92#include "llvm/Transforms/Scalar/BDCE.h"
93#include "llvm/Transforms/Scalar/CallSiteSplitting.h"
94#include "llvm/Transforms/Scalar/ConstraintElimination.h"
95#include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
96#include "llvm/Transforms/Scalar/DFAJumpThreading.h"
97#include "llvm/Transforms/Scalar/DeadStoreElimination.h"
98#include "llvm/Transforms/Scalar/DivRemPairs.h"
99#include "llvm/Transforms/Scalar/DropUnnecessaryAssumes.h"
100#include "llvm/Transforms/Scalar/EarlyCSE.h"
101#include "llvm/Transforms/Scalar/ExpandMemCmp.h"
102#include "llvm/Transforms/Scalar/Float2Int.h"
103#include "llvm/Transforms/Scalar/GVN.h"
104#include "llvm/Transforms/Scalar/GVNHoist.h"
105#include "llvm/Transforms/Scalar/GVNSink.h"
106#include "llvm/Transforms/Scalar/IndVarSimplify.h"
107#include "llvm/Transforms/Scalar/InferAlignment.h"
108#include "llvm/Transforms/Scalar/InstSimplifyPass.h"
109#include "llvm/Transforms/Scalar/JumpTableToSwitch.h"
110#include "llvm/Transforms/Scalar/JumpThreading.h"
111#include "llvm/Transforms/Scalar/LICM.h"
112#include "llvm/Transforms/Scalar/LoopDeletion.h"
113#include "llvm/Transforms/Scalar/LoopDistribute.h"
114#include "llvm/Transforms/Scalar/LoopFlatten.h"
115#include "llvm/Transforms/Scalar/LoopFuse.h"
116#include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
117#include "llvm/Transforms/Scalar/LoopInstSimplify.h"
118#include "llvm/Transforms/Scalar/LoopInterchange.h"
119#include "llvm/Transforms/Scalar/LoopLoadElimination.h"
120#include "llvm/Transforms/Scalar/LoopPassManager.h"
121#include "llvm/Transforms/Scalar/LoopRotation.h"
122#include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
123#include "llvm/Transforms/Scalar/LoopSink.h"
124#include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
125#include "llvm/Transforms/Scalar/LoopUnrollPass.h"
126#include "llvm/Transforms/Scalar/LoopVersioningLICM.h"
127#include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
128#include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
129#include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h"
130#include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
131#include "llvm/Transforms/Scalar/MergeICmps.h"
132#include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"
133#include "llvm/Transforms/Scalar/NewGVN.h"
134#include "llvm/Transforms/Scalar/Reassociate.h"
135#include "llvm/Transforms/Scalar/SCCP.h"
136#include "llvm/Transforms/Scalar/SROA.h"
137#include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
138#include "llvm/Transforms/Scalar/SimplifyCFG.h"
139#include "llvm/Transforms/Scalar/SpeculativeExecution.h"
140#include "llvm/Transforms/Scalar/TailRecursionElimination.h"
141#include "llvm/Transforms/Scalar/WarnMissedTransforms.h"
142#include "llvm/Transforms/Utils/AddDiscriminators.h"
143#include "llvm/Transforms/Utils/AssignGUID.h"
144#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
145#include "llvm/Transforms/Utils/CanonicalizeAliases.h"
146#include "llvm/Transforms/Utils/CountVisits.h"
147#include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
148#include "llvm/Transforms/Utils/ExtraPassManager.h"
149#include "llvm/Transforms/Utils/InjectTLIMappings.h"
150#include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
151#include "llvm/Transforms/Utils/LowerCommentStringPass.h"
152#include "llvm/Transforms/Utils/Mem2Reg.h"
153#include "llvm/Transforms/Utils/MoveAutoInit.h"
154#include "llvm/Transforms/Utils/NameAnonGlobals.h"
155#include "llvm/Transforms/Utils/ProfileVerify.h"
156#include "llvm/Transforms/Utils/RelLookupTableConverter.h"
157#include "llvm/Transforms/Utils/SimplifyCFGOptions.h"
158#include "llvm/Transforms/Vectorize/LoopVectorize.h"
159#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
160#include "llvm/Transforms/Vectorize/VectorCombine.h"
161
162using namespace llvm;
163
164namespace llvm {
165extern cl::opt<std::string> UseCtxProfile;
166
167extern cl::opt<bool> EnableMemProfContextDisambiguation;
168} // namespace llvm
169
170PipelineTuningOptions::PipelineTuningOptions(
171 const cl::LibraryOptionsContext *LibraryOptions)
172 : LibraryOptions(LibraryOptions) {
173 const PassesOptions &Opts =
174 cl::LibraryOptionsContext::get<PassesOptions>(C: LibraryOptions);
175 LoopInterleaving = true;
176 LoopVectorization = true;
177 SLPVectorization = false;
178 LoopUnrolling = true;
179 LoopInterchange = Opts.enable_loopinterchange;
180 LoopFusion = false;
181 ForgetAllSCEVInLoopUnroll = getForgetSCEVInLoopUnroll(LibraryOptions);
182 LicmMssaOptCap = getLicmMssaOptCap(LibraryOptions);
183 LicmMssaNoAccForPromotionCap =
184 getLicmMssaNoAccForPromotionCap(LibraryOptions);
185 CallGraphProfile = true;
186 UnifiedLTO = false;
187 MergeFunctions = Opts.enable_merge_functions;
188 InlinerThreshold = -1;
189 EagerlyInvalidateAnalyses = Opts.eagerly_invalidate_analyses;
190 DevirtualizeSpeculatively = Opts.enable_devirtualize_speculatively;
191}
192
193namespace llvm {
194extern cl::opt<unsigned> MaxDevirtIterations;
195} // namespace llvm
196
197void PassBuilder::invokePeepholeEPCallbacks(FunctionPassManager &FPM,
198 OptimizationLevel Level) {
199 for (auto &C : PeepholeEPCallbacks)
200 C(FPM, Level);
201}
202void PassBuilder::invokeLateLoopOptimizationsEPCallbacks(
203 LoopPassManager &LPM, OptimizationLevel Level) {
204 for (auto &C : LateLoopOptimizationsEPCallbacks)
205 C(LPM, Level);
206}
207void PassBuilder::invokeLoopOptimizerEndEPCallbacks(LoopPassManager &LPM,
208 OptimizationLevel Level) {
209 for (auto &C : LoopOptimizerEndEPCallbacks)
210 C(LPM, Level);
211}
212void PassBuilder::invokeScalarOptimizerLateEPCallbacks(
213 FunctionPassManager &FPM, OptimizationLevel Level) {
214 for (auto &C : ScalarOptimizerLateEPCallbacks)
215 C(FPM, Level);
216}
217void PassBuilder::invokeCGSCCOptimizerLateEPCallbacks(CGSCCPassManager &CGPM,
218 OptimizationLevel Level) {
219 for (auto &C : CGSCCOptimizerLateEPCallbacks)
220 C(CGPM, Level);
221}
222void PassBuilder::invokeVectorizerStartEPCallbacks(FunctionPassManager &FPM,
223 OptimizationLevel Level) {
224 for (auto &C : VectorizerStartEPCallbacks)
225 C(FPM, Level);
226}
227void PassBuilder::invokeVectorizerEndEPCallbacks(FunctionPassManager &FPM,
228 OptimizationLevel Level) {
229 for (auto &C : VectorizerEndEPCallbacks)
230 C(FPM, Level);
231}
232void PassBuilder::invokeOptimizerEarlyEPCallbacks(ModulePassManager &MPM,
233 OptimizationLevel Level,
234 ThinOrFullLTOPhase Phase) {
235 for (auto &C : OptimizerEarlyEPCallbacks)
236 C(MPM, Level, Phase);
237}
238void PassBuilder::invokeOptimizerLastEPCallbacks(ModulePassManager &MPM,
239 OptimizationLevel Level,
240 ThinOrFullLTOPhase Phase) {
241 for (auto &C : OptimizerLastEPCallbacks)
242 C(MPM, Level, Phase);
243}
244void PassBuilder::invokeFullLinkTimeOptimizationEarlyEPCallbacks(
245 ModulePassManager &MPM, OptimizationLevel Level) {
246 for (auto &C : FullLinkTimeOptimizationEarlyEPCallbacks)
247 C(MPM, Level);
248}
249void PassBuilder::invokeFullLinkTimeOptimizationLastEPCallbacks(
250 ModulePassManager &MPM, OptimizationLevel Level) {
251 for (auto &C : FullLinkTimeOptimizationLastEPCallbacks)
252 C(MPM, Level);
253}
254void PassBuilder::invokeThinLinkTimeOptimizationEarlyEPCallbacks(
255 ModulePassManager &MPM, OptimizationLevel Level) {
256 for (auto &C : ThinLinkTimeOptimizationEarlyEPCallbacks)
257 C(MPM, Level);
258}
259void PassBuilder::invokeThinLinkTimeOptimizationLastEPCallbacks(
260 ModulePassManager &MPM, OptimizationLevel Level) {
261 for (auto &C : ThinLinkTimeOptimizationLastEPCallbacks)
262 C(MPM, Level);
263}
264void PassBuilder::invokePipelineStartEPCallbacks(ModulePassManager &MPM,
265 OptimizationLevel Level) {
266 for (auto &C : PipelineStartEPCallbacks)
267 C(MPM, Level);
268}
269void PassBuilder::invokePipelineEarlySimplificationEPCallbacks(
270 ModulePassManager &MPM, OptimizationLevel Level, ThinOrFullLTOPhase Phase) {
271 for (auto &C : PipelineEarlySimplificationEPCallbacks)
272 C(MPM, Level, Phase);
273}
274
275// Get IR stats with InstCount before/after the optimization pipeline
276static void instructionCountersPass(ModulePassManager &MPM,
277 bool IsPreOptimization) {
278 if (AreStatisticsEnabled()) {
279 MPM.addPass(
280 Pass: createModuleToFunctionPassAdaptor(Pass: InstCountPass(IsPreOptimization)));
281 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
282 Pass: FunctionPropertiesStatisticsPass(IsPreOptimization)));
283 }
284}
285
286// Add passes to run at the start of the pipeline. This includes things like
287// statistics passes.
288static void addPipelineStartPasses(ModulePassManager &MPM,
289 const PassesOptions &Opts) {
290 instructionCountersPass(MPM, /*IsPreOptimization=*/true);
291 if (Opts.enable_profcheck_in_pipeline)
292 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: ProfileInjectorPass()));
293}
294
295// Add passes to run at the end of the pipeline. This includes things like
296// statistics passes.
297static void addPipelineEndPasses(ModulePassManager &MPM,
298 const PassesOptions &Opts) {
299 instructionCountersPass(MPM, /*IsPreOptimization=*/false);
300 if (Opts.enable_profcheck_in_pipeline)
301 MPM.addPass(Pass: ProfileVerifierPass());
302}
303
304// Helper to add AnnotationRemarksPass.
305static void addAnnotationRemarksPass(ModulePassManager &MPM) {
306 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: AnnotationRemarksPass()));
307}
308
309// Helper to check if the current compilation phase is preparing for LTO
310static bool isLTOPreLink(ThinOrFullLTOPhase Phase) {
311 return Phase == ThinOrFullLTOPhase::ThinLTOPreLink ||
312 Phase == ThinOrFullLTOPhase::FullLTOPreLink;
313}
314
315// Helper to check if the current compilation phase is preparing for FullLTO
316[[maybe_unused]] static bool isFullLTOPreLink(ThinOrFullLTOPhase Phase) {
317 return Phase == ThinOrFullLTOPhase::FullLTOPreLink;
318}
319
320// Helper to check if the current compilation phase is preparing for ThinLTO
321static bool isThinLTOPreLink(ThinOrFullLTOPhase Phase) {
322 return Phase == ThinOrFullLTOPhase::ThinLTOPreLink;
323}
324
325// Helper to check if the current compilation phase is LTO backend
326static bool isLTOPostLink(ThinOrFullLTOPhase Phase) {
327 return Phase == ThinOrFullLTOPhase::ThinLTOPostLink ||
328 Phase == ThinOrFullLTOPhase::FullLTOPostLink;
329}
330
331// Helper to check if the current compilation phase is FullLTO backend
332static bool isFullLTOPostLink(ThinOrFullLTOPhase Phase) {
333 return Phase == ThinOrFullLTOPhase::FullLTOPostLink;
334}
335
336// Helper to check if the current compilation phase is ThinLTO backend
337static bool isThinLTOPostLink(ThinOrFullLTOPhase Phase) {
338 return Phase == ThinOrFullLTOPhase::ThinLTOPostLink;
339}
340
341// Helper to wrap conditionally Coro passes.
342static CoroConditionalWrapper buildCoroWrapper(ThinOrFullLTOPhase Phase) {
343 // TODO: Skip passes according to Phase.
344 ModulePassManager CoroPM;
345 CoroPM.addPass(Pass: CoroEarlyPass());
346 CGSCCPassManager CGPM;
347 CGPM.addPass(Pass: CoroSplitPass());
348 CoroPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: std::move(CGPM)));
349 CoroPM.addPass(Pass: CoroCleanupPass());
350 CoroPM.addPass(Pass: GlobalDCEPass());
351 return CoroConditionalWrapper(std::move(CoroPM));
352}
353
354static InlineParams getInlineParamsFromOptLevel(OptimizationLevel Level) {
355 return getInlineParamsFromOptLevel(OptLevel: static_cast<unsigned>(Level));
356}
357
358static void addModuleInlinerPass(ModulePassManager &MPM,
359 const PassesOptions &Opts,
360 OptimizationLevel Level,
361 ThinOrFullLTOPhase Phase) {
362 InlineParams IP = ::getInlineParamsFromOptLevel(Level);
363 if (Opts.enable_module_inliner)
364 MPM.addPass(Pass: ModuleInlinerPass(IP, Opts.enable_ml_inliner, Phase));
365 else
366 MPM.addPass(Pass: ModuleInlinerWrapperPass(
367 IP,
368 /* MandatoryFirst */ true,
369 InlineContext{.LTOPhase: Phase, .Pass: InlinePass::CGSCCInliner}));
370}
371
372// TODO: Investigate the cost/benefit of tail call elimination on debugging.
373FunctionPassManager
374PassBuilder::buildO1FunctionSimplificationPipeline(OptimizationLevel Level,
375 ThinOrFullLTOPhase Phase) {
376
377 FunctionPassManager FPM;
378
379 if (AreStatisticsEnabled())
380 FPM.addPass(Pass: CountVisitsPass());
381
382 // Form SSA out of local memory accesses after breaking apart aggregates into
383 // scalars.
384 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
385
386 // Catch trivial redundancies
387 FPM.addPass(Pass: EarlyCSEPass(true /* Enable mem-ssa. */));
388
389 // Hoisting of scalars and load expressions.
390 FPM.addPass(
391 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
392 FPM.addPass(Pass: InstCombinePass());
393
394 FPM.addPass(Pass: LibCallsShrinkWrapPass());
395
396 invokePeepholeEPCallbacks(FPM, Level);
397
398 FPM.addPass(
399 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
400
401 // Form canonically associated expression trees, and simplify the trees using
402 // basic mathematical properties. For example, this will form (nearly)
403 // minimal multiplication trees.
404 FPM.addPass(Pass: ReassociatePass());
405
406 // Add the primary loop simplification pipeline.
407 // FIXME: Currently this is split into two loop pass pipelines because we run
408 // some function passes in between them. These can and should be removed
409 // and/or replaced by scheduling the loop pass equivalents in the correct
410 // positions. But those equivalent passes aren't powerful enough yet.
411 // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
412 // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
413 // fully replace `SimplifyCFGPass`, and the closest to the other we have is
414 // `LoopInstSimplify`.
415 LoopPassManager LPM1, LPM2;
416
417 // Simplify the loop body. We do this initially to clean up after other loop
418 // passes run, either when iterating on a loop or on inner loops with
419 // implications on the outer loop.
420 LPM1.addPass(Pass: LoopInstSimplifyPass());
421 LPM1.addPass(Pass: LoopSimplifyCFGPass());
422
423 // Try to remove as much code from the loop header as possible,
424 // to reduce amount of IR that will have to be duplicated. However,
425 // do not perform speculative hoisting the first time as LICM
426 // will destroy metadata that may not need to be destroyed if run
427 // after loop rotation.
428 // TODO: Investigate promotion cap for O1.
429 LPM1.addPass(Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
430 /*AllowSpeculation=*/false));
431
432 LPM1.addPass(
433 Pass: LoopRotatePass(/*EnableHeaderDuplication=*/true, isLTOPreLink(Phase)));
434 // TODO: Investigate promotion cap for O1.
435 LPM1.addPass(Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
436 /*AllowSpeculation=*/true));
437 LPM1.addPass(Pass: SimpleLoopUnswitchPass());
438 if (Opts.enable_loop_flatten)
439 LPM1.addPass(Pass: LoopFlattenPass());
440
441 LPM2.addPass(Pass: LoopIdiomRecognizePass());
442 LPM2.addPass(Pass: IndVarSimplifyPass());
443
444 invokeLateLoopOptimizationsEPCallbacks(LPM&: LPM2, Level);
445
446 LPM2.addPass(Pass: LoopDeletionPass());
447
448 // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
449 // because it changes IR to makes profile annotation in back compile
450 // inaccurate. The normal unroller doesn't pay attention to forced full unroll
451 // attributes so we need to make sure and allow the full unroll pass to pay
452 // attention to it.
453 if (!isThinLTOPreLink(Phase) || !PGOOpt ||
454 PGOOpt->Action != PGOOptions::SampleUse)
455 LPM2.addPass(Pass: LoopFullUnrollPass(static_cast<int>(Level),
456 /* OnlyWhenForced= */ !PTO.LoopUnrolling,
457 PTO.ForgetAllSCEVInLoopUnroll));
458
459 invokeLoopOptimizerEndEPCallbacks(LPM&: LPM2, Level);
460
461 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM1),
462 /*UseMemorySSA=*/true));
463 FPM.addPass(
464 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
465 FPM.addPass(Pass: InstCombinePass());
466 // The loop passes in LPM2 (LoopFullUnrollPass) do not preserve MemorySSA.
467 // *All* loop passes must preserve it, in order to be able to use it.
468 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM2),
469 /*UseMemorySSA=*/false));
470
471 // Delete small array after loop unroll.
472 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
473
474 // Specially optimize memory movement as it doesn't look like dataflow in SSA.
475 FPM.addPass(Pass: MemCpyOptPass());
476
477 // Sparse conditional constant propagation.
478 // FIXME: It isn't clear why we do this *after* loop passes rather than
479 // before...
480 FPM.addPass(Pass: SCCPPass());
481
482 // Delete dead bit computations (instcombine runs after to fold away the dead
483 // computations, and then ADCE will run later to exploit any new DCE
484 // opportunities that creates).
485 FPM.addPass(Pass: BDCEPass());
486
487 // Run instcombine after redundancy and dead bit elimination to exploit
488 // opportunities opened up by them.
489 FPM.addPass(Pass: InstCombinePass());
490 invokePeepholeEPCallbacks(FPM, Level);
491
492 FPM.addPass(Pass: CoroElidePass());
493
494 invokeScalarOptimizerLateEPCallbacks(FPM, Level);
495
496 // Finally, do an expensive DCE pass to catch all the dead code exposed by
497 // the simplifications and basic cleanup after all the simplifications.
498 // TODO: Investigate if this is too expensive.
499 FPM.addPass(Pass: ADCEPass());
500 FPM.addPass(
501 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
502 FPM.addPass(Pass: InstCombinePass());
503 invokePeepholeEPCallbacks(FPM, Level);
504
505 return FPM;
506}
507
508FunctionPassManager
509PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
510 ThinOrFullLTOPhase Phase) {
511 assert(Level != OptimizationLevel::O0 && "Must request optimizations!");
512
513 // The O1 pipeline has a separate pipeline creation function to simplify
514 // construction readability.
515 if (Level == OptimizationLevel::O1)
516 return buildO1FunctionSimplificationPipeline(Level, Phase);
517
518 FunctionPassManager FPM;
519
520 if (AreStatisticsEnabled())
521 FPM.addPass(Pass: CountVisitsPass());
522
523 // Form SSA out of local memory accesses after breaking apart aggregates into
524 // scalars.
525 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
526
527 // Catch trivial redundancies
528 FPM.addPass(Pass: EarlyCSEPass(true /* Enable mem-ssa. */));
529 if (EnableKnowledgeRetention)
530 FPM.addPass(Pass: AssumeSimplifyPass());
531
532 // Hoisting of scalars and load expressions.
533 if (Opts.enable_gvn_hoist)
534 FPM.addPass(Pass: GVNHoistPass());
535
536 // Global value numbering based sinking.
537 if (Opts.enable_gvn_sink) {
538 FPM.addPass(Pass: GVNSinkPass());
539 FPM.addPass(
540 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
541 }
542
543 // Speculative execution if the target has divergent branches; otherwise nop.
544 FPM.addPass(Pass: SpeculativeExecutionPass(/* OnlyIfDivergentTarget =*/true));
545
546 // Optimize based on known information about branches, and cleanup afterward.
547 FPM.addPass(Pass: JumpThreadingPass());
548 FPM.addPass(Pass: CorrelatedValuePropagationPass());
549
550 // Jump table to switch conversion.
551 if (Opts.enable_jump_table_to_switch)
552 FPM.addPass(Pass: JumpTableToSwitchPass(/*InLTO=*/isLTOPostLink(Phase)));
553
554 FPM.addPass(
555 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
556 FPM.addPass(Pass: InstCombinePass());
557 FPM.addPass(Pass: AggressiveInstCombinePass());
558 FPM.addPass(Pass: LibCallsShrinkWrapPass());
559
560 invokePeepholeEPCallbacks(FPM, Level);
561
562 // For PGO use pipeline, try to optimize memory intrinsics such as memcpy
563 // using the size value profile. Don't perform this when optimizing for size.
564 if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse)
565 FPM.addPass(Pass: PGOMemOPSizeOpt());
566
567 FPM.addPass(Pass: TailCallElimPass(/*UpdateFunctionEntryCount=*/
568 isInstrumentedPGOUse()));
569 FPM.addPass(
570 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
571
572 // Form canonically associated expression trees, and simplify the trees using
573 // basic mathematical properties. For example, this will form (nearly)
574 // minimal multiplication trees.
575 FPM.addPass(Pass: ReassociatePass());
576
577 if (Opts.enable_constraint_elimination)
578 FPM.addPass(Pass: ConstraintEliminationPass());
579
580 // Add the primary loop simplification pipeline.
581 // FIXME: Currently this is split into two loop pass pipelines because we run
582 // some function passes in between them. These can and should be removed
583 // and/or replaced by scheduling the loop pass equivalents in the correct
584 // positions. But those equivalent passes aren't powerful enough yet.
585 // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
586 // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
587 // fully replace `SimplifyCFGPass`, and the closest to the other we have is
588 // `LoopInstSimplify`.
589 LoopPassManager LPM1, LPM2;
590
591 // Simplify the loop body. We do this initially to clean up after other loop
592 // passes run, either when iterating on a loop or on inner loops with
593 // implications on the outer loop.
594 LPM1.addPass(Pass: LoopInstSimplifyPass());
595 LPM1.addPass(Pass: LoopSimplifyCFGPass());
596
597 // Try to remove as much code from the loop header as possible,
598 // to reduce amount of IR that will have to be duplicated. However,
599 // do not perform speculative hoisting the first time as LICM
600 // will destroy metadata that may not need to be destroyed if run
601 // after loop rotation.
602 // TODO: Investigate promotion cap for O1.
603 LPM1.addPass(Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
604 /*AllowSpeculation=*/false));
605
606 LPM1.addPass(
607 Pass: LoopRotatePass(/*EnableHeaderDuplication=*/true, isLTOPreLink(Phase)));
608 // TODO: Investigate promotion cap for O1.
609 LPM1.addPass(Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
610 /*AllowSpeculation=*/true));
611 LPM1.addPass(
612 Pass: SimpleLoopUnswitchPass(/* NonTrivial */ Level == OptimizationLevel::O3));
613 if (Opts.enable_loop_flatten)
614 LPM1.addPass(Pass: LoopFlattenPass());
615
616 LPM2.addPass(Pass: LoopIdiomRecognizePass());
617 LPM2.addPass(Pass: IndVarSimplifyPass());
618
619 {
620 ExtraLoopPassManager<ShouldRunExtraSimpleLoopUnswitch> ExtraPasses;
621 ExtraPasses.addPass(Pass: SimpleLoopUnswitchPass(/* NonTrivial */ Level ==
622 OptimizationLevel::O3));
623 LPM2.addPass(Pass: std::move(ExtraPasses));
624 }
625
626 invokeLateLoopOptimizationsEPCallbacks(LPM&: LPM2, Level);
627
628 LPM2.addPass(Pass: LoopDeletionPass());
629
630 // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
631 // because it changes IR to makes profile annotation in back compile
632 // inaccurate. The normal unroller doesn't pay attention to forced full unroll
633 // attributes so we need to make sure and allow the full unroll pass to pay
634 // attention to it.
635 if (!isThinLTOPreLink(Phase) || !PGOOpt ||
636 PGOOpt->Action != PGOOptions::SampleUse)
637 LPM2.addPass(Pass: LoopFullUnrollPass(static_cast<int>(Level),
638 /* OnlyWhenForced= */ !PTO.LoopUnrolling,
639 PTO.ForgetAllSCEVInLoopUnroll));
640
641 invokeLoopOptimizerEndEPCallbacks(LPM&: LPM2, Level);
642
643 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM1),
644 /*UseMemorySSA=*/true));
645 FPM.addPass(
646 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
647 FPM.addPass(Pass: InstCombinePass());
648 // The loop passes in LPM2 (LoopIdiomRecognizePass, IndVarSimplifyPass,
649 // LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA.
650 // *All* loop passes must preserve it, in order to be able to use it.
651 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM2),
652 /*UseMemorySSA=*/false));
653
654 // Delete small array after loop unroll.
655 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
656
657 // Try vectorization/scalarization transforms that are both improvements
658 // themselves and can allow further folds with GVN and InstCombine.
659 FPM.addPass(Pass: VectorCombinePass(/*TryEarlyFoldsOnly=*/true));
660
661 // Eliminate redundancies.
662 FPM.addPass(Pass: MergedLoadStoreMotionPass());
663 if (Opts.enable_newgvn)
664 FPM.addPass(Pass: NewGVNPass());
665 else
666 FPM.addPass(Pass: GVNPass());
667
668 // Sparse conditional constant propagation.
669 // FIXME: It isn't clear why we do this *after* loop passes rather than
670 // before...
671 FPM.addPass(Pass: SCCPPass());
672
673 // Delete dead bit computations (instcombine runs after to fold away the dead
674 // computations, and then ADCE will run later to exploit any new DCE
675 // opportunities that creates).
676 FPM.addPass(Pass: BDCEPass());
677
678 // Run instcombine after redundancy and dead bit elimination to exploit
679 // opportunities opened up by them.
680 FPM.addPass(Pass: InstCombinePass());
681 invokePeepholeEPCallbacks(FPM, Level);
682
683 // Re-consider control flow based optimizations after redundancy elimination,
684 // redo DCE, etc.
685 if (Opts.enable_dfa_jump_thread)
686 FPM.addPass(Pass: DFAJumpThreadingPass());
687
688 FPM.addPass(Pass: JumpThreadingPass());
689 FPM.addPass(Pass: CorrelatedValuePropagationPass());
690
691 // Finally, do an expensive DCE pass to catch all the dead code exposed by
692 // the simplifications and basic cleanup after all the simplifications.
693 // TODO: Investigate if this is too expensive.
694 FPM.addPass(Pass: ADCEPass());
695
696 // Specially optimize memory movement as it doesn't look like dataflow in SSA.
697 FPM.addPass(Pass: MemCpyOptPass());
698
699 FPM.addPass(Pass: DSEPass());
700 FPM.addPass(Pass: MoveAutoInitPass());
701
702 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(
703 Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
704 /*AllowSpeculation=*/true),
705 /*UseMemorySSA=*/true));
706
707 FPM.addPass(Pass: CoroElidePass());
708
709 invokeScalarOptimizerLateEPCallbacks(FPM, Level);
710
711 FPM.addPass(Pass: SimplifyCFGPass(SimplifyCFGOptions()
712 .convertSwitchRangeToICmp(B: true)
713 .convertSwitchToArithmetic(B: true)
714 .hoistCommonInsts(B: true)
715 .sinkCommonInsts(B: true)));
716 FPM.addPass(Pass: InstCombinePass());
717 invokePeepholeEPCallbacks(FPM, Level);
718
719 return FPM;
720}
721
722void PassBuilder::addRequiredLTOPreLinkPasses(ModulePassManager &MPM) {
723 MPM.addPass(Pass: CanonicalizeAliasesPass());
724 MPM.addPass(Pass: NameAnonGlobalPass());
725 MPM.addPass(Pass: AssignGUIDPass());
726}
727
728void PassBuilder::addPreInlinerPasses(ModulePassManager &MPM,
729 OptimizationLevel Level,
730 ThinOrFullLTOPhase LTOPhase) {
731 assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");
732 if (Opts.disable_preinline)
733 return;
734 InlineParams IP;
735
736 IP.DefaultThreshold = Opts.preinline_threshold;
737
738 // FIXME: The hint threshold has the same value used by the regular inliner
739 // when not optimzing for size. This should probably be lowered after
740 // performance testing.
741 // FIXME: this comment is cargo culted from the old pass manager, revisit).
742 IP.HintThreshold = 325;
743 IP.OptSizeHintThreshold = Opts.preinline_threshold;
744 ModuleInlinerWrapperPass MIWP(
745 IP, /* MandatoryFirst */ true,
746 InlineContext{.LTOPhase: LTOPhase, .Pass: InlinePass::EarlyInliner});
747 CGSCCPassManager &CGPipeline = MIWP.getPM();
748
749 FunctionPassManager FPM;
750 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
751 FPM.addPass(Pass: EarlyCSEPass()); // Catch trivial redundancies.
752 FPM.addPass(Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(
753 B: true))); // Merge & remove basic blocks.
754 FPM.addPass(Pass: InstCombinePass()); // Combine silly sequences.
755 invokePeepholeEPCallbacks(FPM, Level);
756
757 CGPipeline.addPass(Pass: createCGSCCToFunctionPassAdaptor(
758 Pass: std::move(FPM), EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
759
760 MPM.addPass(Pass: std::move(MIWP));
761
762 // Delete anything that is now dead to make sure that we don't instrument
763 // dead code. Instrumentation can end up keeping dead code around and
764 // dramatically increase code size.
765 MPM.addPass(Pass: GlobalDCEPass());
766}
767
768void PassBuilder::addPostPGOLoopRotation(ModulePassManager &MPM,
769 OptimizationLevel Level) {
770 if (Opts.enable_post_pgo_loop_rotation) {
771 // Disable header duplication in loop rotation at -Oz.
772 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
773 Pass: createFunctionToLoopPassAdaptor(Pass: LoopRotatePass(),
774 /*UseMemorySSA=*/false),
775 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
776 }
777}
778
779void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM,
780 OptimizationLevel Level, bool RunProfileGen,
781 bool IsCS, bool AtomicCounterUpdate,
782 std::string ProfileFile,
783 std::string ProfileRemappingFile) {
784 assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");
785
786 if (!RunProfileGen) {
787 assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
788 MPM.addPass(
789 Pass: PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS, FS));
790 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
791 // RequireAnalysisPass for PSI before subsequent non-module passes.
792 MPM.addPass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
793 return;
794 }
795
796 // Perform PGO instrumentation.
797 MPM.addPass(Pass: PGOInstrumentationGen(IsCS ? PGOInstrumentationType::CSFDO
798 : PGOInstrumentationType::FDO));
799
800 addPostPGOLoopRotation(MPM, Level);
801 // Add the profile lowering pass.
802 InstrProfOptions Options;
803 if (!ProfileFile.empty())
804 Options.InstrProfileOutput = ProfileFile;
805 // Do counter promotion at Level greater than O0.
806 Options.DoCounterPromotion = true;
807 Options.UseBFIInPromotion = IsCS;
808 if (Opts.enable_sampled_instrumentation) {
809 Options.Sampling = true;
810 // With sampling, there is little beneifit to enable counter promotion.
811 // But note that sampling does work with counter promotion.
812 Options.DoCounterPromotion = false;
813 }
814 Options.Atomic = AtomicCounterUpdate;
815 MPM.addPass(Pass: InstrProfilingLoweringPass(Options, IsCS));
816}
817
818void PassBuilder::addPGOInstrPassesForO0(ModulePassManager &MPM,
819 bool RunProfileGen, bool IsCS,
820 bool AtomicCounterUpdate,
821 std::string ProfileFile,
822 std::string ProfileRemappingFile) {
823 if (!RunProfileGen) {
824 assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
825 MPM.addPass(
826 Pass: PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS, FS));
827 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
828 // RequireAnalysisPass for PSI before subsequent non-module passes.
829 MPM.addPass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
830 return;
831 }
832
833 // Perform PGO instrumentation.
834 MPM.addPass(Pass: PGOInstrumentationGen(IsCS ? PGOInstrumentationType::CSFDO
835 : PGOInstrumentationType::FDO));
836 // Add the profile lowering pass.
837 InstrProfOptions Options;
838 if (!ProfileFile.empty())
839 Options.InstrProfileOutput = ProfileFile;
840 // Do not do counter promotion at O0.
841 Options.DoCounterPromotion = false;
842 Options.UseBFIInPromotion = IsCS;
843 Options.Atomic = AtomicCounterUpdate;
844 MPM.addPass(Pass: InstrProfilingLoweringPass(Options, IsCS));
845}
846
847ModuleInlinerWrapperPass
848PassBuilder::buildInlinerPipeline(OptimizationLevel Level,
849 ThinOrFullLTOPhase Phase) {
850 InlineParams IP;
851 if (PTO.InlinerThreshold == -1)
852 IP = ::getInlineParamsFromOptLevel(Level);
853 else
854 IP = getInlineParams(Threshold: PTO.InlinerThreshold);
855 // For PreLinkThinLTO + SamplePGO or PreLinkFullLTO + SamplePGO,
856 // set hot-caller threshold to 0 to disable hot
857 // callsite inline (as much as possible [1]) because it makes
858 // profile annotation in the backend inaccurate.
859 //
860 // [1] Note the cost of a function could be below zero due to erased
861 // prologue / epilogue.
862 if (isLTOPreLink(Phase) && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse)
863 IP.HotCallSiteThreshold = 0;
864
865 if (PGOOpt)
866 IP.EnableDeferral = Opts.enable_npm_pgo_inline_deferral;
867
868 ModuleInlinerWrapperPass MIWP(IP, Opts.mandatory_inlining_first,
869 InlineContext{.LTOPhase: Phase, .Pass: InlinePass::CGSCCInliner},
870 Opts.enable_ml_inliner, MaxDevirtIterations);
871
872 // Require the GlobalsAA analysis for the module so we can query it within
873 // the CGSCC pipeline.
874 if (Opts.enable_global_analyses) {
875 MIWP.addModulePass(Pass: RequireAnalysisPass<GlobalsAA, Module>());
876 // Invalidate AAManager so it can be recreated and pick up the newly
877 // available GlobalsAA.
878 MIWP.addModulePass(
879 Pass: createModuleToFunctionPassAdaptor(Pass: InvalidateAnalysisPass<AAManager>()));
880 }
881
882 // Require the ProfileSummaryAnalysis for the module so we can query it within
883 // the inliner pass.
884 MIWP.addModulePass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
885
886 // Now begin the main postorder CGSCC pipeline.
887 // FIXME: The current CGSCC pipeline has its origins in the legacy pass
888 // manager and trying to emulate its precise behavior. Much of this doesn't
889 // make a lot of sense and we should revisit the core CGSCC structure.
890 CGSCCPassManager &MainCGPipeline = MIWP.getPM();
891
892 // Note: historically, the PruneEH pass was run first to deduce nounwind and
893 // generally clean up exception handling overhead. It isn't clear this is
894 // valuable as the inliner doesn't currently care whether it is inlining an
895 // invoke or a call.
896
897 if (Opts.attributor_enable & AttributorRunOption::CGSCC)
898 MainCGPipeline.addPass(Pass: AttributorCGSCCPass());
899
900 // Deduce function attributes. We do another run of this after the function
901 // simplification pipeline, so this only needs to run when it could affect the
902 // function simplification pipeline, which is only the case with recursive
903 // functions.
904 if (Opts.attributor_enable & AttributorRunOption::CGSCC_LIGHT) {
905 MainCGPipeline.addPass(Pass: AttributorLightCGSCCPass());
906 } else {
907 MainCGPipeline.addPass(
908 Pass: PostOrderFunctionAttrsPass(/*SkipNonRecursive*/ true));
909 }
910
911 // When at O3 add argument promotion to the pass pipeline.
912 // FIXME: It isn't at all clear why this should be limited to O3.
913 if (Level == OptimizationLevel::O3)
914 MainCGPipeline.addPass(Pass: ArgumentPromotionPass());
915
916 // Try to perform OpenMP specific optimizations. This is a (quick!) no-op if
917 // there are no OpenMP runtime calls present in the module.
918 if (Level == OptimizationLevel::O2 || Level == OptimizationLevel::O3)
919 MainCGPipeline.addPass(Pass: OpenMPOptCGSCCPass(Phase));
920
921 invokeCGSCCOptimizerLateEPCallbacks(CGPM&: MainCGPipeline, Level);
922
923 // Add the core function simplification pipeline nested inside the
924 // CGSCC walk.
925 MainCGPipeline.addPass(Pass: createCGSCCToFunctionPassAdaptor(
926 Pass: buildFunctionSimplificationPipeline(Level, Phase),
927 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses, /*NoRerun=*/true));
928
929 // Finally, deduce any function attributes based on the fully simplified
930 // function.
931 if (Opts.attributor_enable & AttributorRunOption::CGSCC_LIGHT)
932 MainCGPipeline.addPass(Pass: AttributorLightCGSCCPass());
933 else
934 MainCGPipeline.addPass(Pass: PostOrderFunctionAttrsPass());
935
936 // Mark that the function is fully simplified and that it shouldn't be
937 // simplified again if we somehow revisit it due to CGSCC mutations unless
938 // it's been modified since.
939 MainCGPipeline.addPass(Pass: createCGSCCToFunctionPassAdaptor(
940 Pass: RequireAnalysisPass<ShouldNotRunFunctionPassesAnalysis, Function>()));
941
942 if (!isThinLTOPreLink(Phase)) {
943 MainCGPipeline.addPass(Pass: CoroSplitPass(Level != OptimizationLevel::O0));
944 MainCGPipeline.addPass(Pass: CoroAnnotationElidePass());
945 }
946
947 // Make sure we don't affect potential future NoRerun CGSCC adaptors.
948 MIWP.addLateModulePass(Pass: createModuleToFunctionPassAdaptor(
949 Pass: InvalidateAnalysisPass<ShouldNotRunFunctionPassesAnalysis>()));
950
951 return MIWP;
952}
953
954ModulePassManager
955PassBuilder::buildModuleInlinerPipeline(OptimizationLevel Level,
956 ThinOrFullLTOPhase Phase) {
957 ModulePassManager MPM;
958
959 InlineParams IP = ::getInlineParamsFromOptLevel(Level);
960 // For PreLinkThinLTO + SamplePGO or PreLinkFullLTO + SamplePGO,
961 // set hot-caller threshold to 0 to disable hot
962 // callsite inline (as much as possible [1]) because it makes
963 // profile annotation in the backend inaccurate.
964 //
965 // [1] Note the cost of a function could be below zero due to erased
966 // prologue / epilogue.
967 if (isLTOPreLink(Phase) && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse)
968 IP.HotCallSiteThreshold = 0;
969
970 if (PGOOpt)
971 IP.EnableDeferral = Opts.enable_npm_pgo_inline_deferral;
972
973 // The inline deferral logic is used to avoid losing some
974 // inlining chance in future. It is helpful in SCC inliner, in which
975 // inlining is processed in bottom-up order.
976 // While in module inliner, the inlining order is a priority-based order
977 // by default. The inline deferral is unnecessary there. So we disable the
978 // inline deferral logic in module inliner.
979 IP.EnableDeferral = false;
980
981 MPM.addPass(Pass: ModuleInlinerPass(IP, Opts.enable_ml_inliner, Phase));
982 if (!UseCtxProfile.empty() && Phase == ThinOrFullLTOPhase::ThinLTOPostLink) {
983 MPM.addPass(Pass: GlobalOptPass());
984 MPM.addPass(Pass: GlobalDCEPass());
985 MPM.addPass(Pass: AssignGUIDPass());
986 MPM.addPass(Pass: PGOCtxProfFlatteningPass(/*IsPreThinlink=*/false));
987 }
988
989 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
990 Pass: buildFunctionSimplificationPipeline(Level, Phase),
991 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
992
993 if (!isThinLTOPreLink(Phase)) {
994 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(
995 Pass: CoroSplitPass(Level != OptimizationLevel::O0)));
996 MPM.addPass(
997 Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: CoroAnnotationElidePass()));
998 }
999
1000 return MPM;
1001}
1002
1003ModulePassManager
1004PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
1005 ThinOrFullLTOPhase Phase) {
1006 assert(Level != OptimizationLevel::O0 &&
1007 "Should not be used for O0 pipeline");
1008
1009 assert(!isFullLTOPostLink(Phase) &&
1010 "FullLTOPostLink shouldn't call buildModuleSimplificationPipeline!");
1011
1012 ModulePassManager MPM;
1013
1014 // Place pseudo probe instrumentation as the first pass of the pipeline to
1015 // minimize the impact of optimization changes.
1016 if (PGOOpt && PGOOpt->PseudoProbeForProfiling && !isThinLTOPostLink(Phase))
1017 MPM.addPass(Pass: SampleProfileProbePass(TM));
1018
1019 bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse);
1020
1021 // In ThinLTO mode, when flattened profile is used, all the available
1022 // profile information will be annotated in PreLink phase so there is
1023 // no need to load the profile again in PostLink.
1024 bool LoadSampleProfile = HasSampleProfile && !(Opts.flattened_profile_used &&
1025 isThinLTOPostLink(Phase));
1026
1027 // During the ThinLTO backend phase we perform early indirect call promotion
1028 // here, before globalopt. Otherwise imported available_externally functions
1029 // look unreferenced and are removed. If we are going to load the sample
1030 // profile then defer until later.
1031 // TODO: See if we can move later and consolidate with the location where
1032 // we perform ICP when we are loading a sample profile.
1033 // TODO: We pass HasSampleProfile (whether there was a sample profile file
1034 // passed to the compile) to the SamplePGO flag of ICP. This is used to
1035 // determine whether the new direct calls are annotated with prof metadata.
1036 // Ideally this should be determined from whether the IR is annotated with
1037 // sample profile, and not whether the a sample profile was provided on the
1038 // command line. E.g. for flattened profiles where we will not be reloading
1039 // the sample profile in the ThinLTO backend, we ideally shouldn't have to
1040 // provide the sample profile file.
1041 if (isThinLTOPostLink(Phase) && !LoadSampleProfile)
1042 MPM.addPass(Pass: PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile));
1043
1044 // Create an early function pass manager to cleanup the output of the
1045 // frontend. Not necessary with LTO post link pipelines since the pre link
1046 // pipeline already cleaned up the frontend output.
1047 if (!isThinLTOPostLink(Phase)) {
1048 // Do basic inference of function attributes from known properties of system
1049 // libraries and other oracles.
1050 MPM.addPass(Pass: InferFunctionAttrsPass());
1051 MPM.addPass(Pass: CoroEarlyPass());
1052
1053 FunctionPassManager EarlyFPM;
1054 EarlyFPM.addPass(Pass: EntryExitInstrumenterPass(/*PostInlining=*/false));
1055 // Lower llvm.expect to metadata before attempting transforms.
1056 // Compare/branch metadata may alter the behavior of passes like
1057 // SimplifyCFG.
1058 EarlyFPM.addPass(Pass: LowerExpectIntrinsicPass());
1059 EarlyFPM.addPass(Pass: SimplifyCFGPass());
1060 EarlyFPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
1061 EarlyFPM.addPass(Pass: EarlyCSEPass());
1062 if (Level == OptimizationLevel::O3)
1063 EarlyFPM.addPass(Pass: CallSiteSplittingPass());
1064 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
1065 Pass: std::move(EarlyFPM), EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
1066 }
1067
1068 if (LoadSampleProfile) {
1069 // Annotate sample profile right after early FPM to ensure freshness of
1070 // the debug info.
1071 MPM.addPass(Pass: SampleProfileLoaderPass(
1072 PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile, Phase, FS));
1073 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1074 // RequireAnalysisPass for PSI before subsequent non-module passes.
1075 MPM.addPass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
1076 // Do not invoke ICP in the LTOPrelink phase as it makes it hard
1077 // for the profile annotation to be accurate in the LTO backend.
1078 if (!isLTOPreLink(Phase))
1079 // We perform early indirect call promotion here, before globalopt.
1080 // This is important for the ThinLTO backend phase because otherwise
1081 // imported available_externally functions look unreferenced and are
1082 // removed.
1083 MPM.addPass(
1084 Pass: PGOIndirectCallPromotion(true /* IsInLTO */, true /* SamplePGO */));
1085 }
1086
1087 // Try to perform OpenMP specific optimizations on the module. This is a
1088 // (quick!) no-op if there are no OpenMP runtime calls present in the module.
1089 MPM.addPass(Pass: OpenMPOptPass(Phase));
1090
1091 if (Opts.attributor_enable & AttributorRunOption::MODULE)
1092 MPM.addPass(Pass: AttributorPass());
1093 else if (Opts.attributor_enable & AttributorRunOption::MODULE_LIGHT)
1094 MPM.addPass(Pass: AttributorLightPass());
1095
1096 // Lower type metadata and the type.test intrinsic in the ThinLTO
1097 // post link pipeline after ICP. This is to enable usage of the type
1098 // tests in ICP sequences.
1099 if (isThinLTOPostLink(Phase))
1100 MPM.addPass(Pass: DropTypeTestsPass());
1101
1102 invokePipelineEarlySimplificationEPCallbacks(MPM, Level, Phase);
1103
1104 // Interprocedural constant propagation now that basic cleanup has occurred
1105 // and prior to optimizing globals.
1106 // FIXME: This position in the pipeline hasn't been carefully considered in
1107 // years, it should be re-analyzed.
1108 MPM.addPass(
1109 Pass: IPSCCPPass(IPSCCPOptions(/*AllowFuncSpec=*/!isLTOPreLink(Phase))));
1110
1111 // Attach metadata to indirect call sites indicating the set of functions
1112 // they may target at run-time. This should follow IPSCCP.
1113 MPM.addPass(Pass: CalledValuePropagationPass());
1114
1115 // Optimize globals to try and fold them into constants.
1116 MPM.addPass(Pass: GlobalOptPass());
1117
1118 // Create a small function pass pipeline to cleanup after all the global
1119 // optimizations.
1120 FunctionPassManager GlobalCleanupPM;
1121 // FIXME: Should this instead by a run of SROA?
1122 GlobalCleanupPM.addPass(Pass: PromotePass());
1123 GlobalCleanupPM.addPass(Pass: InstCombinePass());
1124 invokePeepholeEPCallbacks(FPM&: GlobalCleanupPM, Level);
1125 GlobalCleanupPM.addPass(
1126 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
1127 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(GlobalCleanupPM),
1128 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
1129
1130 // We already asserted this happens in non-FullLTOPostLink earlier.
1131 const bool IsPreLink = !isThinLTOPostLink(Phase);
1132 // Enable contextual profiling instrumentation.
1133 const bool IsCtxProfGen =
1134 IsPreLink &&
1135 PGOCtxProfLoweringPass::isCtxIRPGOInstrEnabled(LibraryOptions);
1136 const bool IsPGOPreLink = !IsCtxProfGen && PGOOpt && IsPreLink;
1137 const bool IsPGOInstrGen =
1138 IsPGOPreLink && PGOOpt->Action == PGOOptions::IRInstr;
1139 const bool IsPGOInstrUse =
1140 IsPGOPreLink && PGOOpt->Action == PGOOptions::IRUse;
1141 const bool IsMemprofUse = IsPGOPreLink && !PGOOpt->MemoryProfile.empty();
1142 // We don't want to mix pgo ctx gen and pgo gen; we also don't currently
1143 // enable ctx profiling from the frontend.
1144 assert(!(IsPGOInstrGen &&
1145 PGOCtxProfLoweringPass::isCtxIRPGOInstrEnabled(LibraryOptions)) &&
1146 "Enabling both instrumented PGO and contextual instrumentation is not "
1147 "supported.");
1148 const bool IsCtxProfUse = !UseCtxProfile.empty() && isThinLTOPreLink(Phase);
1149
1150 assert((Opts.instrument_cold_function_only_path.empty() ||
1151 isPGOInstrumentColdFunctionOnly(LibraryOptions)) &&
1152 "--instrument-cold-function-only-path is provided but "
1153 "--pgo-instrument-cold-function-only is not enabled");
1154 const bool IsColdFuncOnlyInstrGen =
1155 isPGOInstrumentColdFunctionOnly(LibraryOptions) && IsPGOPreLink &&
1156 !Opts.instrument_cold_function_only_path.empty();
1157
1158 if (IsPGOInstrGen || IsPGOInstrUse || IsMemprofUse || IsCtxProfGen ||
1159 IsCtxProfUse || IsColdFuncOnlyInstrGen)
1160 addPreInlinerPasses(MPM, Level, LTOPhase: Phase);
1161
1162 // Add all the requested passes for instrumentation PGO, if requested.
1163 if (IsPGOInstrGen || IsPGOInstrUse) {
1164 addPGOInstrPasses(MPM, Level,
1165 /*RunProfileGen=*/IsPGOInstrGen,
1166 /*IsCS=*/false, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate,
1167 ProfileFile: PGOOpt->ProfileFile, ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
1168 } else if (IsCtxProfGen || IsCtxProfUse) {
1169 MPM.addPass(Pass: PGOInstrumentationGen(PGOInstrumentationType::CTXPROF));
1170 // In pre-link, we just want the instrumented IR. We use the contextual
1171 // profile in the post-thinlink phase.
1172 // The instrumentation will be removed in post-thinlink after IPO.
1173 if (IsCtxProfUse) {
1174 MPM.addPass(Pass: AssignGUIDPass());
1175 MPM.addPass(Pass: PGOCtxProfFlatteningPass(/*IsPreThinlink=*/true));
1176 return MPM;
1177 }
1178 // Block further inlining in the instrumented ctxprof case. This avoids
1179 // confusingly collecting profiles for the same GUID corresponding to
1180 // different variants of the function. We could do like PGO and identify
1181 // functions by a (GUID, Hash) tuple, but since the ctxprof "use" waits for
1182 // thinlto to happen before performing any further optimizations, it's
1183 // unnecessary to collect profiles for non-prevailing copies.
1184 MPM.addPass(Pass: NoinlineNonPrevailing());
1185 addPostPGOLoopRotation(MPM, Level);
1186 MPM.addPass(Pass: AssignGUIDPass());
1187 MPM.addPass(Pass: PGOCtxProfLoweringPass());
1188 } else if (IsColdFuncOnlyInstrGen) {
1189 addPGOInstrPasses(MPM, Level, /* RunProfileGen */ true, /* IsCS */ false,
1190 /* AtomicCounterUpdate */ false,
1191 ProfileFile: Opts.instrument_cold_function_only_path.str(),
1192 /* ProfileRemappingFile */ "");
1193 }
1194
1195 if (IsPGOInstrGen || IsPGOInstrUse || IsCtxProfGen)
1196 MPM.addPass(Pass: PGOIndirectCallPromotion(false, false));
1197
1198 if (IsPGOPreLink && PGOOpt->CSAction == PGOOptions::CSIRInstr)
1199 MPM.addPass(Pass: PGOInstrumentationGenCreateVar(
1200 PGOOpt->CSProfileGenFile, Opts.enable_sampled_instrumentation));
1201
1202 if (IsMemprofUse)
1203 MPM.addPass(Pass: MemProfUsePass(PGOOpt->MemoryProfile, FS));
1204
1205 if (PGOOpt && (PGOOpt->Action == PGOOptions::IRUse ||
1206 PGOOpt->Action == PGOOptions::SampleUse))
1207 MPM.addPass(Pass: PGOForceFunctionAttrsPass(PGOOpt->ColdOptType));
1208
1209 MPM.addPass(Pass: AlwaysInlinerPass(/*InsertLifetimeIntrinsics=*/true));
1210
1211 if (Opts.enable_module_inliner)
1212 MPM.addPass(Pass: buildModuleInlinerPipeline(Level, Phase));
1213 else
1214 MPM.addPass(Pass: buildInlinerPipeline(Level, Phase));
1215
1216 // Remove any dead arguments exposed by cleanups, constant folding globals,
1217 // and argument promotion.
1218 MPM.addPass(Pass: DeadArgumentEliminationPass());
1219
1220 if (isThinLTOPostLink(Phase))
1221 MPM.addPass(Pass: SimplifyTypeTestsPass());
1222
1223 if (!isThinLTOPreLink(Phase))
1224 MPM.addPass(Pass: CoroCleanupPass());
1225
1226 // Optimize globals now that functions are fully simplified.
1227 MPM.addPass(Pass: GlobalOptPass());
1228 MPM.addPass(Pass: GlobalDCEPass());
1229
1230 return MPM;
1231}
1232
1233/// TODO: Should LTO cause any differences to this set of passes?
1234void PassBuilder::addVectorPasses(OptimizationLevel Level,
1235 FunctionPassManager &FPM,
1236 ThinOrFullLTOPhase LTOPhase) {
1237 FPM.addPass(Pass: LoopVectorizePass(
1238 LoopVectorizeOptions(!PTO.LoopInterleaving, !PTO.LoopVectorization)));
1239
1240 // Drop dereferenceable assumes after vectorization, as they are no longer
1241 // needed and can inhibit further optimization.
1242 if (!isLTOPreLink(Phase: LTOPhase))
1243 FPM.addPass(Pass: DropUnnecessaryAssumesPass(/*DropDereferenceable=*/true));
1244
1245 FPM.addPass(Pass: InferAlignmentPass());
1246 if (isFullLTOPostLink(Phase: LTOPhase)) {
1247 // The vectorizer may have significantly shortened a loop body; unroll
1248 // again. Unroll small loops to hide loop backedge latency and saturate any
1249 // parallel execution resources of an out-of-order processor. We also then
1250 // need to clean up redundancies and loop invariant code.
1251 // FIXME: It would be really good to use a loop-integrated instruction
1252 // combiner for cleanup here so that the unrolling and LICM can be pipelined
1253 // across the loop nests.
1254 // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1255 if (Opts.enable_unroll_and_jam && PTO.LoopUnrolling)
1256 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(
1257 Pass: LoopUnrollAndJamPass(static_cast<int>(Level))));
1258 FPM.addPass(Pass: LoopUnrollPass(LoopUnrollOptions(
1259 static_cast<int>(Level), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1260 PTO.ForgetAllSCEVInLoopUnroll)));
1261 FPM.addPass(Pass: WarnMissedTransformationsPass());
1262 // Now that we are done with loop unrolling, be it either by LoopVectorizer,
1263 // or LoopUnroll passes, some variable-offset GEP's into alloca's could have
1264 // become constant-offset, thus enabling SROA and alloca promotion. Do so.
1265 // NOTE: we are very late in the pipeline, and we don't have any LICM
1266 // or SimplifyCFG passes scheduled after us, that would cleanup
1267 // the CFG mess this may created if allowed to modify CFG, so forbid that.
1268
1269 // We also turn on struct to vector canonicalization here, which allows
1270 // converting allocas of homogeneous structs into vector allocas when the
1271 // allocas' users are all memory intrinsics. This allows promotion in some
1272 // cases because structs cannot promote to SSA values, but vectors can. We
1273 // only turn this on after memcpyopt runs because this might hinder
1274 // memcpyopt's optimizations if done before. Look at the documentation for
1275 // `tryCanonicalizeStructToVector` in SROA.cpp to see why.
1276 FPM.addPass(Pass: SROAPass(SROAOptions(SROAOptions::PreserveCFG,
1277 /*AggregateToVector=*/true)));
1278 }
1279
1280 if (!isFullLTOPostLink(Phase: LTOPhase)) {
1281 // Eliminate loads by forwarding stores from the previous iteration to loads
1282 // of the current iteration.
1283 FPM.addPass(Pass: LoopLoadEliminationPass());
1284 }
1285 // Cleanup after the loop optimization passes.
1286 FPM.addPass(Pass: InstCombinePass());
1287
1288 if (Level > OptimizationLevel::O1 && Opts.extra_vectorizer_passes) {
1289 ExtraFunctionPassManager<ShouldRunExtraVectorPasses> ExtraPasses;
1290 // At higher optimization levels, try to clean up any runtime overlap and
1291 // alignment checks inserted by the vectorizer. We want to track correlated
1292 // runtime checks for two inner loops in the same outer loop, fold any
1293 // common computations, hoist loop-invariant aspects out of any outer loop,
1294 // and unswitch the runtime checks if possible. Once hoisted, we may have
1295 // dead (or speculatable) control flows or more combining opportunities.
1296 ExtraPasses.addPass(Pass: EarlyCSEPass());
1297 ExtraPasses.addPass(Pass: CorrelatedValuePropagationPass());
1298 ExtraPasses.addPass(Pass: InstCombinePass());
1299 LoopPassManager LPM;
1300 LPM.addPass(Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
1301 /*AllowSpeculation=*/true));
1302 LPM.addPass(Pass: SimpleLoopUnswitchPass(/* NonTrivial */ Level ==
1303 OptimizationLevel::O3));
1304 ExtraPasses.addPass(
1305 Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM), /*UseMemorySSA=*/true));
1306 ExtraPasses.addPass(
1307 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
1308 ExtraPasses.addPass(Pass: InstCombinePass());
1309 FPM.addPass(Pass: std::move(ExtraPasses));
1310 }
1311
1312 // Now that we've formed fast to execute loop structures, we do further
1313 // optimizations. These are run afterward as they might block doing complex
1314 // analyses and transforms such as what are needed for loop vectorization.
1315
1316 // Cleanup after loop vectorization, etc. Simplification passes like CVP and
1317 // GVN, loop transforms, and others have already run, so it's now better to
1318 // convert to more optimized IR using more aggressive simplify CFG options.
1319 // The extra sinking transform can create larger basic blocks, so do this
1320 // before SLP vectorization.
1321 FPM.addPass(Pass: SimplifyCFGPass(SimplifyCFGOptions()
1322 .forwardSwitchCondToPhi(B: true)
1323 .convertSwitchRangeToICmp(B: true)
1324 .convertSwitchToArithmetic(B: true)
1325 .convertSwitchToLookupTable(B: true)
1326 .needCanonicalLoops(B: false)
1327 .hoistCommonInsts(B: true)
1328 .sinkCommonInsts(B: true)));
1329
1330 if (isFullLTOPostLink(Phase: LTOPhase)) {
1331 FPM.addPass(Pass: SCCPPass());
1332 FPM.addPass(Pass: InstCombinePass());
1333 FPM.addPass(Pass: BDCEPass());
1334 }
1335
1336 // Optimize parallel scalar instruction chains into SIMD instructions.
1337 if (PTO.SLPVectorization) {
1338 FPM.addPass(Pass: SLPVectorizerPass());
1339 if (Level >= OptimizationLevel::O2 && Opts.extra_vectorizer_passes) {
1340 FPM.addPass(Pass: EarlyCSEPass());
1341 }
1342 }
1343 // Enhance/cleanup vector code.
1344 FPM.addPass(Pass: VectorCombinePass());
1345
1346 if (!isFullLTOPostLink(Phase: LTOPhase)) {
1347 FPM.addPass(Pass: InstCombinePass());
1348 // Unroll small loops to hide loop backedge latency and saturate any
1349 // parallel execution resources of an out-of-order processor. We also then
1350 // need to clean up redundancies and loop invariant code.
1351 // FIXME: It would be really good to use a loop-integrated instruction
1352 // combiner for cleanup here so that the unrolling and LICM can be pipelined
1353 // across the loop nests.
1354 // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1355 if (Opts.enable_unroll_and_jam && PTO.LoopUnrolling) {
1356 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(
1357 Pass: LoopUnrollAndJamPass(static_cast<int>(Level))));
1358 }
1359 FPM.addPass(Pass: LoopUnrollPass(LoopUnrollOptions(
1360 static_cast<int>(Level), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1361 PTO.ForgetAllSCEVInLoopUnroll)));
1362 FPM.addPass(Pass: WarnMissedTransformationsPass());
1363 // Now that we are done with loop unrolling, be it either by LoopVectorizer,
1364 // or LoopUnroll passes, some variable-offset GEP's into alloca's could have
1365 // become constant-offset, thus enabling SROA and alloca promotion. Do so.
1366 // NOTE: we are very late in the pipeline, and we don't have any LICM
1367 // or SimplifyCFG passes scheduled after us, that would cleanup
1368 // the CFG mess this may created if allowed to modify CFG, so forbid that.
1369
1370 // We also turn on struct to vector canonicalization here, which allows
1371 // converting allocas of homogeneous structs into vector allocas when the
1372 // allocas' users are all memory intrinsics. This allows promotion in some
1373 // cases because structs cannot promote to SSA values, but vectors can. We
1374 // only turn this on after memcpyopt runs because this might hinder
1375 // memcpyopt's optimizations if done before. Look at the documentation for
1376 // `tryCanonicalizeStructToVector` in SROA.cpp to see why.
1377 FPM.addPass(Pass: SROAPass(SROAOptions(SROAOptions::PreserveCFG,
1378 /*AggregateToVector=*/true)));
1379 }
1380
1381 FPM.addPass(Pass: InferAlignmentPass());
1382 FPM.addPass(Pass: InstCombinePass());
1383
1384 // This is needed for two reasons:
1385 // 1. It works around problems that instcombine introduces, such as sinking
1386 // expensive FP divides into loops containing multiplications using the
1387 // divide result.
1388 // 2. It helps to clean up some loop-invariant code created by the loop
1389 // unroll pass when IsFullLTO=false.
1390 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(
1391 Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
1392 /*AllowSpeculation=*/true),
1393 /*UseMemorySSA=*/true));
1394
1395 // Now that we've vectorized and unrolled loops, we may have more refined
1396 // alignment information, try to re-derive it here.
1397 FPM.addPass(Pass: AlignmentFromAssumptionsPass());
1398}
1399
1400ModulePassManager
1401PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
1402 ThinOrFullLTOPhase LTOPhase) {
1403 ModulePassManager MPM;
1404
1405 // Run partial inlining pass to partially inline functions that have
1406 // large bodies.
1407 if (Opts.enable_partial_inlining)
1408 MPM.addPass(Pass: PartialInlinerPass());
1409
1410 // Remove avail extern fns and globals definitions since we aren't compiling
1411 // an object file for later LTO. For LTO we want to preserve these so they
1412 // are eligible for inlining at link-time. Note if they are unreferenced they
1413 // will be removed by GlobalDCE later, so this only impacts referenced
1414 // available externally globals. Eventually they will be suppressed during
1415 // codegen, but eliminating here enables more opportunity for GlobalDCE as it
1416 // may make globals referenced by available external functions dead and saves
1417 // running remaining passes on the eliminated functions. These should be
1418 // preserved during prelinking for link-time inlining decisions.
1419 if (!isLTOPreLink(Phase: LTOPhase))
1420 MPM.addPass(Pass: EliminateAvailableExternallyPass());
1421
1422 // Do RPO function attribute inference across the module to forward-propagate
1423 // attributes where applicable.
1424 // FIXME: Is this really an optimization rather than a canonicalization?
1425 if (Opts.attributor_enable & AttributorRunOption::MODULE_LIGHT)
1426 MPM.addPass(Pass: AttributorLightPass());
1427 else
1428 MPM.addPass(Pass: ReversePostOrderFunctionAttrsPass());
1429
1430 // Do a post inline PGO instrumentation and use pass. This is a context
1431 // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as
1432 // cross-module inline has not been done yet. The context sensitive
1433 // instrumentation is after all the inlines are done.
1434 if (!isLTOPreLink(Phase: LTOPhase) && PGOOpt) {
1435 if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1436 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/true,
1437 /*IsCS=*/true, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate,
1438 ProfileFile: PGOOpt->CSProfileGenFile, ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
1439 else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1440 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/false,
1441 /*IsCS=*/true, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate,
1442 ProfileFile: PGOOpt->ProfileFile, ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
1443 }
1444
1445 // Re-compute GlobalsAA here prior to function passes. This is particularly
1446 // useful as the above will have inlined, DCE'ed, and function-attr
1447 // propagated everything. We should at this point have a reasonably minimal
1448 // and richly annotated call graph. By computing aliasing and mod/ref
1449 // information for all local globals here, the late loop passes and notably
1450 // the vectorizer will be able to use them to help recognize vectorizable
1451 // memory operations.
1452 if (Opts.enable_global_analyses)
1453 MPM.addPass(Pass: RecomputeGlobalsAAPass());
1454
1455 invokeOptimizerEarlyEPCallbacks(MPM, Level, Phase: LTOPhase);
1456
1457 FunctionPassManager OptimizePM;
1458
1459 // Only drop unnecessary assumes post-inline and post-link, as otherwise
1460 // additional uses of the affected value may be introduced through inlining
1461 // and CSE.
1462 if (!isLTOPreLink(Phase: LTOPhase))
1463 OptimizePM.addPass(Pass: DropUnnecessaryAssumesPass());
1464
1465 // Scheduling LoopVersioningLICM when inlining is over, because after that
1466 // we may see more accurate aliasing. Reason to run this late is that too
1467 // early versioning may prevent further inlining due to increase of code
1468 // size. Other optimizations which runs later might get benefit of no-alias
1469 // assumption in clone loop.
1470 if (Opts.enable_loop_versioning_licm) {
1471 OptimizePM.addPass(
1472 Pass: createFunctionToLoopPassAdaptor(Pass: LoopVersioningLICMPass()));
1473 // LoopVersioningLICM pass might increase new LICM opportunities.
1474 OptimizePM.addPass(Pass: createFunctionToLoopPassAdaptor(
1475 Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
1476 /*AllowSpeculation=*/true),
1477 /*USeMemorySSA=*/UseMemorySSA: true));
1478 }
1479
1480 OptimizePM.addPass(Pass: Float2IntPass());
1481 // Defer until LTO post-link where some constants may become known.
1482 if (!isLTOPreLink(Phase: LTOPhase))
1483 OptimizePM.addPass(Pass: LowerConstantIntrinsicsPass());
1484
1485 if (Opts.enable_matrix) {
1486 OptimizePM.addPass(Pass: LowerMatrixIntrinsicsPass());
1487 OptimizePM.addPass(Pass: EarlyCSEPass());
1488 }
1489
1490 // CHR pass should only be applied with the profile information.
1491 // The check is to check the profile summary information in CHR.
1492 if (Opts.enable_chr && Level == OptimizationLevel::O3)
1493 OptimizePM.addPass(Pass: ControlHeightReductionPass());
1494
1495 // FIXME: We need to run some loop optimizations to re-rotate loops after
1496 // simplifycfg and others undo their rotation.
1497
1498 // Optimize the loop execution. These passes operate on entire loop nests
1499 // rather than on each loop in an inside-out manner, and so they are actually
1500 // function passes.
1501
1502 invokeVectorizerStartEPCallbacks(FPM&: OptimizePM, Level);
1503
1504 LoopPassManager LPM;
1505 // First rotate loops that may have been un-rotated by prior passes.
1506 // Disable header duplication at -Oz.
1507 LPM.addPass(Pass: LoopRotatePass(/*EnableLoopHeaderDuplication=*/true,
1508 isLTOPreLink(Phase: LTOPhase),
1509 /*CheckExitCount=*/true));
1510 // Some loops may have become dead by now. Try to delete them.
1511 // FIXME: see discussion in https://reviews.llvm.org/D112851,
1512 // this may need to be revisited once we run GVN before loop deletion
1513 // in the simplification pipeline.
1514 LPM.addPass(Pass: LoopDeletionPass());
1515
1516 if (PTO.LoopInterchange)
1517 LPM.addPass(Pass: LoopInterchangePass());
1518
1519 OptimizePM.addPass(
1520 Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM), /*UseMemorySSA=*/false));
1521
1522 // FIXME: This may not be the right place in the pipeline.
1523 // We need to have the data to support the right place.
1524 if (PTO.LoopFusion)
1525 OptimizePM.addPass(Pass: LoopFusePass());
1526
1527 // Distribute loops to allow partial vectorization. I.e. isolate dependences
1528 // into separate loop that would otherwise inhibit vectorization. This is
1529 // currently only performed for loops marked with the metadata
1530 // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
1531 OptimizePM.addPass(Pass: LoopDistributePass());
1532
1533 // Populates the VFABI attribute with the scalar-to-vector mappings
1534 // from the TargetLibraryInfo.
1535 OptimizePM.addPass(Pass: InjectTLIMappings());
1536
1537 addVectorPasses(Level, FPM&: OptimizePM, LTOPhase);
1538
1539 invokeVectorizerEndEPCallbacks(FPM&: OptimizePM, Level);
1540
1541 // LoopSink pass sinks instructions hoisted by LICM, which serves as a
1542 // canonicalization pass that enables other optimizations. As a result,
1543 // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
1544 // result too early.
1545 OptimizePM.addPass(Pass: LoopSinkPass());
1546
1547 // And finally clean up LCSSA form before generating code.
1548 OptimizePM.addPass(Pass: InstSimplifyPass());
1549
1550 // This hoists/decomposes div/rem ops. It should run after other sink/hoist
1551 // passes to avoid re-sinking, but before SimplifyCFG because it can allow
1552 // flattening of blocks.
1553 OptimizePM.addPass(Pass: DivRemPairsPass());
1554
1555 // Merge adjacent icmps into memcmp, then expand memcmp to loads/compares.
1556 // TODO: move this further up so that it can be optimized by GVN, etc.
1557 if (Opts.enable_mergeicmps)
1558 OptimizePM.addPass(Pass: MergeICmpsPass());
1559 OptimizePM.addPass(Pass: ExpandMemCmpPass());
1560
1561 // Try to annotate calls that were created during optimization.
1562 OptimizePM.addPass(
1563 Pass: TailCallElimPass(/*UpdateFunctionEntryCount=*/isInstrumentedPGOUse()));
1564
1565 // LoopSink (and other loop passes since the last simplifyCFG) might have
1566 // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
1567 OptimizePM.addPass(
1568 Pass: SimplifyCFGPass(SimplifyCFGOptions()
1569 .convertSwitchRangeToICmp(B: true)
1570 .convertSwitchToArithmetic(B: true)
1571 .speculateUnpredictables(B: true)
1572 .hoistLoadsStoresWithCondFaulting(B: true)));
1573
1574 // Add the core optimizing pipeline.
1575 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(OptimizePM),
1576 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
1577
1578 // AllocToken transforms heap allocation calls; this needs to run late after
1579 // other allocation call transformations (such as those in InstCombine).
1580 if (!isLTOPreLink(Phase: LTOPhase))
1581 MPM.addPass(Pass: AllocTokenPass());
1582
1583 invokeOptimizerLastEPCallbacks(MPM, Level, Phase: LTOPhase);
1584
1585 // Run the Instrumentor pass late.
1586 if (Opts.enable_instrumentor)
1587 MPM.addPass(Pass: InstrumentorPass(FS));
1588
1589 // Split out cold code. Splitting is done late to avoid hiding context from
1590 // other optimizations and inadvertently regressing performance. The tradeoff
1591 // is that this has a higher code size cost than splitting early.
1592 if (Opts.hot_cold_split && !isLTOPreLink(Phase: LTOPhase))
1593 MPM.addPass(Pass: HotColdSplittingPass());
1594
1595 // Now we need to do some global optimization transforms.
1596 // FIXME: It would seem like these should come first in the optimization
1597 // pipeline and maybe be the bottom of the canonicalization pipeline? Weird
1598 // ordering here.
1599 MPM.addPass(Pass: GlobalDCEPass());
1600 MPM.addPass(Pass: ConstantMergePass());
1601
1602 // Merge functions if requested. It has a better chance to merge functions
1603 // after ConstantMerge folded jump tables.
1604 if (PTO.MergeFunctions)
1605 MPM.addPass(Pass: MergeFunctionsPass());
1606
1607 if (PTO.CallGraphProfile && !isLTOPreLink(Phase: LTOPhase))
1608 MPM.addPass(Pass: CGProfilePass(isLTOPostLink(Phase: LTOPhase)));
1609
1610 // RelLookupTableConverterPass runs later in LTO post-link pipeline.
1611 if (!isLTOPreLink(Phase: LTOPhase))
1612 MPM.addPass(Pass: RelLookupTableConverterPass());
1613
1614 // Add devirtualization pass only when LTO is not enabled, as otherwise
1615 // the pass is already enabled in the LTO pipeline.
1616 if (PTO.DevirtualizeSpeculatively && LTOPhase == ThinOrFullLTOPhase::None) {
1617 // TODO: explore a better pipeline configuration that can improve
1618 // compilation time overhead.
1619 // FIXME: move this earlier (lots of pass ordering tests will need fixing)
1620 MPM.addPass(Pass: AssignGUIDPass());
1621 MPM.addPass(Pass: WholeProgramDevirtPass(
1622 /*ExportSummary*/ nullptr,
1623 /*ImportSummary*/ nullptr,
1624 /*DevirtSpeculatively*/ PTO.DevirtualizeSpeculatively));
1625 MPM.addPass(Pass: DropTypeTestsPass());
1626 // Given that the devirtualization creates more opportunities for inlining,
1627 // we run the Inliner again here to maximize the optimization gain we
1628 // get from devirtualization.
1629 // Also, we can't run devirtualization before inlining because the
1630 // devirtualization depends on the passes optimizing/eliminating vtable GVs
1631 // and those passes are only effective after inlining.
1632 addModuleInlinerPass(MPM, Opts, Level, Phase: ThinOrFullLTOPhase::None);
1633 }
1634
1635 // Attach !implicit.ref metadata from all functions to copyright strings.
1636 MPM.addPass(Pass: LowerCommentStringPass());
1637
1638 return MPM;
1639}
1640
1641ModulePassManager
1642PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level,
1643 ThinOrFullLTOPhase Phase) {
1644 if (Level == OptimizationLevel::O0)
1645 return buildO0DefaultPipeline(Level, Phase);
1646
1647 ModulePassManager MPM;
1648 addPipelineStartPasses(MPM, Opts);
1649 // Currently this pipeline is only invoked in an LTO pre link pass or when we
1650 // are not running LTO. If that changes the below checks may need updating.
1651 assert(isLTOPreLink(Phase) || Phase == ThinOrFullLTOPhase::None);
1652
1653 // If we are invoking this in non-LTO mode, remove any MemProf related
1654 // attributes and metadata, as we don't know whether we are linking with
1655 // a library containing the necessary interfaces.
1656 if (Phase == ThinOrFullLTOPhase::None)
1657 MPM.addPass(Pass: MemProfRemoveInfo());
1658
1659 // Convert @llvm.global.annotations to !annotation metadata.
1660 MPM.addPass(Pass: Annotation2MetadataPass());
1661
1662 // Force any function attributes we want the rest of the pipeline to observe.
1663 MPM.addPass(Pass: ForceFunctionAttrsPass());
1664
1665 if (Opts.opt_pipeline_trigger_crash)
1666 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: TriggerCrashFunctionPass()));
1667
1668 if (PGOOpt && PGOOpt->DebugInfoForProfiling)
1669 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: AddDiscriminatorsPass()));
1670
1671 // Apply module pipeline start EP callback.
1672 invokePipelineStartEPCallbacks(MPM, Level);
1673
1674 // Add the core simplification pipeline.
1675 MPM.addPass(Pass: buildModuleSimplificationPipeline(Level, Phase));
1676
1677 // Now add the optimization pipeline.
1678 MPM.addPass(Pass: buildModuleOptimizationPipeline(Level, LTOPhase: Phase));
1679
1680 if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&
1681 PGOOpt->Action == PGOOptions::SampleUse)
1682 MPM.addPass(Pass: PseudoProbeUpdatePass());
1683
1684 // Emit annotation remarks.
1685 addAnnotationRemarksPass(MPM);
1686
1687 if (isLTOPreLink(Phase))
1688 addRequiredLTOPreLinkPasses(MPM);
1689
1690 addPipelineEndPasses(MPM, Opts);
1691 return MPM;
1692}
1693
1694ModulePassManager
1695PassBuilder::buildFatLTODefaultPipeline(OptimizationLevel Level, bool ThinLTO,
1696 bool EmitSummary, bool Verify) {
1697 ModulePassManager MPM;
1698
1699 addPipelineStartPasses(MPM, Opts);
1700
1701 if (ThinLTO)
1702 MPM.addPass(Pass: buildThinLTOPreLinkDefaultPipeline(Level));
1703 else
1704 MPM.addPass(Pass: buildLTOPreLinkDefaultPipeline(Level));
1705 // AssignGUIDPass attaches !guid metadata (MD_unique_id) to global objects,
1706 // triggering the bitcode writer to emit a METADATA_KIND_BLOCK. Standard LTO
1707 // bitcode emission runs VerifierPass by default, which registers metadata
1708 // kind IDs in LLVMContext. Running VerifierPass here before EmbedBitcodePass
1709 // to get the same behavior.
1710 if (Verify)
1711 MPM.addPass(Pass: VerifierPass());
1712 MPM.addPass(Pass: EmbedBitcodePass(ThinLTO, EmitSummary));
1713
1714 // Perform any cleanups to the IR that aren't suitable for per TU compilation,
1715 // like removing CFI/WPD related instructions. Note, we reuse
1716 // DropTypeTestsPass to clean up type tests rather than duplicate that logic
1717 // in FatLtoCleanup; checked-load cleanup is shared the other way, with
1718 // FatLtoCleanup calling lowertypetests::cleanUpTypeCheckedLoad.
1719 MPM.addPass(Pass: FatLtoCleanup());
1720
1721 // If we're doing FatLTO w/ CFI enabled, we don't want the type tests in the
1722 // object code, only in the bitcode section, so drop it before we run
1723 // module optimization and generate machine code. If llvm.type.test() isn't in
1724 // the IR, this won't do anything.
1725 MPM.addPass(Pass: DropTypeTestsPass(lowertypetests::DropTestKind::All));
1726
1727 // Use the ThinLTO post-link pipeline with sample profiling
1728 if (ThinLTO && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse)
1729 MPM.addPass(Pass: buildThinLTODefaultPipeline(Level, /*ImportSummary=*/nullptr));
1730 else {
1731 // ModuleSimplification does not run the coroutine passes for
1732 // ThinLTOPreLink, so we need the coroutine passes to run for ThinLTO
1733 // builds, otherwise they will miscompile.
1734 if (ThinLTO) {
1735 // TODO: replace w/ buildCoroWrapper() when it takes phase and level into
1736 // consideration.
1737 CGSCCPassManager CGPM;
1738 CGPM.addPass(Pass: CoroSplitPass(Level != OptimizationLevel::O0));
1739 CGPM.addPass(Pass: CoroAnnotationElidePass());
1740 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: std::move(CGPM)));
1741 MPM.addPass(Pass: CoroCleanupPass());
1742 }
1743
1744 // otherwise, just use module optimization
1745 MPM.addPass(
1746 Pass: buildModuleOptimizationPipeline(Level, LTOPhase: ThinOrFullLTOPhase::None));
1747 // Emit annotation remarks.
1748 addAnnotationRemarksPass(MPM);
1749 }
1750
1751 addPipelineEndPasses(MPM, Opts);
1752
1753 return MPM;
1754}
1755
1756ModulePassManager
1757PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level) {
1758 if (Level == OptimizationLevel::O0)
1759 return buildO0DefaultPipeline(Level, Phase: ThinOrFullLTOPhase::ThinLTOPreLink);
1760
1761 ModulePassManager MPM;
1762
1763 addPipelineStartPasses(MPM, Opts);
1764
1765 // Convert @llvm.global.annotations to !annotation metadata.
1766 MPM.addPass(Pass: Annotation2MetadataPass());
1767
1768 // Force any function attributes we want the rest of the pipeline to observe.
1769 MPM.addPass(Pass: ForceFunctionAttrsPass());
1770
1771 if (PGOOpt && PGOOpt->DebugInfoForProfiling)
1772 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: AddDiscriminatorsPass()));
1773
1774 // Apply module pipeline start EP callback.
1775 invokePipelineStartEPCallbacks(MPM, Level);
1776
1777 // If we are planning to perform ThinLTO later, we don't bloat the code with
1778 // unrolling/vectorization/... now. Just simplify the module as much as we
1779 // can.
1780 MPM.addPass(Pass: buildModuleSimplificationPipeline(
1781 Level, Phase: ThinOrFullLTOPhase::ThinLTOPreLink));
1782 // In pre-link, for ctx prof use, we stop here with an instrumented IR. We let
1783 // thinlto use the contextual info to perform imports; then use the contextual
1784 // profile in the post-thinlink phase.
1785 if (!UseCtxProfile.empty()) {
1786 addRequiredLTOPreLinkPasses(MPM);
1787 return MPM;
1788 }
1789
1790 // Run partial inlining pass to partially inline functions that have
1791 // large bodies.
1792 // FIXME: It isn't clear whether this is really the right place to run this
1793 // in ThinLTO. Because there is another canonicalization and simplification
1794 // phase that will run after the thin link, running this here ends up with
1795 // less information than will be available later and it may grow functions in
1796 // ways that aren't beneficial.
1797 if (Opts.enable_partial_inlining)
1798 MPM.addPass(Pass: PartialInlinerPass());
1799
1800 if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&
1801 PGOOpt->Action == PGOOptions::SampleUse)
1802 MPM.addPass(Pass: PseudoProbeUpdatePass());
1803
1804 // Handle Optimizer{Early,Last}EPCallbacks added by clang on PreLink. Actual
1805 // optimization is going to be done in PostLink stage, but clang can't add
1806 // callbacks there in case of in-process ThinLTO called by linker.
1807 invokeOptimizerEarlyEPCallbacks(MPM, Level,
1808 /*Phase=*/ThinOrFullLTOPhase::ThinLTOPreLink);
1809 invokeOptimizerLastEPCallbacks(MPM, Level,
1810 /*Phase=*/ThinOrFullLTOPhase::ThinLTOPreLink);
1811
1812 // Emit annotation remarks.
1813 addAnnotationRemarksPass(MPM);
1814
1815 // Attach !implicit.ref metadata from all functions to copyright strings.
1816 MPM.addPass(Pass: LowerCommentStringPass());
1817
1818 addRequiredLTOPreLinkPasses(MPM);
1819
1820 addPipelineEndPasses(MPM, Opts);
1821
1822 return MPM;
1823}
1824
1825ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
1826 OptimizationLevel Level, const ModuleSummaryIndex *ImportSummary) {
1827 ModulePassManager MPM;
1828
1829 addPipelineStartPasses(MPM, Opts);
1830
1831 invokeThinLinkTimeOptimizationEarlyEPCallbacks(MPM, Level);
1832
1833 // If we are invoking this without a summary index noting that we are linking
1834 // with a library containing the necessary APIs, remove any MemProf related
1835 // attributes and metadata.
1836 if (!ImportSummary || !ImportSummary->withSupportsHotColdNew())
1837 MPM.addPass(Pass: MemProfRemoveInfo());
1838
1839 if (ImportSummary) {
1840 // For ThinLTO we must apply the context disambiguation decisions early, to
1841 // ensure we can correctly match the callsites to summary data.
1842 if (EnableMemProfContextDisambiguation)
1843 MPM.addPass(Pass: MemProfContextDisambiguation(
1844 ImportSummary, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
1845
1846 // These passes import type identifier resolutions for whole-program
1847 // devirtualization and CFI. They must run early because other passes may
1848 // disturb the specific instruction patterns that these passes look for,
1849 // creating dependencies on resolutions that may not appear in the summary.
1850 //
1851 // For example, GVN may transform the pattern assume(type.test) appearing in
1852 // two basic blocks into assume(phi(type.test, type.test)), which would
1853 // transform a dependency on a WPD resolution into a dependency on a type
1854 // identifier resolution for CFI.
1855 //
1856 // Also, WPD has access to more precise information than ICP and can
1857 // devirtualize more effectively, so it should operate on the IR first.
1858 //
1859 // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1860 // metadata and intrinsics.
1861 MPM.addPass(Pass: WholeProgramDevirtPass(nullptr, ImportSummary));
1862 MPM.addPass(Pass: LowerTypeTestsPass(nullptr, ImportSummary));
1863 }
1864
1865 if (Level == OptimizationLevel::O0) {
1866 // Run a second time to clean up any type tests left behind by WPD for use
1867 // in ICP.
1868 MPM.addPass(Pass: DropTypeTestsPass());
1869 MPM.addPass(Pass: buildCoroWrapper(Phase: ThinOrFullLTOPhase::ThinLTOPostLink));
1870
1871 // AllocToken transforms heap allocation calls; this needs to run late after
1872 // other allocation call transformations (such as those in InstCombine).
1873 MPM.addPass(Pass: AllocTokenPass());
1874
1875 // Drop available_externally and unreferenced globals. This is necessary
1876 // with ThinLTO in order to avoid leaving undefined references to dead
1877 // globals in the object file.
1878 MPM.addPass(Pass: EliminateAvailableExternallyPass());
1879 MPM.addPass(Pass: GlobalDCEPass());
1880
1881 invokeThinLinkTimeOptimizationLastEPCallbacks(MPM, Level);
1882
1883 return MPM;
1884 }
1885 if (!UseCtxProfile.empty()) {
1886 MPM.addPass(
1887 Pass: buildModuleInlinerPipeline(Level, Phase: ThinOrFullLTOPhase::ThinLTOPostLink));
1888 } else {
1889 // Add the core simplification pipeline.
1890 MPM.addPass(Pass: buildModuleSimplificationPipeline(
1891 Level, Phase: ThinOrFullLTOPhase::ThinLTOPostLink));
1892 }
1893 // Now add the optimization pipeline.
1894 MPM.addPass(Pass: buildModuleOptimizationPipeline(
1895 Level, LTOPhase: ThinOrFullLTOPhase::ThinLTOPostLink));
1896
1897 invokeThinLinkTimeOptimizationLastEPCallbacks(MPM, Level);
1898
1899 // Emit annotation remarks.
1900 addAnnotationRemarksPass(MPM);
1901
1902 addPipelineEndPasses(MPM, Opts);
1903
1904 return MPM;
1905}
1906
1907ModulePassManager
1908PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level) {
1909 // FIXME: We should use a customized pre-link pipeline!
1910 return buildPerModuleDefaultPipeline(Level,
1911 Phase: ThinOrFullLTOPhase::FullLTOPreLink);
1912}
1913
1914ModulePassManager
1915PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
1916 ModuleSummaryIndex *ExportSummary) {
1917 ModulePassManager MPM;
1918
1919 addPipelineStartPasses(MPM, Opts);
1920
1921 invokeFullLinkTimeOptimizationEarlyEPCallbacks(MPM, Level);
1922
1923 // If we are invoking this without a summary index noting that we are linking
1924 // with a library containing the necessary APIs, remove any MemProf related
1925 // attributes and metadata.
1926 if (!ExportSummary || !ExportSummary->withSupportsHotColdNew())
1927 MPM.addPass(Pass: MemProfRemoveInfo());
1928
1929 // Create a function that performs CFI checks for cross-DSO calls with targets
1930 // in the current module.
1931 MPM.addPass(Pass: CrossDSOCFIPass());
1932
1933 if (Level == OptimizationLevel::O0) {
1934 // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1935 // metadata and intrinsics.
1936 MPM.addPass(Pass: WholeProgramDevirtPass(ExportSummary, nullptr));
1937 MPM.addPass(Pass: LowerTypeTestsPass(ExportSummary, nullptr));
1938 // Run a second time to clean up any type tests left behind by WPD for use
1939 // in ICP.
1940 MPM.addPass(Pass: DropTypeTestsPass());
1941
1942 MPM.addPass(Pass: buildCoroWrapper(Phase: ThinOrFullLTOPhase::FullLTOPostLink));
1943
1944 // AllocToken transforms heap allocation calls; this needs to run late after
1945 // other allocation call transformations (such as those in InstCombine).
1946 MPM.addPass(Pass: AllocTokenPass());
1947
1948 invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);
1949
1950 // Emit annotation remarks.
1951 addAnnotationRemarksPass(MPM);
1952
1953 return MPM;
1954 }
1955
1956 if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
1957 // Load sample profile before running the LTO optimization pipeline.
1958 MPM.addPass(Pass: SampleProfileLoaderPass(PGOOpt->ProfileFile,
1959 PGOOpt->ProfileRemappingFile,
1960 ThinOrFullLTOPhase::FullLTOPostLink));
1961 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1962 // RequireAnalysisPass for PSI before subsequent non-module passes.
1963 MPM.addPass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
1964 }
1965
1966 // Try to run OpenMP optimizations, quick no-op if no OpenMP metadata present.
1967 MPM.addPass(Pass: OpenMPOptPass(ThinOrFullLTOPhase::FullLTOPostLink));
1968
1969 // Remove unused virtual tables to improve the quality of code generated by
1970 // whole-program devirtualization and bitset lowering.
1971 MPM.addPass(Pass: GlobalDCEPass(/*InLTOPostLink=*/true));
1972
1973 // Do basic inference of function attributes from known properties of system
1974 // libraries and other oracles.
1975 MPM.addPass(Pass: InferFunctionAttrsPass());
1976
1977 if (Level >= OptimizationLevel::O2) {
1978 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
1979 Pass: CallSiteSplittingPass(), EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
1980
1981 // Indirect call promotion. This should promote all the targets that are
1982 // left by the earlier promotion pass that promotes intra-module targets.
1983 // This two-step promotion is to save the compile time. For LTO, it should
1984 // produce the same result as if we only do promotion here.
1985 MPM.addPass(Pass: PGOIndirectCallPromotion(
1986 true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
1987
1988 // Promoting by-reference arguments to by-value exposes more constants to
1989 // IPSCCP.
1990 CGSCCPassManager CGPM;
1991 if (Opts.attributor_enable & AttributorRunOption::CGSCC_LIGHT)
1992 CGPM.addPass(Pass: AttributorLightCGSCCPass());
1993 else
1994 CGPM.addPass(Pass: PostOrderFunctionAttrsPass());
1995 CGPM.addPass(Pass: ArgumentPromotionPass());
1996 CGPM.addPass(
1997 Pass: createCGSCCToFunctionPassAdaptor(Pass: SROAPass(SROAOptions::ModifyCFG)));
1998 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: std::move(CGPM)));
1999
2000 // Propagate constants at call sites into the functions they call. This
2001 // opens opportunities for globalopt (and inlining) by substituting function
2002 // pointers passed as arguments to direct uses of functions.
2003 MPM.addPass(Pass: IPSCCPPass(IPSCCPOptions(/*AllowFuncSpec=*/true)));
2004
2005 // Attach metadata to indirect call sites indicating the set of functions
2006 // they may target at run-time. This should follow IPSCCP.
2007 MPM.addPass(Pass: CalledValuePropagationPass());
2008 }
2009
2010 // Do RPO function attribute inference across the module to forward-propagate
2011 // attributes where applicable.
2012 // FIXME: Is this really an optimization rather than a canonicalization?
2013 if (Opts.attributor_enable & AttributorRunOption::MODULE_LIGHT)
2014 MPM.addPass(Pass: AttributorLightPass());
2015 else
2016 MPM.addPass(Pass: ReversePostOrderFunctionAttrsPass());
2017
2018 // Use in-range annotations on GEP indices to split globals where beneficial.
2019 MPM.addPass(Pass: GlobalSplitPass());
2020
2021 // Run whole program optimization of virtual call when the list of callees
2022 // is fixed.
2023 MPM.addPass(Pass: WholeProgramDevirtPass(ExportSummary, nullptr));
2024
2025 MPM.addPass(Pass: NoRecurseLTOInferencePass());
2026 // Stop here at -O1.
2027 if (Level == OptimizationLevel::O1) {
2028 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
2029 Pass: LowerConstantIntrinsicsPass(), EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
2030
2031 // The LowerTypeTestsPass needs to run to lower type metadata and the
2032 // type.test intrinsics. The pass does nothing if CFI is disabled.
2033 MPM.addPass(Pass: LowerTypeTestsPass(ExportSummary, nullptr));
2034 // Run a second time to clean up any type tests left behind by WPD for use
2035 // in ICP (which is performed earlier than this in the regular LTO
2036 // pipeline).
2037 MPM.addPass(Pass: DropTypeTestsPass());
2038
2039 MPM.addPass(Pass: buildCoroWrapper(Phase: ThinOrFullLTOPhase::FullLTOPostLink));
2040
2041 // AllocToken transforms heap allocation calls; this needs to run late after
2042 // other allocation call transformations (such as those in InstCombine).
2043 MPM.addPass(Pass: AllocTokenPass());
2044
2045 invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);
2046
2047 // Emit annotation remarks.
2048 addAnnotationRemarksPass(MPM);
2049
2050 addPipelineEndPasses(MPM, Opts);
2051
2052 return MPM;
2053 }
2054
2055 // TODO: Skip to match buildCoroWrapper.
2056 MPM.addPass(Pass: CoroEarlyPass());
2057
2058 // Optimize globals to try and fold them into constants.
2059 MPM.addPass(Pass: GlobalOptPass());
2060
2061 // Promote any localized globals to SSA registers.
2062 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: PromotePass()));
2063
2064 // Linking modules together can lead to duplicate global constant, only
2065 // keep one copy of each constant.
2066 MPM.addPass(Pass: ConstantMergePass());
2067
2068 // Remove unused arguments from functions.
2069 MPM.addPass(Pass: DeadArgumentEliminationPass());
2070
2071 // Reduce the code after globalopt and ipsccp. Both can open up significant
2072 // simplification opportunities, and both can propagate functions through
2073 // function pointers. When this happens, we often have to resolve varargs
2074 // calls, etc, so let instcombine do this.
2075 FunctionPassManager PeepholeFPM;
2076 PeepholeFPM.addPass(Pass: InstCombinePass());
2077 if (Level >= OptimizationLevel::O2)
2078 PeepholeFPM.addPass(Pass: AggressiveInstCombinePass());
2079 invokePeepholeEPCallbacks(FPM&: PeepholeFPM, Level);
2080
2081 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(PeepholeFPM),
2082 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
2083
2084 // Lower variadic functions for supported targets prior to inlining.
2085 MPM.addPass(Pass: ExpandVariadicsPass(ExpandVariadicsMode::Optimize));
2086
2087 // Note: historically, the PruneEH pass was run first to deduce nounwind and
2088 // generally clean up exception handling overhead. It isn't clear this is
2089 // valuable as the inliner doesn't currently care whether it is inlining an
2090 // invoke or a call.
2091 // Run the inliner now.
2092 addModuleInlinerPass(MPM, Opts, Level, Phase: ThinOrFullLTOPhase::FullLTOPostLink);
2093
2094 // Perform context disambiguation after inlining, since that would reduce the
2095 // amount of additional cloning required to distinguish the allocation
2096 // contexts.
2097 if (EnableMemProfContextDisambiguation)
2098 MPM.addPass(Pass: MemProfContextDisambiguation(
2099 /*Summary=*/nullptr,
2100 PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
2101
2102 // Optimize globals again after we ran the inliner.
2103 MPM.addPass(Pass: GlobalOptPass());
2104
2105 // Run the OpenMPOpt pass again after global optimizations.
2106 MPM.addPass(Pass: OpenMPOptPass(ThinOrFullLTOPhase::FullLTOPostLink));
2107
2108 // Garbage collect dead functions.
2109 MPM.addPass(Pass: GlobalDCEPass(/*InLTOPostLink=*/true));
2110
2111 // If we didn't decide to inline a function, check to see if we can
2112 // transform it to pass arguments by value instead of by reference.
2113 CGSCCPassManager CGPM;
2114 CGPM.addPass(Pass: ArgumentPromotionPass());
2115 CGPM.addPass(Pass: CoroSplitPass(Level != OptimizationLevel::O0));
2116 CGPM.addPass(Pass: CoroAnnotationElidePass());
2117 invokeCGSCCOptimizerLateEPCallbacks(CGPM, Level);
2118 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: std::move(CGPM)));
2119
2120 FunctionPassManager FPM;
2121 // The IPO Passes may leave cruft around. Clean up after them.
2122 FPM.addPass(Pass: InstCombinePass());
2123 invokePeepholeEPCallbacks(FPM, Level);
2124
2125 if (Opts.enable_constraint_elimination)
2126 FPM.addPass(Pass: ConstraintEliminationPass());
2127
2128 FPM.addPass(Pass: JumpThreadingPass());
2129
2130 // Do a post inline PGO instrumentation and use pass. This is a context
2131 // sensitive PGO pass.
2132 if (PGOOpt) {
2133 if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
2134 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/true,
2135 /*IsCS=*/true, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate,
2136 ProfileFile: PGOOpt->CSProfileGenFile, ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
2137 else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
2138 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/false,
2139 /*IsCS=*/true, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate,
2140 ProfileFile: PGOOpt->ProfileFile, ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
2141 }
2142
2143 // Break up allocas
2144 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
2145
2146 // LTO provides additional opportunities for tailcall elimination due to
2147 // link-time inlining, and visibility of nocapture attribute.
2148 FPM.addPass(
2149 Pass: TailCallElimPass(/*UpdateFunctionEntryCount=*/isInstrumentedPGOUse()));
2150
2151 // Run a few AA driver optimizations here and now to cleanup the code.
2152 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM),
2153 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
2154
2155 if (Opts.attributor_enable & AttributorRunOption::CGSCC_LIGHT) {
2156 MPM.addPass(
2157 Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: AttributorLightCGSCCPass()));
2158 } else {
2159 MPM.addPass(
2160 Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: PostOrderFunctionAttrsPass()));
2161 }
2162
2163 // Require the GlobalsAA analysis for the module so we can query it within
2164 // MainFPM.
2165 if (Opts.enable_global_analyses) {
2166 MPM.addPass(Pass: RequireAnalysisPass<GlobalsAA, Module>());
2167 // Invalidate AAManager so it can be recreated and pick up the newly
2168 // available GlobalsAA.
2169 MPM.addPass(
2170 Pass: createModuleToFunctionPassAdaptor(Pass: InvalidateAnalysisPass<AAManager>()));
2171 }
2172
2173 FunctionPassManager MainFPM;
2174 MainFPM.addPass(Pass: createFunctionToLoopPassAdaptor(
2175 Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
2176 /*AllowSpeculation=*/true),
2177 /*USeMemorySSA=*/UseMemorySSA: true));
2178
2179 if (Opts.enable_newgvn)
2180 MainFPM.addPass(Pass: NewGVNPass());
2181 else
2182 MainFPM.addPass(Pass: GVNPass());
2183
2184 // Remove dead memcpy()'s.
2185 MainFPM.addPass(Pass: MemCpyOptPass());
2186
2187 // Nuke dead stores.
2188 MainFPM.addPass(Pass: DSEPass());
2189 MainFPM.addPass(Pass: MoveAutoInitPass());
2190 MainFPM.addPass(Pass: MergedLoadStoreMotionPass());
2191
2192 MainFPM.addPass(Pass: LowerConstantIntrinsicsPass());
2193
2194 invokeVectorizerStartEPCallbacks(FPM&: MainFPM, Level);
2195
2196 LoopPassManager LPM;
2197 if (Opts.enable_loop_flatten && Level >= OptimizationLevel::O2)
2198 LPM.addPass(Pass: LoopFlattenPass());
2199 LPM.addPass(Pass: IndVarSimplifyPass());
2200 LPM.addPass(Pass: LoopDeletionPass());
2201 // FIXME: Add loop interchange.
2202
2203 // Unroll small loops and perform peeling.
2204 LPM.addPass(Pass: LoopFullUnrollPass(static_cast<int>(Level),
2205 /* OnlyWhenForced= */ !PTO.LoopUnrolling,
2206 PTO.ForgetAllSCEVInLoopUnroll));
2207 // The loop passes in LPM (LoopFullUnrollPass) do not preserve MemorySSA.
2208 // *All* loop passes must preserve it, in order to be able to use it.
2209 MainFPM.addPass(
2210 Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM), /*UseMemorySSA=*/false));
2211
2212 MainFPM.addPass(Pass: LoopDistributePass());
2213
2214 addVectorPasses(Level, FPM&: MainFPM, LTOPhase: ThinOrFullLTOPhase::FullLTOPostLink);
2215
2216 invokeVectorizerEndEPCallbacks(FPM&: MainFPM, Level);
2217
2218 // Run the OpenMPOpt CGSCC pass again late.
2219 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(
2220 Pass: OpenMPOptCGSCCPass(ThinOrFullLTOPhase::FullLTOPostLink)));
2221
2222 invokePeepholeEPCallbacks(FPM&: MainFPM, Level);
2223 MainFPM.addPass(Pass: JumpThreadingPass());
2224 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(MainFPM),
2225 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
2226
2227 // Lower type metadata and the type.test intrinsic. This pass supports
2228 // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
2229 // to be run at link time if CFI is enabled. This pass does nothing if
2230 // CFI is disabled.
2231 MPM.addPass(Pass: LowerTypeTestsPass(ExportSummary, nullptr));
2232 // Run a second time to clean up any type tests left behind by WPD for use
2233 // in ICP (which is performed earlier than this in the regular LTO pipeline).
2234 MPM.addPass(Pass: DropTypeTestsPass());
2235
2236 // Enable splitting late in the FullLTO post-link pipeline.
2237 if (Opts.hot_cold_split)
2238 MPM.addPass(Pass: HotColdSplittingPass());
2239
2240 // Add late LTO optimization passes.
2241 FunctionPassManager LateFPM;
2242
2243 // LoopSink pass sinks instructions hoisted by LICM, which serves as a
2244 // canonicalization pass that enables other optimizations. As a result,
2245 // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
2246 // result too early.
2247 LateFPM.addPass(Pass: LoopSinkPass());
2248
2249 // This hoists/decomposes div/rem ops. It should run after other sink/hoist
2250 // passes to avoid re-sinking, but before SimplifyCFG because it can allow
2251 // flattening of blocks.
2252 LateFPM.addPass(Pass: DivRemPairsPass());
2253
2254 // Merge adjacent icmps into memcmp, then expand memcmp to loads/compares.
2255 // TODO: move this further up so that it can be optimized by GVN, etc.
2256 if (Opts.enable_mergeicmps)
2257 LateFPM.addPass(Pass: MergeICmpsPass());
2258 LateFPM.addPass(Pass: ExpandMemCmpPass());
2259
2260 // Delete basic blocks, which optimization passes may have killed.
2261 LateFPM.addPass(Pass: SimplifyCFGPass(SimplifyCFGOptions()
2262 .convertSwitchRangeToICmp(B: true)
2263 .convertSwitchToArithmetic(B: true)
2264 .hoistCommonInsts(B: true)
2265 .speculateUnpredictables(B: true)));
2266 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(LateFPM)));
2267
2268 // Drop bodies of available eternally objects to improve GlobalDCE.
2269 MPM.addPass(Pass: EliminateAvailableExternallyPass());
2270
2271 // Now that we have optimized the program, discard unreachable functions.
2272 MPM.addPass(Pass: GlobalDCEPass(/*InLTOPostLink=*/true));
2273
2274 if (PTO.MergeFunctions)
2275 MPM.addPass(Pass: MergeFunctionsPass());
2276
2277 MPM.addPass(Pass: RelLookupTableConverterPass());
2278
2279 if (PTO.CallGraphProfile)
2280 MPM.addPass(Pass: CGProfilePass(/*InLTOPostLink=*/true));
2281
2282 MPM.addPass(Pass: CoroCleanupPass());
2283
2284 // AllocToken transforms heap allocation calls; this needs to run late after
2285 // other allocation call transformations (such as those in InstCombine).
2286 MPM.addPass(Pass: AllocTokenPass());
2287
2288 invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);
2289
2290 // Emit annotation remarks.
2291 addAnnotationRemarksPass(MPM);
2292
2293 addPipelineEndPasses(MPM, Opts);
2294
2295 return MPM;
2296}
2297
2298ModulePassManager
2299PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level,
2300 ThinOrFullLTOPhase Phase) {
2301 assert(Level == OptimizationLevel::O0 &&
2302 "buildO0DefaultPipeline should only be used with O0");
2303
2304 ModulePassManager MPM;
2305
2306 addPipelineStartPasses(MPM, Opts);
2307
2308 // Perform pseudo probe instrumentation in O0 mode. This is for the
2309 // consistency between different build modes. For example, a LTO build can be
2310 // mixed with an O0 prelink and an O2 postlink. Loading a sample profile in
2311 // the postlink will require pseudo probe instrumentation in the prelink.
2312 if (PGOOpt && PGOOpt->PseudoProbeForProfiling)
2313 MPM.addPass(Pass: SampleProfileProbePass(TM));
2314
2315 if (PGOOpt && (PGOOpt->Action == PGOOptions::IRInstr ||
2316 PGOOpt->Action == PGOOptions::IRUse))
2317 addPGOInstrPassesForO0(
2318 MPM,
2319 /*RunProfileGen=*/(PGOOpt->Action == PGOOptions::IRInstr),
2320 /*IsCS=*/false, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate, ProfileFile: PGOOpt->ProfileFile,
2321 ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
2322
2323 // Instrument function entry and exit before all inlining.
2324 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
2325 Pass: EntryExitInstrumenterPass(/*PostInlining=*/false)));
2326
2327 invokePipelineStartEPCallbacks(MPM, Level);
2328
2329 if (PGOOpt && PGOOpt->DebugInfoForProfiling)
2330 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: AddDiscriminatorsPass()));
2331
2332 if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
2333 // Explicitly disable sample loader inlining and use flattened profile in O0
2334 // pipeline.
2335 MPM.addPass(Pass: SampleProfileLoaderPass(PGOOpt->ProfileFile,
2336 PGOOpt->ProfileRemappingFile,
2337 ThinOrFullLTOPhase::None, FS,
2338 /*DisableSampleProfileInlining=*/true,
2339 /*UseFlattenedProfile=*/true));
2340 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
2341 // RequireAnalysisPass for PSI before subsequent non-module passes.
2342 MPM.addPass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
2343 }
2344
2345 invokePipelineEarlySimplificationEPCallbacks(MPM, Level, Phase);
2346
2347 // Build a minimal pipeline based on the semantics required by LLVM,
2348 // which is just that always inlining occurs. Further, disable generating
2349 // lifetime intrinsics to avoid enabling further optimizations during
2350 // code generation.
2351 MPM.addPass(Pass: AlwaysInlinerPass(
2352 /*InsertLifetimeIntrinsics=*/false));
2353
2354 if (PTO.MergeFunctions)
2355 MPM.addPass(Pass: MergeFunctionsPass());
2356
2357 if (Opts.enable_matrix)
2358 MPM.addPass(
2359 Pass: createModuleToFunctionPassAdaptor(Pass: LowerMatrixIntrinsicsPass(true)));
2360
2361 if (!CGSCCOptimizerLateEPCallbacks.empty()) {
2362 CGSCCPassManager CGPM;
2363 invokeCGSCCOptimizerLateEPCallbacks(CGPM, Level);
2364 if (!CGPM.isEmpty())
2365 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: std::move(CGPM)));
2366 }
2367 if (!LateLoopOptimizationsEPCallbacks.empty()) {
2368 LoopPassManager LPM;
2369 invokeLateLoopOptimizationsEPCallbacks(LPM, Level);
2370 if (!LPM.isEmpty()) {
2371 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
2372 Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM))));
2373 }
2374 }
2375 if (!LoopOptimizerEndEPCallbacks.empty()) {
2376 LoopPassManager LPM;
2377 invokeLoopOptimizerEndEPCallbacks(LPM, Level);
2378 if (!LPM.isEmpty()) {
2379 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
2380 Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM))));
2381 }
2382 }
2383 if (!ScalarOptimizerLateEPCallbacks.empty()) {
2384 FunctionPassManager FPM;
2385 invokeScalarOptimizerLateEPCallbacks(FPM, Level);
2386 if (!FPM.isEmpty())
2387 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
2388 }
2389
2390 invokeOptimizerEarlyEPCallbacks(MPM, Level, Phase);
2391
2392 if (!VectorizerStartEPCallbacks.empty()) {
2393 FunctionPassManager FPM;
2394 invokeVectorizerStartEPCallbacks(FPM, Level);
2395 if (!FPM.isEmpty())
2396 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
2397 }
2398
2399 if (!VectorizerEndEPCallbacks.empty()) {
2400 FunctionPassManager FPM;
2401 invokeVectorizerEndEPCallbacks(FPM, Level);
2402 if (!FPM.isEmpty())
2403 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
2404 }
2405
2406 MPM.addPass(Pass: buildCoroWrapper(Phase));
2407
2408 // AllocToken transforms heap allocation calls; this needs to run late after
2409 // other allocation call transformations (such as those in InstCombine).
2410 if (!isLTOPreLink(Phase))
2411 MPM.addPass(Pass: AllocTokenPass());
2412
2413 invokeOptimizerLastEPCallbacks(MPM, Level, Phase);
2414
2415 if (Opts.enable_instrumentor)
2416 MPM.addPass(Pass: InstrumentorPass(FS));
2417
2418 // Attach !implicit.ref metadata from all functions to copyright strings.
2419 MPM.addPass(Pass: LowerCommentStringPass());
2420
2421 if (isLTOPreLink(Phase))
2422 addRequiredLTOPreLinkPasses(MPM);
2423
2424 // Emit annotation remarks.
2425 addAnnotationRemarksPass(MPM);
2426
2427 addPipelineEndPasses(MPM, Opts);
2428
2429 return MPM;
2430}
2431
2432AAManager PassBuilder::buildDefaultAAPipeline() {
2433 AAManager AA;
2434
2435 // The order in which these are registered determines their priority when
2436 // being queried.
2437
2438 // Add any target-specific alias analyses that should be run early.
2439 if (TM)
2440 TM->registerEarlyDefaultAliasAnalyses(AA);
2441
2442 // First we register the basic alias analysis that provides the majority of
2443 // per-function local AA logic. This is a stateless, on-demand local set of
2444 // AA techniques.
2445 AA.registerFunctionAnalysis<BasicAA>();
2446
2447 // Next we query fast, specialized alias analyses that wrap IR-embedded
2448 // information about aliasing.
2449 AA.registerFunctionAnalysis<ScopedNoAliasAA>();
2450 AA.registerFunctionAnalysis<TypeBasedAA>();
2451
2452 // Add support for querying global aliasing information when available.
2453 // Because the `AAManager` is a function analysis and `GlobalsAA` is a module
2454 // analysis, all that the `AAManager` can do is query for any *cached*
2455 // results from `GlobalsAA` through a readonly proxy.
2456 if (Opts.enable_global_analyses)
2457 AA.registerModuleAnalysis<GlobalsAA>();
2458
2459 // Add target-specific alias analyses.
2460 if (TM)
2461 TM->registerDefaultAliasAnalyses(AA);
2462
2463 return AA;
2464}
2465
2466bool PassBuilder::isInstrumentedPGOUse() const {
2467 return (PGOOpt && PGOOpt->Action == PGOOptions::IRUse) ||
2468 !UseCtxProfile.empty();
2469}
2470