1//===- Construction of pass pipelines -------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9///
10/// This file provides the implementation of the PassBuilder based on our
11/// static pass registry as well as related functionality. It also provides
12/// helpers to aid in analyzing, debugging, and testing passes and pass
13/// pipelines.
14///
15//===----------------------------------------------------------------------===//
16
17#include "llvm/ADT/Statistic.h"
18#include "llvm/Analysis/AliasAnalysis.h"
19#include "llvm/Analysis/BasicAliasAnalysis.h"
20#include "llvm/Analysis/CGSCCPassManager.h"
21#include "llvm/Analysis/CtxProfAnalysis.h"
22#include "llvm/Analysis/FunctionPropertiesAnalysis.h"
23#include "llvm/Analysis/GlobalsModRef.h"
24#include "llvm/Analysis/InlineAdvisor.h"
25#include "llvm/Analysis/InstCount.h"
26#include "llvm/Analysis/ProfileSummaryInfo.h"
27#include "llvm/Analysis/ScopedNoAliasAA.h"
28#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
29#include "llvm/IR/PassManager.h"
30#include "llvm/IR/Verifier.h"
31#include "llvm/Pass.h"
32#include "llvm/Passes/OptimizationLevel.h"
33#include "llvm/Passes/PassBuilder.h"
34#include "llvm/Support/CommandLine.h"
35#include "llvm/Support/ErrorHandling.h"
36#include "llvm/Support/PGOOptions.h"
37#include "llvm/Support/VirtualFileSystem.h"
38#include "llvm/Target/TargetMachine.h"
39#include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
40#include "llvm/Transforms/Coroutines/CoroAnnotationElide.h"
41#include "llvm/Transforms/Coroutines/CoroCleanup.h"
42#include "llvm/Transforms/Coroutines/CoroConditionalWrapper.h"
43#include "llvm/Transforms/Coroutines/CoroEarly.h"
44#include "llvm/Transforms/Coroutines/CoroElide.h"
45#include "llvm/Transforms/Coroutines/CoroSplit.h"
46#include "llvm/Transforms/HipStdPar/HipStdPar.h"
47#include "llvm/Transforms/IPO/AlwaysInliner.h"
48#include "llvm/Transforms/IPO/Annotation2Metadata.h"
49#include "llvm/Transforms/IPO/ArgumentPromotion.h"
50#include "llvm/Transforms/IPO/Attributor.h"
51#include "llvm/Transforms/IPO/CalledValuePropagation.h"
52#include "llvm/Transforms/IPO/ConstantMerge.h"
53#include "llvm/Transforms/IPO/CrossDSOCFI.h"
54#include "llvm/Transforms/IPO/DeadArgumentElimination.h"
55#include "llvm/Transforms/IPO/ElimAvailExtern.h"
56#include "llvm/Transforms/IPO/EmbedBitcodePass.h"
57#include "llvm/Transforms/IPO/ExpandVariadics.h"
58#include "llvm/Transforms/IPO/FatLTOCleanup.h"
59#include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
60#include "llvm/Transforms/IPO/FunctionAttrs.h"
61#include "llvm/Transforms/IPO/GlobalDCE.h"
62#include "llvm/Transforms/IPO/GlobalOpt.h"
63#include "llvm/Transforms/IPO/GlobalSplit.h"
64#include "llvm/Transforms/IPO/HotColdSplitting.h"
65#include "llvm/Transforms/IPO/InferFunctionAttrs.h"
66#include "llvm/Transforms/IPO/Inliner.h"
67#include "llvm/Transforms/IPO/Instrumentor.h"
68#include "llvm/Transforms/IPO/LowerTypeTests.h"
69#include "llvm/Transforms/IPO/MemProfContextDisambiguation.h"
70#include "llvm/Transforms/IPO/MergeFunctions.h"
71#include "llvm/Transforms/IPO/ModuleInliner.h"
72#include "llvm/Transforms/IPO/OpenMPOpt.h"
73#include "llvm/Transforms/IPO/PartialInlining.h"
74#include "llvm/Transforms/IPO/SCCP.h"
75#include "llvm/Transforms/IPO/SampleProfile.h"
76#include "llvm/Transforms/IPO/SampleProfileProbe.h"
77#include "llvm/Transforms/IPO/WholeProgramDevirt.h"
78#include "llvm/Transforms/InstCombine/InstCombine.h"
79#include "llvm/Transforms/Instrumentation/AllocToken.h"
80#include "llvm/Transforms/Instrumentation/CGProfile.h"
81#include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
82#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
83#include "llvm/Transforms/Instrumentation/MemProfInstrumentation.h"
84#include "llvm/Transforms/Instrumentation/MemProfUse.h"
85#include "llvm/Transforms/Instrumentation/PGOCtxProfFlattening.h"
86#include "llvm/Transforms/Instrumentation/PGOCtxProfLowering.h"
87#include "llvm/Transforms/Instrumentation/PGOForceFunctionAttrs.h"
88#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
89#include "llvm/Transforms/Scalar/ADCE.h"
90#include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
91#include "llvm/Transforms/Scalar/AnnotationRemarks.h"
92#include "llvm/Transforms/Scalar/BDCE.h"
93#include "llvm/Transforms/Scalar/CallSiteSplitting.h"
94#include "llvm/Transforms/Scalar/ConstraintElimination.h"
95#include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
96#include "llvm/Transforms/Scalar/DFAJumpThreading.h"
97#include "llvm/Transforms/Scalar/DeadStoreElimination.h"
98#include "llvm/Transforms/Scalar/DivRemPairs.h"
99#include "llvm/Transforms/Scalar/DropUnnecessaryAssumes.h"
100#include "llvm/Transforms/Scalar/EarlyCSE.h"
101#include "llvm/Transforms/Scalar/ExpandMemCmp.h"
102#include "llvm/Transforms/Scalar/Float2Int.h"
103#include "llvm/Transforms/Scalar/GVN.h"
104#include "llvm/Transforms/Scalar/IndVarSimplify.h"
105#include "llvm/Transforms/Scalar/InferAlignment.h"
106#include "llvm/Transforms/Scalar/InstSimplifyPass.h"
107#include "llvm/Transforms/Scalar/JumpTableToSwitch.h"
108#include "llvm/Transforms/Scalar/JumpThreading.h"
109#include "llvm/Transforms/Scalar/LICM.h"
110#include "llvm/Transforms/Scalar/LoopDeletion.h"
111#include "llvm/Transforms/Scalar/LoopDistribute.h"
112#include "llvm/Transforms/Scalar/LoopFlatten.h"
113#include "llvm/Transforms/Scalar/LoopFuse.h"
114#include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
115#include "llvm/Transforms/Scalar/LoopInstSimplify.h"
116#include "llvm/Transforms/Scalar/LoopInterchange.h"
117#include "llvm/Transforms/Scalar/LoopLoadElimination.h"
118#include "llvm/Transforms/Scalar/LoopPassManager.h"
119#include "llvm/Transforms/Scalar/LoopRotation.h"
120#include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
121#include "llvm/Transforms/Scalar/LoopSink.h"
122#include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
123#include "llvm/Transforms/Scalar/LoopUnrollPass.h"
124#include "llvm/Transforms/Scalar/LoopVersioningLICM.h"
125#include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
126#include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
127#include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h"
128#include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
129#include "llvm/Transforms/Scalar/MergeICmps.h"
130#include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"
131#include "llvm/Transforms/Scalar/NewGVN.h"
132#include "llvm/Transforms/Scalar/Reassociate.h"
133#include "llvm/Transforms/Scalar/SCCP.h"
134#include "llvm/Transforms/Scalar/SROA.h"
135#include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
136#include "llvm/Transforms/Scalar/SimplifyCFG.h"
137#include "llvm/Transforms/Scalar/SpeculativeExecution.h"
138#include "llvm/Transforms/Scalar/TailRecursionElimination.h"
139#include "llvm/Transforms/Scalar/WarnMissedTransforms.h"
140#include "llvm/Transforms/Utils/AddDiscriminators.h"
141#include "llvm/Transforms/Utils/AssignGUID.h"
142#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
143#include "llvm/Transforms/Utils/CanonicalizeAliases.h"
144#include "llvm/Transforms/Utils/CountVisits.h"
145#include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
146#include "llvm/Transforms/Utils/ExtraPassManager.h"
147#include "llvm/Transforms/Utils/InjectTLIMappings.h"
148#include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
149#include "llvm/Transforms/Utils/LowerCommentStringPass.h"
150#include "llvm/Transforms/Utils/Mem2Reg.h"
151#include "llvm/Transforms/Utils/MoveAutoInit.h"
152#include "llvm/Transforms/Utils/NameAnonGlobals.h"
153#include "llvm/Transforms/Utils/RelLookupTableConverter.h"
154#include "llvm/Transforms/Utils/SimplifyCFGOptions.h"
155#include "llvm/Transforms/Utils/TriggerCrashPass.h"
156#include "llvm/Transforms/Vectorize/LoopVectorize.h"
157#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
158#include "llvm/Transforms/Vectorize/VectorCombine.h"
159
160using namespace llvm;
161
162namespace llvm {
163
164static cl::opt<InliningAdvisorMode> UseInlineAdvisor(
165 "enable-ml-inliner", cl::init(Val: InliningAdvisorMode::Default), cl::Hidden,
166 cl::desc("Enable ML policy for inliner. Currently trained for -Oz only"),
167 cl::values(clEnumValN(InliningAdvisorMode::Default, "default",
168 "Heuristics-based inliner version"),
169 clEnumValN(InliningAdvisorMode::Development, "development",
170 "Use development mode (runtime-loadable model)"),
171 clEnumValN(InliningAdvisorMode::Release, "release",
172 "Use release mode (AOT-compiled model)")));
173
174/// Flag to enable inline deferral during PGO.
175static cl::opt<bool>
176 EnablePGOInlineDeferral("enable-npm-pgo-inline-deferral", cl::init(Val: true),
177 cl::Hidden,
178 cl::desc("Enable inline deferral during PGO"));
179
180static cl::opt<bool> EnableModuleInliner("enable-module-inliner",
181 cl::init(Val: false), cl::Hidden,
182 cl::desc("Enable module inliner"));
183
184static cl::opt<bool> PerformMandatoryInliningsFirst(
185 "mandatory-inlining-first", cl::init(Val: false), cl::Hidden,
186 cl::desc("Perform mandatory inlinings module-wide, before performing "
187 "inlining"));
188
189static cl::opt<bool> EnableEagerlyInvalidateAnalyses(
190 "eagerly-invalidate-analyses", cl::init(Val: true), cl::Hidden,
191 cl::desc("Eagerly invalidate more analyses in default pipelines"));
192
193static cl::opt<bool> EnableMergeFunctions(
194 "enable-merge-functions", cl::init(Val: false), cl::Hidden,
195 cl::desc("Enable function merging as part of the optimization pipeline"));
196
197static cl::opt<bool> EnablePostPGOLoopRotation(
198 "enable-post-pgo-loop-rotation", cl::init(Val: true), cl::Hidden,
199 cl::desc("Run the loop rotation transformation after PGO instrumentation"));
200
201static cl::opt<bool>
202 TriggerCrash("opt-pipeline-trigger-crash", cl::init(Val: false), cl::Hidden,
203 cl::desc("Trigger crash in optimization pipeline"));
204
205static cl::opt<bool> EnableGlobalAnalyses(
206 "enable-global-analyses", cl::init(Val: true), cl::Hidden,
207 cl::desc("Enable inter-procedural analyses"));
208
209static cl::opt<bool> RunPartialInlining("enable-partial-inlining",
210 cl::init(Val: false), cl::Hidden,
211 cl::desc("Run Partial inlining pass"));
212
213static cl::opt<bool> ExtraVectorizerPasses(
214 "extra-vectorizer-passes", cl::init(Val: false), cl::Hidden,
215 cl::desc("Run cleanup optimization passes after vectorization"));
216
217static cl::opt<bool> RunNewGVN("enable-newgvn", cl::init(Val: false), cl::Hidden,
218 cl::desc("Run the NewGVN pass"));
219
220static cl::opt<bool>
221 EnableLoopInterchange("enable-loopinterchange", cl::init(Val: true), cl::Hidden,
222 cl::desc("Enable the LoopInterchange Pass"));
223
224static cl::opt<bool> EnableUnrollAndJam("enable-unroll-and-jam",
225 cl::init(Val: false), cl::Hidden,
226 cl::desc("Enable Unroll And Jam Pass"));
227
228static cl::opt<bool> EnableLoopFlatten("enable-loop-flatten", cl::init(Val: false),
229 cl::Hidden,
230 cl::desc("Enable the LoopFlatten Pass"));
231
232static cl::opt<bool>
233 EnableInstrumentor("enable-instrumentor", cl::init(Val: false), cl::Hidden,
234 cl::desc("Enable the Instrumentor Pass"));
235
236static cl::opt<bool>
237 EnableDFAJumpThreading("enable-dfa-jump-thread",
238 cl::desc("Enable DFA jump threading"),
239 cl::init(Val: true), cl::Hidden);
240
241static cl::opt<bool>
242 EnableHotColdSplit("hot-cold-split",
243 cl::desc("Enable hot-cold splitting pass"));
244
245static cl::opt<bool>
246 DisablePreInliner("disable-preinline", cl::init(Val: false), cl::Hidden,
247 cl::desc("Disable pre-instrumentation inliner"));
248
249static cl::opt<int> PreInlineThreshold(
250 "preinline-threshold", cl::Hidden, cl::init(Val: 75),
251 cl::desc("Control the amount of inlining in pre-instrumentation inliner "
252 "(default = 75)"));
253
254static cl::opt<bool>
255 EnableGVNHoist("enable-gvn-hoist",
256 cl::desc("Enable the GVN hoisting pass (default = off)"));
257
258static cl::opt<bool>
259 EnableGVNSink("enable-gvn-sink",
260 cl::desc("Enable the GVN sinking pass (default = off)"));
261
262static cl::opt<bool> EnableJumpTableToSwitch(
263 "enable-jump-table-to-switch", cl::init(Val: true),
264 cl::desc("Enable JumpTableToSwitch pass (default = true)"));
265
266// This option is used in simplifying testing SampleFDO optimizations for
267// profile loading.
268static cl::opt<bool>
269 EnableCHR("enable-chr", cl::init(Val: true), cl::Hidden,
270 cl::desc("Enable control height reduction optimization (CHR)"));
271
272static cl::opt<bool> FlattenedProfileUsed(
273 "flattened-profile-used", cl::init(Val: false), cl::Hidden,
274 cl::desc("Indicate the sample profile being used is flattened, i.e., "
275 "no inline hierarchy exists in the profile"));
276
277static cl::opt<bool>
278 EnableMatrix("enable-matrix", cl::init(Val: false), cl::Hidden,
279 cl::desc("Enable lowering of the matrix intrinsics"));
280
281static cl::opt<bool> EnableMergeICmps(
282 "enable-mergeicmps", cl::init(Val: true), cl::Hidden,
283 cl::desc("Enable MergeICmps pass in the optimization pipeline"));
284
285static cl::opt<bool> EnableConstraintElimination(
286 "enable-constraint-elimination", cl::init(Val: true), cl::Hidden,
287 cl::desc(
288 "Enable pass to eliminate conditions based on linear constraints"));
289
290static cl::opt<AttributorRunOption> AttributorRun(
291 "attributor-enable", cl::Hidden, cl::init(Val: AttributorRunOption::NONE),
292 cl::desc("Enable the attributor inter-procedural deduction pass"),
293 cl::values(clEnumValN(AttributorRunOption::FULL, "full",
294 "enable all full attributor runs"),
295 clEnumValN(AttributorRunOption::LIGHT, "light",
296 "enable all attributor-light runs"),
297 clEnumValN(AttributorRunOption::MODULE, "module",
298 "enable module-wide attributor runs"),
299 clEnumValN(AttributorRunOption::MODULE_LIGHT, "module-light",
300 "enable module-wide attributor-light runs"),
301 clEnumValN(AttributorRunOption::CGSCC, "cgscc",
302 "enable call graph SCC attributor runs"),
303 clEnumValN(AttributorRunOption::CGSCC_LIGHT, "cgscc-light",
304 "enable call graph SCC attributor-light runs"),
305 clEnumValN(AttributorRunOption::NONE, "none",
306 "disable attributor runs")));
307
308static cl::opt<bool> EnableSampledInstr(
309 "enable-sampled-instrumentation", cl::init(Val: false), cl::Hidden,
310 cl::desc("Enable profile instrumentation sampling (default = off)"));
311static cl::opt<bool> UseLoopVersioningLICM(
312 "enable-loop-versioning-licm", cl::init(Val: false), cl::Hidden,
313 cl::desc("Enable the experimental Loop Versioning LICM pass"));
314
315static cl::opt<std::string> InstrumentColdFuncOnlyPath(
316 "instrument-cold-function-only-path", cl::init(Val: ""),
317 cl::desc("File path for cold function only instrumentation(requires use "
318 "with --pgo-instrument-cold-function-only)"),
319 cl::Hidden);
320
321// TODO: There is a similar flag in WPD pass, we should consolidate them by
322// parsing the option only once in PassBuilder and share it across both places.
323static cl::opt<bool> EnableDevirtualizeSpeculatively(
324 "enable-devirtualize-speculatively",
325 cl::desc("Enable speculative devirtualization optimization"),
326 cl::init(Val: false));
327
328extern cl::opt<std::string> UseCtxProfile;
329extern cl::opt<bool> PGOInstrumentColdFunctionOnly;
330
331extern cl::opt<bool> EnableMemProfContextDisambiguation;
332} // namespace llvm
333
334PipelineTuningOptions::PipelineTuningOptions() {
335 LoopInterleaving = true;
336 LoopVectorization = true;
337 SLPVectorization = false;
338 LoopUnrolling = true;
339 LoopInterchange = EnableLoopInterchange;
340 LoopFusion = false;
341 ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll;
342 LicmMssaOptCap = SetLicmMssaOptCap;
343 LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;
344 CallGraphProfile = true;
345 UnifiedLTO = false;
346 MergeFunctions = EnableMergeFunctions;
347 InlinerThreshold = -1;
348 EagerlyInvalidateAnalyses = EnableEagerlyInvalidateAnalyses;
349 DevirtualizeSpeculatively = EnableDevirtualizeSpeculatively;
350}
351
352namespace llvm {
353extern cl::opt<unsigned> MaxDevirtIterations;
354} // namespace llvm
355
356void PassBuilder::invokePeepholeEPCallbacks(FunctionPassManager &FPM,
357 OptimizationLevel Level) {
358 for (auto &C : PeepholeEPCallbacks)
359 C(FPM, Level);
360}
361void PassBuilder::invokeLateLoopOptimizationsEPCallbacks(
362 LoopPassManager &LPM, OptimizationLevel Level) {
363 for (auto &C : LateLoopOptimizationsEPCallbacks)
364 C(LPM, Level);
365}
366void PassBuilder::invokeLoopOptimizerEndEPCallbacks(LoopPassManager &LPM,
367 OptimizationLevel Level) {
368 for (auto &C : LoopOptimizerEndEPCallbacks)
369 C(LPM, Level);
370}
371void PassBuilder::invokeScalarOptimizerLateEPCallbacks(
372 FunctionPassManager &FPM, OptimizationLevel Level) {
373 for (auto &C : ScalarOptimizerLateEPCallbacks)
374 C(FPM, Level);
375}
376void PassBuilder::invokeCGSCCOptimizerLateEPCallbacks(CGSCCPassManager &CGPM,
377 OptimizationLevel Level) {
378 for (auto &C : CGSCCOptimizerLateEPCallbacks)
379 C(CGPM, Level);
380}
381void PassBuilder::invokeVectorizerStartEPCallbacks(FunctionPassManager &FPM,
382 OptimizationLevel Level) {
383 for (auto &C : VectorizerStartEPCallbacks)
384 C(FPM, Level);
385}
386void PassBuilder::invokeVectorizerEndEPCallbacks(FunctionPassManager &FPM,
387 OptimizationLevel Level) {
388 for (auto &C : VectorizerEndEPCallbacks)
389 C(FPM, Level);
390}
391void PassBuilder::invokeOptimizerEarlyEPCallbacks(ModulePassManager &MPM,
392 OptimizationLevel Level,
393 ThinOrFullLTOPhase Phase) {
394 for (auto &C : OptimizerEarlyEPCallbacks)
395 C(MPM, Level, Phase);
396}
397void PassBuilder::invokeOptimizerLastEPCallbacks(ModulePassManager &MPM,
398 OptimizationLevel Level,
399 ThinOrFullLTOPhase Phase) {
400 for (auto &C : OptimizerLastEPCallbacks)
401 C(MPM, Level, Phase);
402}
403void PassBuilder::invokeFullLinkTimeOptimizationEarlyEPCallbacks(
404 ModulePassManager &MPM, OptimizationLevel Level) {
405 for (auto &C : FullLinkTimeOptimizationEarlyEPCallbacks)
406 C(MPM, Level);
407}
408void PassBuilder::invokeFullLinkTimeOptimizationLastEPCallbacks(
409 ModulePassManager &MPM, OptimizationLevel Level) {
410 for (auto &C : FullLinkTimeOptimizationLastEPCallbacks)
411 C(MPM, Level);
412}
413void PassBuilder::invokePipelineStartEPCallbacks(ModulePassManager &MPM,
414 OptimizationLevel Level) {
415 for (auto &C : PipelineStartEPCallbacks)
416 C(MPM, Level);
417}
418void PassBuilder::invokePipelineEarlySimplificationEPCallbacks(
419 ModulePassManager &MPM, OptimizationLevel Level, ThinOrFullLTOPhase Phase) {
420 for (auto &C : PipelineEarlySimplificationEPCallbacks)
421 C(MPM, Level, Phase);
422}
423
424// Get IR stats with InstCount before/after the optimization pipeline
425static void instructionCountersPass(ModulePassManager &MPM,
426 bool IsPreOptimization) {
427 if (AreStatisticsEnabled()) {
428 MPM.addPass(
429 Pass: createModuleToFunctionPassAdaptor(Pass: InstCountPass(IsPreOptimization)));
430 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
431 Pass: FunctionPropertiesStatisticsPass(IsPreOptimization)));
432 }
433}
434
435// Helper to add AnnotationRemarksPass.
436static void addAnnotationRemarksPass(ModulePassManager &MPM) {
437 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: AnnotationRemarksPass()));
438}
439
440// Helper to check if the current compilation phase is preparing for LTO
441static bool isLTOPreLink(ThinOrFullLTOPhase Phase) {
442 return Phase == ThinOrFullLTOPhase::ThinLTOPreLink ||
443 Phase == ThinOrFullLTOPhase::FullLTOPreLink;
444}
445
446// Helper to check if the current compilation phase is preparing for FullLTO
447[[maybe_unused]] static bool isFullLTOPreLink(ThinOrFullLTOPhase Phase) {
448 return Phase == ThinOrFullLTOPhase::FullLTOPreLink;
449}
450
451// Helper to check if the current compilation phase is preparing for ThinLTO
452static bool isThinLTOPreLink(ThinOrFullLTOPhase Phase) {
453 return Phase == ThinOrFullLTOPhase::ThinLTOPreLink;
454}
455
456// Helper to check if the current compilation phase is LTO backend
457static bool isLTOPostLink(ThinOrFullLTOPhase Phase) {
458 return Phase == ThinOrFullLTOPhase::ThinLTOPostLink ||
459 Phase == ThinOrFullLTOPhase::FullLTOPostLink;
460}
461
462// Helper to check if the current compilation phase is FullLTO backend
463static bool isFullLTOPostLink(ThinOrFullLTOPhase Phase) {
464 return Phase == ThinOrFullLTOPhase::FullLTOPostLink;
465}
466
467// Helper to check if the current compilation phase is ThinLTO backend
468static bool isThinLTOPostLink(ThinOrFullLTOPhase Phase) {
469 return Phase == ThinOrFullLTOPhase::ThinLTOPostLink;
470}
471
472// Helper to wrap conditionally Coro passes.
473static CoroConditionalWrapper buildCoroWrapper(ThinOrFullLTOPhase Phase) {
474 // TODO: Skip passes according to Phase.
475 ModulePassManager CoroPM;
476 CoroPM.addPass(Pass: CoroEarlyPass());
477 CGSCCPassManager CGPM;
478 CGPM.addPass(Pass: CoroSplitPass());
479 CoroPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: std::move(CGPM)));
480 CoroPM.addPass(Pass: CoroCleanupPass());
481 CoroPM.addPass(Pass: GlobalDCEPass());
482 return CoroConditionalWrapper(std::move(CoroPM));
483}
484
485// TODO: Investigate the cost/benefit of tail call elimination on debugging.
486FunctionPassManager
487PassBuilder::buildO1FunctionSimplificationPipeline(OptimizationLevel Level,
488 ThinOrFullLTOPhase Phase) {
489
490 FunctionPassManager FPM;
491
492 if (AreStatisticsEnabled())
493 FPM.addPass(Pass: CountVisitsPass());
494
495 // Form SSA out of local memory accesses after breaking apart aggregates into
496 // scalars.
497 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
498
499 // Catch trivial redundancies
500 FPM.addPass(Pass: EarlyCSEPass(true /* Enable mem-ssa. */));
501
502 // Hoisting of scalars and load expressions.
503 FPM.addPass(
504 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
505 FPM.addPass(Pass: InstCombinePass());
506
507 FPM.addPass(Pass: LibCallsShrinkWrapPass());
508
509 invokePeepholeEPCallbacks(FPM, Level);
510
511 FPM.addPass(
512 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
513
514 // Form canonically associated expression trees, and simplify the trees using
515 // basic mathematical properties. For example, this will form (nearly)
516 // minimal multiplication trees.
517 FPM.addPass(Pass: ReassociatePass());
518
519 // Add the primary loop simplification pipeline.
520 // FIXME: Currently this is split into two loop pass pipelines because we run
521 // some function passes in between them. These can and should be removed
522 // and/or replaced by scheduling the loop pass equivalents in the correct
523 // positions. But those equivalent passes aren't powerful enough yet.
524 // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
525 // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
526 // fully replace `SimplifyCFGPass`, and the closest to the other we have is
527 // `LoopInstSimplify`.
528 LoopPassManager LPM1, LPM2;
529
530 // Simplify the loop body. We do this initially to clean up after other loop
531 // passes run, either when iterating on a loop or on inner loops with
532 // implications on the outer loop.
533 LPM1.addPass(Pass: LoopInstSimplifyPass());
534 LPM1.addPass(Pass: LoopSimplifyCFGPass());
535
536 // Try to remove as much code from the loop header as possible,
537 // to reduce amount of IR that will have to be duplicated. However,
538 // do not perform speculative hoisting the first time as LICM
539 // will destroy metadata that may not need to be destroyed if run
540 // after loop rotation.
541 // TODO: Investigate promotion cap for O1.
542 LPM1.addPass(Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
543 /*AllowSpeculation=*/false));
544
545 LPM1.addPass(
546 Pass: LoopRotatePass(/*EnableHeaderDuplication=*/true, isLTOPreLink(Phase)));
547 // TODO: Investigate promotion cap for O1.
548 LPM1.addPass(Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
549 /*AllowSpeculation=*/true));
550 LPM1.addPass(Pass: SimpleLoopUnswitchPass());
551 if (EnableLoopFlatten)
552 LPM1.addPass(Pass: LoopFlattenPass());
553
554 LPM2.addPass(Pass: LoopIdiomRecognizePass());
555 LPM2.addPass(Pass: IndVarSimplifyPass());
556
557 invokeLateLoopOptimizationsEPCallbacks(LPM&: LPM2, Level);
558
559 LPM2.addPass(Pass: LoopDeletionPass());
560
561 // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
562 // because it changes IR to makes profile annotation in back compile
563 // inaccurate. The normal unroller doesn't pay attention to forced full unroll
564 // attributes so we need to make sure and allow the full unroll pass to pay
565 // attention to it.
566 if (!isThinLTOPreLink(Phase) || !PGOOpt ||
567 PGOOpt->Action != PGOOptions::SampleUse)
568 LPM2.addPass(Pass: LoopFullUnrollPass(static_cast<int>(Level),
569 /* OnlyWhenForced= */ !PTO.LoopUnrolling,
570 PTO.ForgetAllSCEVInLoopUnroll,
571 /* PrepareForLTO= */ isLTOPreLink(Phase)));
572
573 invokeLoopOptimizerEndEPCallbacks(LPM&: LPM2, Level);
574
575 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM1),
576 /*UseMemorySSA=*/true));
577 FPM.addPass(
578 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
579 FPM.addPass(Pass: InstCombinePass());
580 // The loop passes in LPM2 (LoopFullUnrollPass) do not preserve MemorySSA.
581 // *All* loop passes must preserve it, in order to be able to use it.
582 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM2),
583 /*UseMemorySSA=*/false));
584
585 // Delete small array after loop unroll.
586 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
587
588 // Specially optimize memory movement as it doesn't look like dataflow in SSA.
589 FPM.addPass(Pass: MemCpyOptPass());
590
591 // Sparse conditional constant propagation.
592 // FIXME: It isn't clear why we do this *after* loop passes rather than
593 // before...
594 FPM.addPass(Pass: SCCPPass());
595
596 // Delete dead bit computations (instcombine runs after to fold away the dead
597 // computations, and then ADCE will run later to exploit any new DCE
598 // opportunities that creates).
599 FPM.addPass(Pass: BDCEPass());
600
601 // Run instcombine after redundancy and dead bit elimination to exploit
602 // opportunities opened up by them.
603 FPM.addPass(Pass: InstCombinePass());
604 invokePeepholeEPCallbacks(FPM, Level);
605
606 FPM.addPass(Pass: CoroElidePass());
607
608 invokeScalarOptimizerLateEPCallbacks(FPM, Level);
609
610 // Finally, do an expensive DCE pass to catch all the dead code exposed by
611 // the simplifications and basic cleanup after all the simplifications.
612 // TODO: Investigate if this is too expensive.
613 FPM.addPass(Pass: ADCEPass());
614 FPM.addPass(
615 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
616 FPM.addPass(Pass: InstCombinePass());
617 invokePeepholeEPCallbacks(FPM, Level);
618
619 return FPM;
620}
621
622FunctionPassManager
623PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
624 ThinOrFullLTOPhase Phase) {
625 assert(Level != OptimizationLevel::O0 && "Must request optimizations!");
626
627 // The O1 pipeline has a separate pipeline creation function to simplify
628 // construction readability.
629 if (Level == OptimizationLevel::O1)
630 return buildO1FunctionSimplificationPipeline(Level, Phase);
631
632 FunctionPassManager FPM;
633
634 if (AreStatisticsEnabled())
635 FPM.addPass(Pass: CountVisitsPass());
636
637 // Form SSA out of local memory accesses after breaking apart aggregates into
638 // scalars.
639 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
640
641 // Catch trivial redundancies
642 FPM.addPass(Pass: EarlyCSEPass(true /* Enable mem-ssa. */));
643 if (EnableKnowledgeRetention)
644 FPM.addPass(Pass: AssumeSimplifyPass());
645
646 // Hoisting of scalars and load expressions.
647 if (EnableGVNHoist)
648 FPM.addPass(Pass: GVNHoistPass());
649
650 // Global value numbering based sinking.
651 if (EnableGVNSink) {
652 FPM.addPass(Pass: GVNSinkPass());
653 FPM.addPass(
654 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
655 }
656
657 // Speculative execution if the target has divergent branches; otherwise nop.
658 FPM.addPass(Pass: SpeculativeExecutionPass(/* OnlyIfDivergentTarget =*/true));
659
660 // Optimize based on known information about branches, and cleanup afterward.
661 FPM.addPass(Pass: JumpThreadingPass());
662 FPM.addPass(Pass: CorrelatedValuePropagationPass());
663
664 // Jump table to switch conversion.
665 if (EnableJumpTableToSwitch)
666 FPM.addPass(Pass: JumpTableToSwitchPass(/*InLTO=*/isLTOPostLink(Phase)));
667
668 FPM.addPass(
669 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
670 FPM.addPass(Pass: InstCombinePass());
671 FPM.addPass(Pass: AggressiveInstCombinePass());
672 FPM.addPass(Pass: LibCallsShrinkWrapPass());
673
674 invokePeepholeEPCallbacks(FPM, Level);
675
676 // For PGO use pipeline, try to optimize memory intrinsics such as memcpy
677 // using the size value profile. Don't perform this when optimizing for size.
678 if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse)
679 FPM.addPass(Pass: PGOMemOPSizeOpt());
680
681 FPM.addPass(Pass: TailCallElimPass(/*UpdateFunctionEntryCount=*/
682 isInstrumentedPGOUse()));
683 FPM.addPass(
684 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
685
686 // Form canonically associated expression trees, and simplify the trees using
687 // basic mathematical properties. For example, this will form (nearly)
688 // minimal multiplication trees.
689 FPM.addPass(Pass: ReassociatePass());
690
691 if (EnableConstraintElimination)
692 FPM.addPass(Pass: ConstraintEliminationPass());
693
694 // Add the primary loop simplification pipeline.
695 // FIXME: Currently this is split into two loop pass pipelines because we run
696 // some function passes in between them. These can and should be removed
697 // and/or replaced by scheduling the loop pass equivalents in the correct
698 // positions. But those equivalent passes aren't powerful enough yet.
699 // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
700 // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
701 // fully replace `SimplifyCFGPass`, and the closest to the other we have is
702 // `LoopInstSimplify`.
703 LoopPassManager LPM1, LPM2;
704
705 // Simplify the loop body. We do this initially to clean up after other loop
706 // passes run, either when iterating on a loop or on inner loops with
707 // implications on the outer loop.
708 LPM1.addPass(Pass: LoopInstSimplifyPass());
709 LPM1.addPass(Pass: LoopSimplifyCFGPass());
710
711 // Try to remove as much code from the loop header as possible,
712 // to reduce amount of IR that will have to be duplicated. However,
713 // do not perform speculative hoisting the first time as LICM
714 // will destroy metadata that may not need to be destroyed if run
715 // after loop rotation.
716 // TODO: Investigate promotion cap for O1.
717 LPM1.addPass(Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
718 /*AllowSpeculation=*/false));
719
720 LPM1.addPass(
721 Pass: LoopRotatePass(/*EnableHeaderDuplication=*/true, isLTOPreLink(Phase)));
722 // TODO: Investigate promotion cap for O1.
723 LPM1.addPass(Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
724 /*AllowSpeculation=*/true));
725 LPM1.addPass(
726 Pass: SimpleLoopUnswitchPass(/* NonTrivial */ Level == OptimizationLevel::O3));
727 if (EnableLoopFlatten)
728 LPM1.addPass(Pass: LoopFlattenPass());
729
730 LPM2.addPass(Pass: LoopIdiomRecognizePass());
731 LPM2.addPass(Pass: IndVarSimplifyPass());
732
733 {
734 ExtraLoopPassManager<ShouldRunExtraSimpleLoopUnswitch> ExtraPasses;
735 ExtraPasses.addPass(Pass: SimpleLoopUnswitchPass(/* NonTrivial */ Level ==
736 OptimizationLevel::O3));
737 LPM2.addPass(Pass: std::move(ExtraPasses));
738 }
739
740 invokeLateLoopOptimizationsEPCallbacks(LPM&: LPM2, Level);
741
742 LPM2.addPass(Pass: LoopDeletionPass());
743
744 // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
745 // because it changes IR to makes profile annotation in back compile
746 // inaccurate. The normal unroller doesn't pay attention to forced full unroll
747 // attributes so we need to make sure and allow the full unroll pass to pay
748 // attention to it.
749 if (!isThinLTOPreLink(Phase) || !PGOOpt ||
750 PGOOpt->Action != PGOOptions::SampleUse)
751 LPM2.addPass(Pass: LoopFullUnrollPass(static_cast<int>(Level),
752 /* OnlyWhenForced= */ !PTO.LoopUnrolling,
753 PTO.ForgetAllSCEVInLoopUnroll,
754 /* PrepareForLTO= */ isLTOPreLink(Phase)));
755
756 invokeLoopOptimizerEndEPCallbacks(LPM&: LPM2, Level);
757
758 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM1),
759 /*UseMemorySSA=*/true));
760 FPM.addPass(
761 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
762 FPM.addPass(Pass: InstCombinePass());
763 // The loop passes in LPM2 (LoopIdiomRecognizePass, IndVarSimplifyPass,
764 // LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA.
765 // *All* loop passes must preserve it, in order to be able to use it.
766 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM2),
767 /*UseMemorySSA=*/false));
768
769 // Delete small array after loop unroll.
770 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
771
772 // Try vectorization/scalarization transforms that are both improvements
773 // themselves and can allow further folds with GVN and InstCombine.
774 FPM.addPass(Pass: VectorCombinePass(/*TryEarlyFoldsOnly=*/true));
775
776 // Eliminate redundancies.
777 FPM.addPass(Pass: MergedLoadStoreMotionPass());
778 if (RunNewGVN)
779 FPM.addPass(Pass: NewGVNPass());
780 else
781 FPM.addPass(Pass: GVNPass());
782
783 // Sparse conditional constant propagation.
784 // FIXME: It isn't clear why we do this *after* loop passes rather than
785 // before...
786 FPM.addPass(Pass: SCCPPass());
787
788 // Delete dead bit computations (instcombine runs after to fold away the dead
789 // computations, and then ADCE will run later to exploit any new DCE
790 // opportunities that creates).
791 FPM.addPass(Pass: BDCEPass());
792
793 // Run instcombine after redundancy and dead bit elimination to exploit
794 // opportunities opened up by them.
795 FPM.addPass(Pass: InstCombinePass());
796 invokePeepholeEPCallbacks(FPM, Level);
797
798 // Re-consider control flow based optimizations after redundancy elimination,
799 // redo DCE, etc.
800 if (EnableDFAJumpThreading)
801 FPM.addPass(Pass: DFAJumpThreadingPass());
802
803 FPM.addPass(Pass: JumpThreadingPass());
804 FPM.addPass(Pass: CorrelatedValuePropagationPass());
805
806 // Finally, do an expensive DCE pass to catch all the dead code exposed by
807 // the simplifications and basic cleanup after all the simplifications.
808 // TODO: Investigate if this is too expensive.
809 FPM.addPass(Pass: ADCEPass());
810
811 // Specially optimize memory movement as it doesn't look like dataflow in SSA.
812 FPM.addPass(Pass: MemCpyOptPass());
813
814 FPM.addPass(Pass: DSEPass());
815 FPM.addPass(Pass: MoveAutoInitPass());
816
817 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(
818 Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
819 /*AllowSpeculation=*/true),
820 /*UseMemorySSA=*/true));
821
822 FPM.addPass(Pass: CoroElidePass());
823
824 invokeScalarOptimizerLateEPCallbacks(FPM, Level);
825
826 FPM.addPass(Pass: SimplifyCFGPass(SimplifyCFGOptions()
827 .convertSwitchRangeToICmp(B: true)
828 .convertSwitchToArithmetic(B: true)
829 .hoistCommonInsts(B: true)
830 .sinkCommonInsts(B: true)));
831 FPM.addPass(Pass: InstCombinePass());
832 invokePeepholeEPCallbacks(FPM, Level);
833
834 return FPM;
835}
836
837void PassBuilder::addRequiredLTOPreLinkPasses(ModulePassManager &MPM) {
838 MPM.addPass(Pass: CanonicalizeAliasesPass());
839 MPM.addPass(Pass: NameAnonGlobalPass());
840 MPM.addPass(Pass: AssignGUIDPass());
841}
842
843void PassBuilder::addPreInlinerPasses(ModulePassManager &MPM,
844 OptimizationLevel Level,
845 ThinOrFullLTOPhase LTOPhase) {
846 assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");
847 if (DisablePreInliner)
848 return;
849 InlineParams IP;
850
851 IP.DefaultThreshold = PreInlineThreshold;
852
853 // FIXME: The hint threshold has the same value used by the regular inliner
854 // when not optimzing for size. This should probably be lowered after
855 // performance testing.
856 // FIXME: this comment is cargo culted from the old pass manager, revisit).
857 IP.HintThreshold = 325;
858 IP.OptSizeHintThreshold = PreInlineThreshold;
859 ModuleInlinerWrapperPass MIWP(
860 IP, /* MandatoryFirst */ true,
861 InlineContext{.LTOPhase: LTOPhase, .Pass: InlinePass::EarlyInliner});
862 CGSCCPassManager &CGPipeline = MIWP.getPM();
863
864 FunctionPassManager FPM;
865 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
866 FPM.addPass(Pass: EarlyCSEPass()); // Catch trivial redundancies.
867 FPM.addPass(Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(
868 B: true))); // Merge & remove basic blocks.
869 FPM.addPass(Pass: InstCombinePass()); // Combine silly sequences.
870 invokePeepholeEPCallbacks(FPM, Level);
871
872 CGPipeline.addPass(Pass: createCGSCCToFunctionPassAdaptor(
873 Pass: std::move(FPM), EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
874
875 MPM.addPass(Pass: std::move(MIWP));
876
877 // Delete anything that is now dead to make sure that we don't instrument
878 // dead code. Instrumentation can end up keeping dead code around and
879 // dramatically increase code size.
880 MPM.addPass(Pass: GlobalDCEPass());
881}
882
883void PassBuilder::addPostPGOLoopRotation(ModulePassManager &MPM,
884 OptimizationLevel Level) {
885 if (EnablePostPGOLoopRotation) {
886 // Disable header duplication in loop rotation at -Oz.
887 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
888 Pass: createFunctionToLoopPassAdaptor(Pass: LoopRotatePass(),
889 /*UseMemorySSA=*/false),
890 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
891 }
892}
893
894void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM,
895 OptimizationLevel Level, bool RunProfileGen,
896 bool IsCS, bool AtomicCounterUpdate,
897 std::string ProfileFile,
898 std::string ProfileRemappingFile) {
899 assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");
900
901 if (!RunProfileGen) {
902 assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
903 MPM.addPass(
904 Pass: PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS, FS));
905 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
906 // RequireAnalysisPass for PSI before subsequent non-module passes.
907 MPM.addPass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
908 return;
909 }
910
911 // Perform PGO instrumentation.
912 MPM.addPass(Pass: PGOInstrumentationGen(IsCS ? PGOInstrumentationType::CSFDO
913 : PGOInstrumentationType::FDO));
914
915 addPostPGOLoopRotation(MPM, Level);
916 // Add the profile lowering pass.
917 InstrProfOptions Options;
918 if (!ProfileFile.empty())
919 Options.InstrProfileOutput = ProfileFile;
920 // Do counter promotion at Level greater than O0.
921 Options.DoCounterPromotion = true;
922 Options.UseBFIInPromotion = IsCS;
923 if (EnableSampledInstr) {
924 Options.Sampling = true;
925 // With sampling, there is little beneifit to enable counter promotion.
926 // But note that sampling does work with counter promotion.
927 Options.DoCounterPromotion = false;
928 }
929 Options.Atomic = AtomicCounterUpdate;
930 MPM.addPass(Pass: InstrProfilingLoweringPass(Options, IsCS));
931}
932
933void PassBuilder::addPGOInstrPassesForO0(ModulePassManager &MPM,
934 bool RunProfileGen, bool IsCS,
935 bool AtomicCounterUpdate,
936 std::string ProfileFile,
937 std::string ProfileRemappingFile) {
938 if (!RunProfileGen) {
939 assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
940 MPM.addPass(
941 Pass: PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS, FS));
942 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
943 // RequireAnalysisPass for PSI before subsequent non-module passes.
944 MPM.addPass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
945 return;
946 }
947
948 // Perform PGO instrumentation.
949 MPM.addPass(Pass: PGOInstrumentationGen(IsCS ? PGOInstrumentationType::CSFDO
950 : PGOInstrumentationType::FDO));
951 // Add the profile lowering pass.
952 InstrProfOptions Options;
953 if (!ProfileFile.empty())
954 Options.InstrProfileOutput = ProfileFile;
955 // Do not do counter promotion at O0.
956 Options.DoCounterPromotion = false;
957 Options.UseBFIInPromotion = IsCS;
958 Options.Atomic = AtomicCounterUpdate;
959 MPM.addPass(Pass: InstrProfilingLoweringPass(Options, IsCS));
960}
961
962static InlineParams getInlineParamsFromOptLevel(OptimizationLevel Level) {
963 return getInlineParamsFromOptLevel(OptLevel: static_cast<unsigned>(Level));
964}
965
966ModuleInlinerWrapperPass
967PassBuilder::buildInlinerPipeline(OptimizationLevel Level,
968 ThinOrFullLTOPhase Phase) {
969 InlineParams IP;
970 if (PTO.InlinerThreshold == -1)
971 IP = ::getInlineParamsFromOptLevel(Level);
972 else
973 IP = getInlineParams(Threshold: PTO.InlinerThreshold);
974 // For PreLinkThinLTO + SamplePGO or PreLinkFullLTO + SamplePGO,
975 // set hot-caller threshold to 0 to disable hot
976 // callsite inline (as much as possible [1]) because it makes
977 // profile annotation in the backend inaccurate.
978 //
979 // [1] Note the cost of a function could be below zero due to erased
980 // prologue / epilogue.
981 if (isLTOPreLink(Phase) && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse)
982 IP.HotCallSiteThreshold = 0;
983
984 if (PGOOpt)
985 IP.EnableDeferral = EnablePGOInlineDeferral;
986
987 ModuleInlinerWrapperPass MIWP(IP, PerformMandatoryInliningsFirst,
988 InlineContext{.LTOPhase: Phase, .Pass: InlinePass::CGSCCInliner},
989 UseInlineAdvisor, MaxDevirtIterations);
990
991 // Require the GlobalsAA analysis for the module so we can query it within
992 // the CGSCC pipeline.
993 if (EnableGlobalAnalyses) {
994 MIWP.addModulePass(Pass: RequireAnalysisPass<GlobalsAA, Module>());
995 // Invalidate AAManager so it can be recreated and pick up the newly
996 // available GlobalsAA.
997 MIWP.addModulePass(
998 Pass: createModuleToFunctionPassAdaptor(Pass: InvalidateAnalysisPass<AAManager>()));
999 }
1000
1001 // Require the ProfileSummaryAnalysis for the module so we can query it within
1002 // the inliner pass.
1003 MIWP.addModulePass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
1004
1005 // Now begin the main postorder CGSCC pipeline.
1006 // FIXME: The current CGSCC pipeline has its origins in the legacy pass
1007 // manager and trying to emulate its precise behavior. Much of this doesn't
1008 // make a lot of sense and we should revisit the core CGSCC structure.
1009 CGSCCPassManager &MainCGPipeline = MIWP.getPM();
1010
1011 // Note: historically, the PruneEH pass was run first to deduce nounwind and
1012 // generally clean up exception handling overhead. It isn't clear this is
1013 // valuable as the inliner doesn't currently care whether it is inlining an
1014 // invoke or a call.
1015
1016 if (AttributorRun & AttributorRunOption::CGSCC)
1017 MainCGPipeline.addPass(Pass: AttributorCGSCCPass());
1018 else if (AttributorRun & AttributorRunOption::CGSCC_LIGHT)
1019 MainCGPipeline.addPass(Pass: AttributorLightCGSCCPass());
1020
1021 // Deduce function attributes. We do another run of this after the function
1022 // simplification pipeline, so this only needs to run when it could affect the
1023 // function simplification pipeline, which is only the case with recursive
1024 // functions.
1025 MainCGPipeline.addPass(Pass: PostOrderFunctionAttrsPass(/*SkipNonRecursive*/ true));
1026
1027 // When at O3 add argument promotion to the pass pipeline.
1028 // FIXME: It isn't at all clear why this should be limited to O3.
1029 if (Level == OptimizationLevel::O3)
1030 MainCGPipeline.addPass(Pass: ArgumentPromotionPass());
1031
1032 // Try to perform OpenMP specific optimizations. This is a (quick!) no-op if
1033 // there are no OpenMP runtime calls present in the module.
1034 if (Level == OptimizationLevel::O2 || Level == OptimizationLevel::O3)
1035 MainCGPipeline.addPass(Pass: OpenMPOptCGSCCPass(Phase));
1036
1037 invokeCGSCCOptimizerLateEPCallbacks(CGPM&: MainCGPipeline, Level);
1038
1039 // Add the core function simplification pipeline nested inside the
1040 // CGSCC walk.
1041 MainCGPipeline.addPass(Pass: createCGSCCToFunctionPassAdaptor(
1042 Pass: buildFunctionSimplificationPipeline(Level, Phase),
1043 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses, /*NoRerun=*/true));
1044
1045 // Finally, deduce any function attributes based on the fully simplified
1046 // function.
1047 MainCGPipeline.addPass(Pass: PostOrderFunctionAttrsPass());
1048
1049 // Mark that the function is fully simplified and that it shouldn't be
1050 // simplified again if we somehow revisit it due to CGSCC mutations unless
1051 // it's been modified since.
1052 MainCGPipeline.addPass(Pass: createCGSCCToFunctionPassAdaptor(
1053 Pass: RequireAnalysisPass<ShouldNotRunFunctionPassesAnalysis, Function>()));
1054
1055 if (!isThinLTOPreLink(Phase)) {
1056 MainCGPipeline.addPass(Pass: CoroSplitPass(Level != OptimizationLevel::O0));
1057 MainCGPipeline.addPass(Pass: CoroAnnotationElidePass());
1058 }
1059
1060 // Make sure we don't affect potential future NoRerun CGSCC adaptors.
1061 MIWP.addLateModulePass(Pass: createModuleToFunctionPassAdaptor(
1062 Pass: InvalidateAnalysisPass<ShouldNotRunFunctionPassesAnalysis>()));
1063
1064 return MIWP;
1065}
1066
1067ModulePassManager
1068PassBuilder::buildModuleInlinerPipeline(OptimizationLevel Level,
1069 ThinOrFullLTOPhase Phase) {
1070 ModulePassManager MPM;
1071
1072 InlineParams IP = ::getInlineParamsFromOptLevel(Level);
1073 // For PreLinkThinLTO + SamplePGO or PreLinkFullLTO + SamplePGO,
1074 // set hot-caller threshold to 0 to disable hot
1075 // callsite inline (as much as possible [1]) because it makes
1076 // profile annotation in the backend inaccurate.
1077 //
1078 // [1] Note the cost of a function could be below zero due to erased
1079 // prologue / epilogue.
1080 if (isLTOPreLink(Phase) && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse)
1081 IP.HotCallSiteThreshold = 0;
1082
1083 if (PGOOpt)
1084 IP.EnableDeferral = EnablePGOInlineDeferral;
1085
1086 // The inline deferral logic is used to avoid losing some
1087 // inlining chance in future. It is helpful in SCC inliner, in which
1088 // inlining is processed in bottom-up order.
1089 // While in module inliner, the inlining order is a priority-based order
1090 // by default. The inline deferral is unnecessary there. So we disable the
1091 // inline deferral logic in module inliner.
1092 IP.EnableDeferral = false;
1093
1094 MPM.addPass(Pass: ModuleInlinerPass(IP, UseInlineAdvisor, Phase));
1095 if (!UseCtxProfile.empty() && Phase == ThinOrFullLTOPhase::ThinLTOPostLink) {
1096 MPM.addPass(Pass: GlobalOptPass());
1097 MPM.addPass(Pass: GlobalDCEPass());
1098 MPM.addPass(Pass: AssignGUIDPass());
1099 MPM.addPass(Pass: PGOCtxProfFlatteningPass(/*IsPreThinlink=*/false));
1100 }
1101
1102 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
1103 Pass: buildFunctionSimplificationPipeline(Level, Phase),
1104 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
1105
1106 if (!isThinLTOPreLink(Phase)) {
1107 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(
1108 Pass: CoroSplitPass(Level != OptimizationLevel::O0)));
1109 MPM.addPass(
1110 Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: CoroAnnotationElidePass()));
1111 }
1112
1113 return MPM;
1114}
1115
1116ModulePassManager
1117PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
1118 ThinOrFullLTOPhase Phase) {
1119 assert(Level != OptimizationLevel::O0 &&
1120 "Should not be used for O0 pipeline");
1121
1122 assert(!isFullLTOPostLink(Phase) &&
1123 "FullLTOPostLink shouldn't call buildModuleSimplificationPipeline!");
1124
1125 ModulePassManager MPM;
1126
1127 // Place pseudo probe instrumentation as the first pass of the pipeline to
1128 // minimize the impact of optimization changes.
1129 if (PGOOpt && PGOOpt->PseudoProbeForProfiling && !isThinLTOPostLink(Phase))
1130 MPM.addPass(Pass: SampleProfileProbePass(TM));
1131
1132 bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse);
1133
1134 // In ThinLTO mode, when flattened profile is used, all the available
1135 // profile information will be annotated in PreLink phase so there is
1136 // no need to load the profile again in PostLink.
1137 bool LoadSampleProfile =
1138 HasSampleProfile && !(FlattenedProfileUsed && isThinLTOPostLink(Phase));
1139
1140 // During the ThinLTO backend phase we perform early indirect call promotion
1141 // here, before globalopt. Otherwise imported available_externally functions
1142 // look unreferenced and are removed. If we are going to load the sample
1143 // profile then defer until later.
1144 // TODO: See if we can move later and consolidate with the location where
1145 // we perform ICP when we are loading a sample profile.
1146 // TODO: We pass HasSampleProfile (whether there was a sample profile file
1147 // passed to the compile) to the SamplePGO flag of ICP. This is used to
1148 // determine whether the new direct calls are annotated with prof metadata.
1149 // Ideally this should be determined from whether the IR is annotated with
1150 // sample profile, and not whether the a sample profile was provided on the
1151 // command line. E.g. for flattened profiles where we will not be reloading
1152 // the sample profile in the ThinLTO backend, we ideally shouldn't have to
1153 // provide the sample profile file.
1154 if (isThinLTOPostLink(Phase) && !LoadSampleProfile)
1155 MPM.addPass(Pass: PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile));
1156
1157 // Create an early function pass manager to cleanup the output of the
1158 // frontend. Not necessary with LTO post link pipelines since the pre link
1159 // pipeline already cleaned up the frontend output.
1160 if (!isThinLTOPostLink(Phase)) {
1161 // Do basic inference of function attributes from known properties of system
1162 // libraries and other oracles.
1163 MPM.addPass(Pass: InferFunctionAttrsPass());
1164 MPM.addPass(Pass: CoroEarlyPass());
1165
1166 FunctionPassManager EarlyFPM;
1167 EarlyFPM.addPass(Pass: EntryExitInstrumenterPass(/*PostInlining=*/false));
1168 // Lower llvm.expect to metadata before attempting transforms.
1169 // Compare/branch metadata may alter the behavior of passes like
1170 // SimplifyCFG.
1171 EarlyFPM.addPass(Pass: LowerExpectIntrinsicPass());
1172 EarlyFPM.addPass(Pass: SimplifyCFGPass());
1173 EarlyFPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
1174 EarlyFPM.addPass(Pass: EarlyCSEPass());
1175 if (Level == OptimizationLevel::O3)
1176 EarlyFPM.addPass(Pass: CallSiteSplittingPass());
1177 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
1178 Pass: std::move(EarlyFPM), EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
1179 }
1180
1181 if (LoadSampleProfile) {
1182 // Annotate sample profile right after early FPM to ensure freshness of
1183 // the debug info.
1184 MPM.addPass(Pass: SampleProfileLoaderPass(
1185 PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile, Phase, FS));
1186 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1187 // RequireAnalysisPass for PSI before subsequent non-module passes.
1188 MPM.addPass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
1189 // Do not invoke ICP in the LTOPrelink phase as it makes it hard
1190 // for the profile annotation to be accurate in the LTO backend.
1191 if (!isLTOPreLink(Phase))
1192 // We perform early indirect call promotion here, before globalopt.
1193 // This is important for the ThinLTO backend phase because otherwise
1194 // imported available_externally functions look unreferenced and are
1195 // removed.
1196 MPM.addPass(
1197 Pass: PGOIndirectCallPromotion(true /* IsInLTO */, true /* SamplePGO */));
1198 }
1199
1200 // Try to perform OpenMP specific optimizations on the module. This is a
1201 // (quick!) no-op if there are no OpenMP runtime calls present in the module.
1202 MPM.addPass(Pass: OpenMPOptPass(Phase));
1203
1204 if (AttributorRun & AttributorRunOption::MODULE)
1205 MPM.addPass(Pass: AttributorPass());
1206 else if (AttributorRun & AttributorRunOption::MODULE_LIGHT)
1207 MPM.addPass(Pass: AttributorLightPass());
1208
1209 // Lower type metadata and the type.test intrinsic in the ThinLTO
1210 // post link pipeline after ICP. This is to enable usage of the type
1211 // tests in ICP sequences.
1212 if (isThinLTOPostLink(Phase))
1213 MPM.addPass(Pass: DropTypeTestsPass());
1214
1215 invokePipelineEarlySimplificationEPCallbacks(MPM, Level, Phase);
1216
1217 // Interprocedural constant propagation now that basic cleanup has occurred
1218 // and prior to optimizing globals.
1219 // FIXME: This position in the pipeline hasn't been carefully considered in
1220 // years, it should be re-analyzed.
1221 MPM.addPass(
1222 Pass: IPSCCPPass(IPSCCPOptions(/*AllowFuncSpec=*/!isLTOPreLink(Phase))));
1223
1224 // Attach metadata to indirect call sites indicating the set of functions
1225 // they may target at run-time. This should follow IPSCCP.
1226 MPM.addPass(Pass: CalledValuePropagationPass());
1227
1228 // Optimize globals to try and fold them into constants.
1229 MPM.addPass(Pass: GlobalOptPass());
1230
1231 // Create a small function pass pipeline to cleanup after all the global
1232 // optimizations.
1233 FunctionPassManager GlobalCleanupPM;
1234 // FIXME: Should this instead by a run of SROA?
1235 GlobalCleanupPM.addPass(Pass: PromotePass());
1236 GlobalCleanupPM.addPass(Pass: InstCombinePass());
1237 invokePeepholeEPCallbacks(FPM&: GlobalCleanupPM, Level);
1238 GlobalCleanupPM.addPass(
1239 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
1240 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(GlobalCleanupPM),
1241 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
1242
1243 // We already asserted this happens in non-FullLTOPostLink earlier.
1244 const bool IsPreLink = !isThinLTOPostLink(Phase);
1245 // Enable contextual profiling instrumentation.
1246 const bool IsCtxProfGen =
1247 IsPreLink && PGOCtxProfLoweringPass::isCtxIRPGOInstrEnabled();
1248 const bool IsPGOPreLink = !IsCtxProfGen && PGOOpt && IsPreLink;
1249 const bool IsPGOInstrGen =
1250 IsPGOPreLink && PGOOpt->Action == PGOOptions::IRInstr;
1251 const bool IsPGOInstrUse =
1252 IsPGOPreLink && PGOOpt->Action == PGOOptions::IRUse;
1253 const bool IsMemprofUse = IsPGOPreLink && !PGOOpt->MemoryProfile.empty();
1254 // We don't want to mix pgo ctx gen and pgo gen; we also don't currently
1255 // enable ctx profiling from the frontend.
1256 assert(!(IsPGOInstrGen && PGOCtxProfLoweringPass::isCtxIRPGOInstrEnabled()) &&
1257 "Enabling both instrumented PGO and contextual instrumentation is not "
1258 "supported.");
1259 const bool IsCtxProfUse = !UseCtxProfile.empty() && isThinLTOPreLink(Phase);
1260
1261 assert(
1262 (InstrumentColdFuncOnlyPath.empty() || PGOInstrumentColdFunctionOnly) &&
1263 "--instrument-cold-function-only-path is provided but "
1264 "--pgo-instrument-cold-function-only is not enabled");
1265 const bool IsColdFuncOnlyInstrGen = PGOInstrumentColdFunctionOnly &&
1266 IsPGOPreLink &&
1267 !InstrumentColdFuncOnlyPath.empty();
1268
1269 if (IsPGOInstrGen || IsPGOInstrUse || IsMemprofUse || IsCtxProfGen ||
1270 IsCtxProfUse || IsColdFuncOnlyInstrGen)
1271 addPreInlinerPasses(MPM, Level, LTOPhase: Phase);
1272
1273 // Add all the requested passes for instrumentation PGO, if requested.
1274 if (IsPGOInstrGen || IsPGOInstrUse) {
1275 addPGOInstrPasses(MPM, Level,
1276 /*RunProfileGen=*/IsPGOInstrGen,
1277 /*IsCS=*/false, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate,
1278 ProfileFile: PGOOpt->ProfileFile, ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
1279 } else if (IsCtxProfGen || IsCtxProfUse) {
1280 MPM.addPass(Pass: PGOInstrumentationGen(PGOInstrumentationType::CTXPROF));
1281 // In pre-link, we just want the instrumented IR. We use the contextual
1282 // profile in the post-thinlink phase.
1283 // The instrumentation will be removed in post-thinlink after IPO.
1284 if (IsCtxProfUse) {
1285 MPM.addPass(Pass: AssignGUIDPass());
1286 MPM.addPass(Pass: PGOCtxProfFlatteningPass(/*IsPreThinlink=*/true));
1287 return MPM;
1288 }
1289 // Block further inlining in the instrumented ctxprof case. This avoids
1290 // confusingly collecting profiles for the same GUID corresponding to
1291 // different variants of the function. We could do like PGO and identify
1292 // functions by a (GUID, Hash) tuple, but since the ctxprof "use" waits for
1293 // thinlto to happen before performing any further optimizations, it's
1294 // unnecessary to collect profiles for non-prevailing copies.
1295 MPM.addPass(Pass: NoinlineNonPrevailing());
1296 addPostPGOLoopRotation(MPM, Level);
1297 MPM.addPass(Pass: AssignGUIDPass());
1298 MPM.addPass(Pass: PGOCtxProfLoweringPass());
1299 } else if (IsColdFuncOnlyInstrGen) {
1300 addPGOInstrPasses(MPM, Level, /* RunProfileGen */ true, /* IsCS */ false,
1301 /* AtomicCounterUpdate */ false,
1302 ProfileFile: InstrumentColdFuncOnlyPath,
1303 /* ProfileRemappingFile */ "");
1304 }
1305
1306 if (IsPGOInstrGen || IsPGOInstrUse || IsCtxProfGen)
1307 MPM.addPass(Pass: PGOIndirectCallPromotion(false, false));
1308
1309 if (IsPGOPreLink && PGOOpt->CSAction == PGOOptions::CSIRInstr)
1310 MPM.addPass(Pass: PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile,
1311 EnableSampledInstr));
1312
1313 if (IsMemprofUse)
1314 MPM.addPass(Pass: MemProfUsePass(PGOOpt->MemoryProfile, FS));
1315
1316 if (PGOOpt && (PGOOpt->Action == PGOOptions::IRUse ||
1317 PGOOpt->Action == PGOOptions::SampleUse))
1318 MPM.addPass(Pass: PGOForceFunctionAttrsPass(PGOOpt->ColdOptType));
1319
1320 MPM.addPass(Pass: AlwaysInlinerPass(/*InsertLifetimeIntrinsics=*/true));
1321
1322 if (EnableModuleInliner)
1323 MPM.addPass(Pass: buildModuleInlinerPipeline(Level, Phase));
1324 else
1325 MPM.addPass(Pass: buildInlinerPipeline(Level, Phase));
1326
1327 // Remove any dead arguments exposed by cleanups, constant folding globals,
1328 // and argument promotion.
1329 MPM.addPass(Pass: DeadArgumentEliminationPass());
1330
1331 if (isThinLTOPostLink(Phase))
1332 MPM.addPass(Pass: SimplifyTypeTestsPass());
1333
1334 if (!isThinLTOPreLink(Phase))
1335 MPM.addPass(Pass: CoroCleanupPass());
1336
1337 // Optimize globals now that functions are fully simplified.
1338 MPM.addPass(Pass: GlobalOptPass());
1339 MPM.addPass(Pass: GlobalDCEPass());
1340
1341 return MPM;
1342}
1343
1344/// TODO: Should LTO cause any differences to this set of passes?
1345void PassBuilder::addVectorPasses(OptimizationLevel Level,
1346 FunctionPassManager &FPM,
1347 ThinOrFullLTOPhase LTOPhase) {
1348 FPM.addPass(Pass: LoopVectorizePass(
1349 LoopVectorizeOptions(!PTO.LoopInterleaving, !PTO.LoopVectorization)));
1350
1351 // Drop dereferenceable assumes after vectorization, as they are no longer
1352 // needed and can inhibit further optimization.
1353 if (!isLTOPreLink(Phase: LTOPhase))
1354 FPM.addPass(Pass: DropUnnecessaryAssumesPass(/*DropDereferenceable=*/true));
1355
1356 FPM.addPass(Pass: InferAlignmentPass());
1357 if (isFullLTOPostLink(Phase: LTOPhase)) {
1358 // The vectorizer may have significantly shortened a loop body; unroll
1359 // again. Unroll small loops to hide loop backedge latency and saturate any
1360 // parallel execution resources of an out-of-order processor. We also then
1361 // need to clean up redundancies and loop invariant code.
1362 // FIXME: It would be really good to use a loop-integrated instruction
1363 // combiner for cleanup here so that the unrolling and LICM can be pipelined
1364 // across the loop nests.
1365 // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1366 if (EnableUnrollAndJam && PTO.LoopUnrolling)
1367 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(
1368 Pass: LoopUnrollAndJamPass(static_cast<int>(Level))));
1369 FPM.addPass(Pass: LoopUnrollPass(LoopUnrollOptions(
1370 static_cast<int>(Level), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1371 PTO.ForgetAllSCEVInLoopUnroll)));
1372 FPM.addPass(Pass: WarnMissedTransformationsPass());
1373 // Now that we are done with loop unrolling, be it either by LoopVectorizer,
1374 // or LoopUnroll passes, some variable-offset GEP's into alloca's could have
1375 // become constant-offset, thus enabling SROA and alloca promotion. Do so.
1376 // NOTE: we are very late in the pipeline, and we don't have any LICM
1377 // or SimplifyCFG passes scheduled after us, that would cleanup
1378 // the CFG mess this may created if allowed to modify CFG, so forbid that.
1379
1380 // We also turn on struct to vector canonicalization here, which allows
1381 // converting allocas of homogeneous structs into vector allocas when the
1382 // allocas' users are all memory intrinsics. This allows promotion in some
1383 // cases because structs cannot promote to SSA values, but vectors can. We
1384 // only turn this on after memcpyopt runs because this might hinder
1385 // memcpyopt's optimizations if done before. Look at the documentation for
1386 // `tryCanonicalizeStructToVector` in SROA.cpp to see why.
1387 FPM.addPass(Pass: SROAPass(SROAOptions(SROAOptions::PreserveCFG,
1388 /*AggregateToVector=*/true)));
1389 }
1390
1391 if (!isFullLTOPostLink(Phase: LTOPhase)) {
1392 // Eliminate loads by forwarding stores from the previous iteration to loads
1393 // of the current iteration.
1394 FPM.addPass(Pass: LoopLoadEliminationPass());
1395 }
1396 // Cleanup after the loop optimization passes.
1397 FPM.addPass(Pass: InstCombinePass());
1398
1399 if (Level > OptimizationLevel::O1 && ExtraVectorizerPasses) {
1400 ExtraFunctionPassManager<ShouldRunExtraVectorPasses> ExtraPasses;
1401 // At higher optimization levels, try to clean up any runtime overlap and
1402 // alignment checks inserted by the vectorizer. We want to track correlated
1403 // runtime checks for two inner loops in the same outer loop, fold any
1404 // common computations, hoist loop-invariant aspects out of any outer loop,
1405 // and unswitch the runtime checks if possible. Once hoisted, we may have
1406 // dead (or speculatable) control flows or more combining opportunities.
1407 ExtraPasses.addPass(Pass: EarlyCSEPass());
1408 ExtraPasses.addPass(Pass: CorrelatedValuePropagationPass());
1409 ExtraPasses.addPass(Pass: InstCombinePass());
1410 LoopPassManager LPM;
1411 LPM.addPass(Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
1412 /*AllowSpeculation=*/true));
1413 LPM.addPass(Pass: SimpleLoopUnswitchPass(/* NonTrivial */ Level ==
1414 OptimizationLevel::O3));
1415 ExtraPasses.addPass(
1416 Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM), /*UseMemorySSA=*/true));
1417 ExtraPasses.addPass(
1418 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
1419 ExtraPasses.addPass(Pass: InstCombinePass());
1420 FPM.addPass(Pass: std::move(ExtraPasses));
1421 }
1422
1423 // Now that we've formed fast to execute loop structures, we do further
1424 // optimizations. These are run afterward as they might block doing complex
1425 // analyses and transforms such as what are needed for loop vectorization.
1426
1427 // Cleanup after loop vectorization, etc. Simplification passes like CVP and
1428 // GVN, loop transforms, and others have already run, so it's now better to
1429 // convert to more optimized IR using more aggressive simplify CFG options.
1430 // The extra sinking transform can create larger basic blocks, so do this
1431 // before SLP vectorization.
1432 FPM.addPass(Pass: SimplifyCFGPass(SimplifyCFGOptions()
1433 .forwardSwitchCondToPhi(B: true)
1434 .convertSwitchRangeToICmp(B: true)
1435 .convertSwitchToArithmetic(B: true)
1436 .convertSwitchToLookupTable(B: true)
1437 .needCanonicalLoops(B: false)
1438 .hoistCommonInsts(B: true)
1439 .sinkCommonInsts(B: true)));
1440
1441 if (isFullLTOPostLink(Phase: LTOPhase)) {
1442 FPM.addPass(Pass: SCCPPass());
1443 FPM.addPass(Pass: InstCombinePass());
1444 FPM.addPass(Pass: BDCEPass());
1445 }
1446
1447 // Optimize parallel scalar instruction chains into SIMD instructions.
1448 if (PTO.SLPVectorization) {
1449 FPM.addPass(Pass: SLPVectorizerPass());
1450 if (Level >= OptimizationLevel::O2 && ExtraVectorizerPasses) {
1451 FPM.addPass(Pass: EarlyCSEPass());
1452 }
1453 }
1454 // Enhance/cleanup vector code.
1455 FPM.addPass(Pass: VectorCombinePass());
1456
1457 if (!isFullLTOPostLink(Phase: LTOPhase)) {
1458 FPM.addPass(Pass: InstCombinePass());
1459 // Unroll small loops to hide loop backedge latency and saturate any
1460 // parallel execution resources of an out-of-order processor. We also then
1461 // need to clean up redundancies and loop invariant code.
1462 // FIXME: It would be really good to use a loop-integrated instruction
1463 // combiner for cleanup here so that the unrolling and LICM can be pipelined
1464 // across the loop nests.
1465 // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1466 if (EnableUnrollAndJam && PTO.LoopUnrolling) {
1467 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(
1468 Pass: LoopUnrollAndJamPass(static_cast<int>(Level))));
1469 }
1470 FPM.addPass(Pass: LoopUnrollPass(LoopUnrollOptions(
1471 static_cast<int>(Level), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1472 PTO.ForgetAllSCEVInLoopUnroll)));
1473 FPM.addPass(Pass: WarnMissedTransformationsPass());
1474 // Now that we are done with loop unrolling, be it either by LoopVectorizer,
1475 // or LoopUnroll passes, some variable-offset GEP's into alloca's could have
1476 // become constant-offset, thus enabling SROA and alloca promotion. Do so.
1477 // NOTE: we are very late in the pipeline, and we don't have any LICM
1478 // or SimplifyCFG passes scheduled after us, that would cleanup
1479 // the CFG mess this may created if allowed to modify CFG, so forbid that.
1480
1481 // We also turn on struct to vector canonicalization here, which allows
1482 // converting allocas of homogeneous structs into vector allocas when the
1483 // allocas' users are all memory intrinsics. This allows promotion in some
1484 // cases because structs cannot promote to SSA values, but vectors can. We
1485 // only turn this on after memcpyopt runs because this might hinder
1486 // memcpyopt's optimizations if done before. Look at the documentation for
1487 // `tryCanonicalizeStructToVector` in SROA.cpp to see why.
1488 FPM.addPass(Pass: SROAPass(SROAOptions(SROAOptions::PreserveCFG,
1489 /*AggregateToVector=*/true)));
1490 }
1491
1492 FPM.addPass(Pass: InferAlignmentPass());
1493 FPM.addPass(Pass: InstCombinePass());
1494
1495 // This is needed for two reasons:
1496 // 1. It works around problems that instcombine introduces, such as sinking
1497 // expensive FP divides into loops containing multiplications using the
1498 // divide result.
1499 // 2. It helps to clean up some loop-invariant code created by the loop
1500 // unroll pass when IsFullLTO=false.
1501 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(
1502 Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
1503 /*AllowSpeculation=*/true),
1504 /*UseMemorySSA=*/true));
1505
1506 // Now that we've vectorized and unrolled loops, we may have more refined
1507 // alignment information, try to re-derive it here.
1508 FPM.addPass(Pass: AlignmentFromAssumptionsPass());
1509}
1510
1511ModulePassManager
1512PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
1513 ThinOrFullLTOPhase LTOPhase) {
1514 ModulePassManager MPM;
1515
1516 // Run partial inlining pass to partially inline functions that have
1517 // large bodies.
1518 if (RunPartialInlining)
1519 MPM.addPass(Pass: PartialInlinerPass());
1520
1521 // Remove avail extern fns and globals definitions since we aren't compiling
1522 // an object file for later LTO. For LTO we want to preserve these so they
1523 // are eligible for inlining at link-time. Note if they are unreferenced they
1524 // will be removed by GlobalDCE later, so this only impacts referenced
1525 // available externally globals. Eventually they will be suppressed during
1526 // codegen, but eliminating here enables more opportunity for GlobalDCE as it
1527 // may make globals referenced by available external functions dead and saves
1528 // running remaining passes on the eliminated functions. These should be
1529 // preserved during prelinking for link-time inlining decisions.
1530 if (!isLTOPreLink(Phase: LTOPhase))
1531 MPM.addPass(Pass: EliminateAvailableExternallyPass());
1532
1533 // Do RPO function attribute inference across the module to forward-propagate
1534 // attributes where applicable.
1535 // FIXME: Is this really an optimization rather than a canonicalization?
1536 MPM.addPass(Pass: ReversePostOrderFunctionAttrsPass());
1537
1538 // Do a post inline PGO instrumentation and use pass. This is a context
1539 // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as
1540 // cross-module inline has not been done yet. The context sensitive
1541 // instrumentation is after all the inlines are done.
1542 if (!isLTOPreLink(Phase: LTOPhase) && PGOOpt) {
1543 if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1544 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/true,
1545 /*IsCS=*/true, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate,
1546 ProfileFile: PGOOpt->CSProfileGenFile, ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
1547 else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1548 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/false,
1549 /*IsCS=*/true, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate,
1550 ProfileFile: PGOOpt->ProfileFile, ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
1551 }
1552
1553 // Re-compute GlobalsAA here prior to function passes. This is particularly
1554 // useful as the above will have inlined, DCE'ed, and function-attr
1555 // propagated everything. We should at this point have a reasonably minimal
1556 // and richly annotated call graph. By computing aliasing and mod/ref
1557 // information for all local globals here, the late loop passes and notably
1558 // the vectorizer will be able to use them to help recognize vectorizable
1559 // memory operations.
1560 if (EnableGlobalAnalyses)
1561 MPM.addPass(Pass: RecomputeGlobalsAAPass());
1562
1563 invokeOptimizerEarlyEPCallbacks(MPM, Level, Phase: LTOPhase);
1564
1565 FunctionPassManager OptimizePM;
1566
1567 // Only drop unnecessary assumes post-inline and post-link, as otherwise
1568 // additional uses of the affected value may be introduced through inlining
1569 // and CSE.
1570 if (!isLTOPreLink(Phase: LTOPhase))
1571 OptimizePM.addPass(Pass: DropUnnecessaryAssumesPass());
1572
1573 // Scheduling LoopVersioningLICM when inlining is over, because after that
1574 // we may see more accurate aliasing. Reason to run this late is that too
1575 // early versioning may prevent further inlining due to increase of code
1576 // size. Other optimizations which runs later might get benefit of no-alias
1577 // assumption in clone loop.
1578 if (UseLoopVersioningLICM) {
1579 OptimizePM.addPass(
1580 Pass: createFunctionToLoopPassAdaptor(Pass: LoopVersioningLICMPass()));
1581 // LoopVersioningLICM pass might increase new LICM opportunities.
1582 OptimizePM.addPass(Pass: createFunctionToLoopPassAdaptor(
1583 Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
1584 /*AllowSpeculation=*/true),
1585 /*USeMemorySSA=*/UseMemorySSA: true));
1586 }
1587
1588 OptimizePM.addPass(Pass: Float2IntPass());
1589 OptimizePM.addPass(Pass: LowerConstantIntrinsicsPass());
1590
1591 if (EnableMatrix) {
1592 OptimizePM.addPass(Pass: LowerMatrixIntrinsicsPass());
1593 OptimizePM.addPass(Pass: EarlyCSEPass());
1594 }
1595
1596 // CHR pass should only be applied with the profile information.
1597 // The check is to check the profile summary information in CHR.
1598 if (EnableCHR && Level == OptimizationLevel::O3)
1599 OptimizePM.addPass(Pass: ControlHeightReductionPass());
1600
1601 // FIXME: We need to run some loop optimizations to re-rotate loops after
1602 // simplifycfg and others undo their rotation.
1603
1604 // Optimize the loop execution. These passes operate on entire loop nests
1605 // rather than on each loop in an inside-out manner, and so they are actually
1606 // function passes.
1607
1608 invokeVectorizerStartEPCallbacks(FPM&: OptimizePM, Level);
1609
1610 LoopPassManager LPM;
1611 // First rotate loops that may have been un-rotated by prior passes.
1612 // Disable header duplication at -Oz.
1613 LPM.addPass(Pass: LoopRotatePass(/*EnableLoopHeaderDuplication=*/true,
1614 isLTOPreLink(Phase: LTOPhase),
1615 /*CheckExitCount=*/true));
1616 // Some loops may have become dead by now. Try to delete them.
1617 // FIXME: see discussion in https://reviews.llvm.org/D112851,
1618 // this may need to be revisited once we run GVN before loop deletion
1619 // in the simplification pipeline.
1620 LPM.addPass(Pass: LoopDeletionPass());
1621
1622 if (PTO.LoopInterchange)
1623 LPM.addPass(Pass: LoopInterchangePass());
1624
1625 OptimizePM.addPass(
1626 Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM), /*UseMemorySSA=*/false));
1627
1628 // FIXME: This may not be the right place in the pipeline.
1629 // We need to have the data to support the right place.
1630 if (PTO.LoopFusion)
1631 OptimizePM.addPass(Pass: LoopFusePass());
1632
1633 // Distribute loops to allow partial vectorization. I.e. isolate dependences
1634 // into separate loop that would otherwise inhibit vectorization. This is
1635 // currently only performed for loops marked with the metadata
1636 // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
1637 OptimizePM.addPass(Pass: LoopDistributePass());
1638
1639 // Populates the VFABI attribute with the scalar-to-vector mappings
1640 // from the TargetLibraryInfo.
1641 OptimizePM.addPass(Pass: InjectTLIMappings());
1642
1643 addVectorPasses(Level, FPM&: OptimizePM, LTOPhase);
1644
1645 invokeVectorizerEndEPCallbacks(FPM&: OptimizePM, Level);
1646
1647 // LoopSink pass sinks instructions hoisted by LICM, which serves as a
1648 // canonicalization pass that enables other optimizations. As a result,
1649 // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
1650 // result too early.
1651 OptimizePM.addPass(Pass: LoopSinkPass());
1652
1653 // And finally clean up LCSSA form before generating code.
1654 OptimizePM.addPass(Pass: InstSimplifyPass());
1655
1656 // This hoists/decomposes div/rem ops. It should run after other sink/hoist
1657 // passes to avoid re-sinking, but before SimplifyCFG because it can allow
1658 // flattening of blocks.
1659 OptimizePM.addPass(Pass: DivRemPairsPass());
1660
1661 // Merge adjacent icmps into memcmp, then expand memcmp to loads/compares.
1662 // TODO: move this furter up so that it can be optimized by GVN, etc.
1663 if (EnableMergeICmps)
1664 OptimizePM.addPass(Pass: MergeICmpsPass());
1665 OptimizePM.addPass(Pass: ExpandMemCmpPass());
1666
1667 // Try to annotate calls that were created during optimization.
1668 OptimizePM.addPass(
1669 Pass: TailCallElimPass(/*UpdateFunctionEntryCount=*/isInstrumentedPGOUse()));
1670
1671 // LoopSink (and other loop passes since the last simplifyCFG) might have
1672 // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
1673 OptimizePM.addPass(
1674 Pass: SimplifyCFGPass(SimplifyCFGOptions()
1675 .convertSwitchRangeToICmp(B: true)
1676 .convertSwitchToArithmetic(B: true)
1677 .speculateUnpredictables(B: true)
1678 .hoistLoadsStoresWithCondFaulting(B: true)));
1679
1680 // Add the core optimizing pipeline.
1681 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(OptimizePM),
1682 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
1683
1684 // AllocToken transforms heap allocation calls; this needs to run late after
1685 // other allocation call transformations (such as those in InstCombine).
1686 if (!isLTOPreLink(Phase: LTOPhase))
1687 MPM.addPass(Pass: AllocTokenPass());
1688
1689 invokeOptimizerLastEPCallbacks(MPM, Level, Phase: LTOPhase);
1690
1691 // Run the Instrumentor pass late.
1692 if (EnableInstrumentor)
1693 MPM.addPass(Pass: InstrumentorPass(FS));
1694
1695 // Split out cold code. Splitting is done late to avoid hiding context from
1696 // other optimizations and inadvertently regressing performance. The tradeoff
1697 // is that this has a higher code size cost than splitting early.
1698 if (EnableHotColdSplit && !isLTOPreLink(Phase: LTOPhase))
1699 MPM.addPass(Pass: HotColdSplittingPass());
1700
1701 // Now we need to do some global optimization transforms.
1702 // FIXME: It would seem like these should come first in the optimization
1703 // pipeline and maybe be the bottom of the canonicalization pipeline? Weird
1704 // ordering here.
1705 MPM.addPass(Pass: GlobalDCEPass());
1706 MPM.addPass(Pass: ConstantMergePass());
1707
1708 // Merge functions if requested. It has a better chance to merge functions
1709 // after ConstantMerge folded jump tables.
1710 if (PTO.MergeFunctions)
1711 MPM.addPass(Pass: MergeFunctionsPass());
1712
1713 if (PTO.CallGraphProfile && !isLTOPreLink(Phase: LTOPhase))
1714 MPM.addPass(Pass: CGProfilePass(isLTOPostLink(Phase: LTOPhase)));
1715
1716 // RelLookupTableConverterPass runs later in LTO post-link pipeline.
1717 if (!isLTOPreLink(Phase: LTOPhase))
1718 MPM.addPass(Pass: RelLookupTableConverterPass());
1719
1720 // Add devirtualization pass only when LTO is not enabled, as otherwise
1721 // the pass is already enabled in the LTO pipeline.
1722 if (PTO.DevirtualizeSpeculatively && LTOPhase == ThinOrFullLTOPhase::None) {
1723 // TODO: explore a better pipeline configuration that can improve
1724 // compilation time overhead.
1725 // FIXME: move this earlier (lots of pass ordering tests will need fixing)
1726 MPM.addPass(Pass: AssignGUIDPass());
1727 MPM.addPass(Pass: WholeProgramDevirtPass(
1728 /*ExportSummary*/ nullptr,
1729 /*ImportSummary*/ nullptr,
1730 /*DevirtSpeculatively*/ PTO.DevirtualizeSpeculatively));
1731 MPM.addPass(Pass: DropTypeTestsPass());
1732 // Given that the devirtualization creates more opportunities for inlining,
1733 // we run the Inliner again here to maximize the optimization gain we
1734 // get from devirtualization.
1735 // Also, we can't run devirtualization before inlining because the
1736 // devirtualization depends on the passes optimizing/eliminating vtable GVs
1737 // and those passes are only effective after inlining.
1738 if (EnableModuleInliner) {
1739 MPM.addPass(Pass: ModuleInlinerPass(::getInlineParamsFromOptLevel(Level),
1740 UseInlineAdvisor,
1741 ThinOrFullLTOPhase::None));
1742 } else {
1743 MPM.addPass(Pass: ModuleInlinerWrapperPass(
1744 ::getInlineParamsFromOptLevel(Level),
1745 /* MandatoryFirst */ true,
1746 InlineContext{.LTOPhase: ThinOrFullLTOPhase::None, .Pass: InlinePass::CGSCCInliner}));
1747 }
1748 }
1749
1750 // Attach !implicit.ref metadata from all functions to copyright strings.
1751 MPM.addPass(Pass: LowerCommentStringPass());
1752
1753 return MPM;
1754}
1755
1756ModulePassManager
1757PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level,
1758 ThinOrFullLTOPhase Phase) {
1759 if (Level == OptimizationLevel::O0)
1760 return buildO0DefaultPipeline(Level, Phase);
1761
1762 ModulePassManager MPM;
1763 instructionCountersPass(MPM, /* IsPreOptimization */ true);
1764 // Currently this pipeline is only invoked in an LTO pre link pass or when we
1765 // are not running LTO. If that changes the below checks may need updating.
1766 assert(isLTOPreLink(Phase) || Phase == ThinOrFullLTOPhase::None);
1767
1768 // If we are invoking this in non-LTO mode, remove any MemProf related
1769 // attributes and metadata, as we don't know whether we are linking with
1770 // a library containing the necessary interfaces.
1771 if (Phase == ThinOrFullLTOPhase::None)
1772 MPM.addPass(Pass: MemProfRemoveInfo());
1773
1774 // Convert @llvm.global.annotations to !annotation metadata.
1775 MPM.addPass(Pass: Annotation2MetadataPass());
1776
1777 // Force any function attributes we want the rest of the pipeline to observe.
1778 MPM.addPass(Pass: ForceFunctionAttrsPass());
1779
1780 if (TriggerCrash)
1781 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: TriggerCrashFunctionPass()));
1782
1783 if (PGOOpt && PGOOpt->DebugInfoForProfiling)
1784 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: AddDiscriminatorsPass()));
1785
1786 // Apply module pipeline start EP callback.
1787 invokePipelineStartEPCallbacks(MPM, Level);
1788
1789 // Add the core simplification pipeline.
1790 MPM.addPass(Pass: buildModuleSimplificationPipeline(Level, Phase));
1791
1792 // Now add the optimization pipeline.
1793 MPM.addPass(Pass: buildModuleOptimizationPipeline(Level, LTOPhase: Phase));
1794
1795 if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&
1796 PGOOpt->Action == PGOOptions::SampleUse)
1797 MPM.addPass(Pass: PseudoProbeUpdatePass());
1798
1799 // Emit annotation remarks.
1800 addAnnotationRemarksPass(MPM);
1801
1802 if (isLTOPreLink(Phase))
1803 addRequiredLTOPreLinkPasses(MPM);
1804
1805 instructionCountersPass(MPM, /* IsPreOptimization */ false);
1806 return MPM;
1807}
1808
1809ModulePassManager
1810PassBuilder::buildFatLTODefaultPipeline(OptimizationLevel Level, bool ThinLTO,
1811 bool EmitSummary, bool Verify) {
1812 ModulePassManager MPM;
1813
1814 instructionCountersPass(MPM, /* IsPreOptimization */ true);
1815
1816 if (ThinLTO)
1817 MPM.addPass(Pass: buildThinLTOPreLinkDefaultPipeline(Level));
1818 else
1819 MPM.addPass(Pass: buildLTOPreLinkDefaultPipeline(Level));
1820 // AssignGUIDPass attaches !guid metadata (MD_unique_id) to global objects,
1821 // triggering the bitcode writer to emit a METADATA_KIND_BLOCK. Standard LTO
1822 // bitcode emission runs VerifierPass by default, which registers metadata
1823 // kind IDs in LLVMContext. Running VerifierPass here before EmbedBitcodePass
1824 // to get the same behavior.
1825 if (Verify)
1826 MPM.addPass(Pass: VerifierPass());
1827 MPM.addPass(Pass: EmbedBitcodePass(ThinLTO, EmitSummary));
1828
1829 // Perform any cleanups to the IR that aren't suitable for per TU compilation,
1830 // like removing CFI/WPD related instructions. Note, we reuse
1831 // DropTypeTestsPass to clean up type tests rather than duplicate that logic
1832 // in FatLtoCleanup.
1833 MPM.addPass(Pass: FatLtoCleanup());
1834
1835 // If we're doing FatLTO w/ CFI enabled, we don't want the type tests in the
1836 // object code, only in the bitcode section, so drop it before we run
1837 // module optimization and generate machine code. If llvm.type.test() isn't in
1838 // the IR, this won't do anything.
1839 MPM.addPass(Pass: DropTypeTestsPass(lowertypetests::DropTestKind::All));
1840
1841 // Use the ThinLTO post-link pipeline with sample profiling
1842 if (ThinLTO && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse)
1843 MPM.addPass(Pass: buildThinLTODefaultPipeline(Level, /*ImportSummary=*/nullptr));
1844 else {
1845 // ModuleSimplification does not run the coroutine passes for
1846 // ThinLTOPreLink, so we need the coroutine passes to run for ThinLTO
1847 // builds, otherwise they will miscompile.
1848 if (ThinLTO) {
1849 // TODO: replace w/ buildCoroWrapper() when it takes phase and level into
1850 // consideration.
1851 CGSCCPassManager CGPM;
1852 CGPM.addPass(Pass: CoroSplitPass(Level != OptimizationLevel::O0));
1853 CGPM.addPass(Pass: CoroAnnotationElidePass());
1854 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: std::move(CGPM)));
1855 MPM.addPass(Pass: CoroCleanupPass());
1856 }
1857
1858 // otherwise, just use module optimization
1859 MPM.addPass(
1860 Pass: buildModuleOptimizationPipeline(Level, LTOPhase: ThinOrFullLTOPhase::None));
1861 // Emit annotation remarks.
1862 addAnnotationRemarksPass(MPM);
1863 }
1864
1865 instructionCountersPass(MPM, /* IsPreOptimization */ false);
1866
1867 return MPM;
1868}
1869
1870ModulePassManager
1871PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level) {
1872 if (Level == OptimizationLevel::O0)
1873 return buildO0DefaultPipeline(Level, Phase: ThinOrFullLTOPhase::ThinLTOPreLink);
1874
1875 ModulePassManager MPM;
1876
1877 instructionCountersPass(MPM, /* IsPreOptimization */ true);
1878
1879 // Convert @llvm.global.annotations to !annotation metadata.
1880 MPM.addPass(Pass: Annotation2MetadataPass());
1881
1882 // Force any function attributes we want the rest of the pipeline to observe.
1883 MPM.addPass(Pass: ForceFunctionAttrsPass());
1884
1885 if (PGOOpt && PGOOpt->DebugInfoForProfiling)
1886 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: AddDiscriminatorsPass()));
1887
1888 // Apply module pipeline start EP callback.
1889 invokePipelineStartEPCallbacks(MPM, Level);
1890
1891 // If we are planning to perform ThinLTO later, we don't bloat the code with
1892 // unrolling/vectorization/... now. Just simplify the module as much as we
1893 // can.
1894 MPM.addPass(Pass: buildModuleSimplificationPipeline(
1895 Level, Phase: ThinOrFullLTOPhase::ThinLTOPreLink));
1896 // In pre-link, for ctx prof use, we stop here with an instrumented IR. We let
1897 // thinlto use the contextual info to perform imports; then use the contextual
1898 // profile in the post-thinlink phase.
1899 if (!UseCtxProfile.empty()) {
1900 addRequiredLTOPreLinkPasses(MPM);
1901 return MPM;
1902 }
1903
1904 // Run partial inlining pass to partially inline functions that have
1905 // large bodies.
1906 // FIXME: It isn't clear whether this is really the right place to run this
1907 // in ThinLTO. Because there is another canonicalization and simplification
1908 // phase that will run after the thin link, running this here ends up with
1909 // less information than will be available later and it may grow functions in
1910 // ways that aren't beneficial.
1911 if (RunPartialInlining)
1912 MPM.addPass(Pass: PartialInlinerPass());
1913
1914 if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&
1915 PGOOpt->Action == PGOOptions::SampleUse)
1916 MPM.addPass(Pass: PseudoProbeUpdatePass());
1917
1918 // Handle Optimizer{Early,Last}EPCallbacks added by clang on PreLink. Actual
1919 // optimization is going to be done in PostLink stage, but clang can't add
1920 // callbacks there in case of in-process ThinLTO called by linker.
1921 invokeOptimizerEarlyEPCallbacks(MPM, Level,
1922 /*Phase=*/ThinOrFullLTOPhase::ThinLTOPreLink);
1923 invokeOptimizerLastEPCallbacks(MPM, Level,
1924 /*Phase=*/ThinOrFullLTOPhase::ThinLTOPreLink);
1925
1926 // Emit annotation remarks.
1927 addAnnotationRemarksPass(MPM);
1928
1929 // Attach !implicit.ref metadata from all functions to copyright strings.
1930 MPM.addPass(Pass: LowerCommentStringPass());
1931
1932 addRequiredLTOPreLinkPasses(MPM);
1933
1934 instructionCountersPass(MPM, /* IsPreOptimization */ false);
1935
1936 return MPM;
1937}
1938
1939ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
1940 OptimizationLevel Level, const ModuleSummaryIndex *ImportSummary) {
1941 ModulePassManager MPM;
1942
1943 instructionCountersPass(MPM, /* IsPreOptimization */ true);
1944
1945 // If we are invoking this without a summary index noting that we are linking
1946 // with a library containing the necessary APIs, remove any MemProf related
1947 // attributes and metadata.
1948 if (!ImportSummary || !ImportSummary->withSupportsHotColdNew())
1949 MPM.addPass(Pass: MemProfRemoveInfo());
1950
1951 if (ImportSummary) {
1952 // For ThinLTO we must apply the context disambiguation decisions early, to
1953 // ensure we can correctly match the callsites to summary data.
1954 if (EnableMemProfContextDisambiguation)
1955 MPM.addPass(Pass: MemProfContextDisambiguation(
1956 ImportSummary, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
1957
1958 // These passes import type identifier resolutions for whole-program
1959 // devirtualization and CFI. They must run early because other passes may
1960 // disturb the specific instruction patterns that these passes look for,
1961 // creating dependencies on resolutions that may not appear in the summary.
1962 //
1963 // For example, GVN may transform the pattern assume(type.test) appearing in
1964 // two basic blocks into assume(phi(type.test, type.test)), which would
1965 // transform a dependency on a WPD resolution into a dependency on a type
1966 // identifier resolution for CFI.
1967 //
1968 // Also, WPD has access to more precise information than ICP and can
1969 // devirtualize more effectively, so it should operate on the IR first.
1970 //
1971 // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1972 // metadata and intrinsics.
1973 MPM.addPass(Pass: WholeProgramDevirtPass(nullptr, ImportSummary));
1974 MPM.addPass(Pass: LowerTypeTestsPass(nullptr, ImportSummary));
1975 }
1976
1977 if (Level == OptimizationLevel::O0) {
1978 // Run a second time to clean up any type tests left behind by WPD for use
1979 // in ICP.
1980 MPM.addPass(Pass: DropTypeTestsPass());
1981 MPM.addPass(Pass: buildCoroWrapper(Phase: ThinOrFullLTOPhase::ThinLTOPostLink));
1982
1983 // AllocToken transforms heap allocation calls; this needs to run late after
1984 // other allocation call transformations (such as those in InstCombine).
1985 MPM.addPass(Pass: AllocTokenPass());
1986
1987 // Drop available_externally and unreferenced globals. This is necessary
1988 // with ThinLTO in order to avoid leaving undefined references to dead
1989 // globals in the object file.
1990 MPM.addPass(Pass: EliminateAvailableExternallyPass());
1991 MPM.addPass(Pass: GlobalDCEPass());
1992 return MPM;
1993 }
1994 if (!UseCtxProfile.empty()) {
1995 MPM.addPass(
1996 Pass: buildModuleInlinerPipeline(Level, Phase: ThinOrFullLTOPhase::ThinLTOPostLink));
1997 } else {
1998 // Add the core simplification pipeline.
1999 MPM.addPass(Pass: buildModuleSimplificationPipeline(
2000 Level, Phase: ThinOrFullLTOPhase::ThinLTOPostLink));
2001 }
2002 // Now add the optimization pipeline.
2003 MPM.addPass(Pass: buildModuleOptimizationPipeline(
2004 Level, LTOPhase: ThinOrFullLTOPhase::ThinLTOPostLink));
2005
2006 // Emit annotation remarks.
2007 addAnnotationRemarksPass(MPM);
2008
2009 instructionCountersPass(MPM, /* IsPreOptimization */ false);
2010
2011 return MPM;
2012}
2013
2014ModulePassManager
2015PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level) {
2016 // FIXME: We should use a customized pre-link pipeline!
2017 return buildPerModuleDefaultPipeline(Level,
2018 Phase: ThinOrFullLTOPhase::FullLTOPreLink);
2019}
2020
2021ModulePassManager
2022PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
2023 ModuleSummaryIndex *ExportSummary) {
2024 ModulePassManager MPM;
2025
2026 instructionCountersPass(MPM, /* IsPreOptimization */ true);
2027
2028 invokeFullLinkTimeOptimizationEarlyEPCallbacks(MPM, Level);
2029
2030 // If we are invoking this without a summary index noting that we are linking
2031 // with a library containing the necessary APIs, remove any MemProf related
2032 // attributes and metadata.
2033 if (!ExportSummary || !ExportSummary->withSupportsHotColdNew())
2034 MPM.addPass(Pass: MemProfRemoveInfo());
2035
2036 // Create a function that performs CFI checks for cross-DSO calls with targets
2037 // in the current module.
2038 MPM.addPass(Pass: CrossDSOCFIPass());
2039
2040 if (Level == OptimizationLevel::O0) {
2041 // The WPD and LowerTypeTest passes need to run at -O0 to lower type
2042 // metadata and intrinsics.
2043 MPM.addPass(Pass: WholeProgramDevirtPass(ExportSummary, nullptr));
2044 MPM.addPass(Pass: LowerTypeTestsPass(ExportSummary, nullptr));
2045 // Run a second time to clean up any type tests left behind by WPD for use
2046 // in ICP.
2047 MPM.addPass(Pass: DropTypeTestsPass());
2048
2049 MPM.addPass(Pass: buildCoroWrapper(Phase: ThinOrFullLTOPhase::FullLTOPostLink));
2050
2051 // AllocToken transforms heap allocation calls; this needs to run late after
2052 // other allocation call transformations (such as those in InstCombine).
2053 MPM.addPass(Pass: AllocTokenPass());
2054
2055 invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);
2056
2057 // Emit annotation remarks.
2058 addAnnotationRemarksPass(MPM);
2059
2060 return MPM;
2061 }
2062
2063 if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
2064 // Load sample profile before running the LTO optimization pipeline.
2065 MPM.addPass(Pass: SampleProfileLoaderPass(PGOOpt->ProfileFile,
2066 PGOOpt->ProfileRemappingFile,
2067 ThinOrFullLTOPhase::FullLTOPostLink));
2068 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
2069 // RequireAnalysisPass for PSI before subsequent non-module passes.
2070 MPM.addPass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
2071 }
2072
2073 // Try to run OpenMP optimizations, quick no-op if no OpenMP metadata present.
2074 MPM.addPass(Pass: OpenMPOptPass(ThinOrFullLTOPhase::FullLTOPostLink));
2075
2076 // Remove unused virtual tables to improve the quality of code generated by
2077 // whole-program devirtualization and bitset lowering.
2078 MPM.addPass(Pass: GlobalDCEPass(/*InLTOPostLink=*/true));
2079
2080 // Do basic inference of function attributes from known properties of system
2081 // libraries and other oracles.
2082 MPM.addPass(Pass: InferFunctionAttrsPass());
2083
2084 if (Level >= OptimizationLevel::O2) {
2085 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
2086 Pass: CallSiteSplittingPass(), EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
2087
2088 // Indirect call promotion. This should promote all the targets that are
2089 // left by the earlier promotion pass that promotes intra-module targets.
2090 // This two-step promotion is to save the compile time. For LTO, it should
2091 // produce the same result as if we only do promotion here.
2092 MPM.addPass(Pass: PGOIndirectCallPromotion(
2093 true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
2094
2095 // Promoting by-reference arguments to by-value exposes more constants to
2096 // IPSCCP.
2097 CGSCCPassManager CGPM;
2098 CGPM.addPass(Pass: PostOrderFunctionAttrsPass());
2099 CGPM.addPass(Pass: ArgumentPromotionPass());
2100 CGPM.addPass(
2101 Pass: createCGSCCToFunctionPassAdaptor(Pass: SROAPass(SROAOptions::ModifyCFG)));
2102 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: std::move(CGPM)));
2103
2104 // Propagate constants at call sites into the functions they call. This
2105 // opens opportunities for globalopt (and inlining) by substituting function
2106 // pointers passed as arguments to direct uses of functions.
2107 MPM.addPass(Pass: IPSCCPPass(IPSCCPOptions(/*AllowFuncSpec=*/true)));
2108
2109 // Attach metadata to indirect call sites indicating the set of functions
2110 // they may target at run-time. This should follow IPSCCP.
2111 MPM.addPass(Pass: CalledValuePropagationPass());
2112 }
2113
2114 // Do RPO function attribute inference across the module to forward-propagate
2115 // attributes where applicable.
2116 // FIXME: Is this really an optimization rather than a canonicalization?
2117 MPM.addPass(Pass: ReversePostOrderFunctionAttrsPass());
2118
2119 // Use in-range annotations on GEP indices to split globals where beneficial.
2120 MPM.addPass(Pass: GlobalSplitPass());
2121
2122 // Run whole program optimization of virtual call when the list of callees
2123 // is fixed.
2124 MPM.addPass(Pass: WholeProgramDevirtPass(ExportSummary, nullptr));
2125
2126 MPM.addPass(Pass: NoRecurseLTOInferencePass());
2127 // Stop here at -O1.
2128 if (Level == OptimizationLevel::O1) {
2129 // The LowerTypeTestsPass needs to run to lower type metadata and the
2130 // type.test intrinsics. The pass does nothing if CFI is disabled.
2131 MPM.addPass(Pass: LowerTypeTestsPass(ExportSummary, nullptr));
2132 // Run a second time to clean up any type tests left behind by WPD for use
2133 // in ICP (which is performed earlier than this in the regular LTO
2134 // pipeline).
2135 MPM.addPass(Pass: DropTypeTestsPass());
2136
2137 MPM.addPass(Pass: buildCoroWrapper(Phase: ThinOrFullLTOPhase::FullLTOPostLink));
2138
2139 // AllocToken transforms heap allocation calls; this needs to run late after
2140 // other allocation call transformations (such as those in InstCombine).
2141 MPM.addPass(Pass: AllocTokenPass());
2142
2143 invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);
2144
2145 // Emit annotation remarks.
2146 addAnnotationRemarksPass(MPM);
2147
2148 instructionCountersPass(MPM, /* IsPreOptimization */ false);
2149
2150 return MPM;
2151 }
2152
2153 // TODO: Skip to match buildCoroWrapper.
2154 MPM.addPass(Pass: CoroEarlyPass());
2155
2156 // Optimize globals to try and fold them into constants.
2157 MPM.addPass(Pass: GlobalOptPass());
2158
2159 // Promote any localized globals to SSA registers.
2160 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: PromotePass()));
2161
2162 // Linking modules together can lead to duplicate global constant, only
2163 // keep one copy of each constant.
2164 MPM.addPass(Pass: ConstantMergePass());
2165
2166 // Remove unused arguments from functions.
2167 MPM.addPass(Pass: DeadArgumentEliminationPass());
2168
2169 // Reduce the code after globalopt and ipsccp. Both can open up significant
2170 // simplification opportunities, and both can propagate functions through
2171 // function pointers. When this happens, we often have to resolve varargs
2172 // calls, etc, so let instcombine do this.
2173 FunctionPassManager PeepholeFPM;
2174 PeepholeFPM.addPass(Pass: InstCombinePass());
2175 if (Level >= OptimizationLevel::O2)
2176 PeepholeFPM.addPass(Pass: AggressiveInstCombinePass());
2177 invokePeepholeEPCallbacks(FPM&: PeepholeFPM, Level);
2178
2179 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(PeepholeFPM),
2180 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
2181
2182 // Lower variadic functions for supported targets prior to inlining.
2183 MPM.addPass(Pass: ExpandVariadicsPass(ExpandVariadicsMode::Optimize));
2184
2185 // Note: historically, the PruneEH pass was run first to deduce nounwind and
2186 // generally clean up exception handling overhead. It isn't clear this is
2187 // valuable as the inliner doesn't currently care whether it is inlining an
2188 // invoke or a call.
2189 // Run the inliner now.
2190 if (EnableModuleInliner) {
2191 MPM.addPass(Pass: ModuleInlinerPass(::getInlineParamsFromOptLevel(Level),
2192 UseInlineAdvisor,
2193 ThinOrFullLTOPhase::FullLTOPostLink));
2194 } else {
2195 MPM.addPass(Pass: ModuleInlinerWrapperPass(
2196 ::getInlineParamsFromOptLevel(Level),
2197 /* MandatoryFirst */ true,
2198 InlineContext{.LTOPhase: ThinOrFullLTOPhase::FullLTOPostLink,
2199 .Pass: InlinePass::CGSCCInliner}));
2200 }
2201
2202 // Perform context disambiguation after inlining, since that would reduce the
2203 // amount of additional cloning required to distinguish the allocation
2204 // contexts.
2205 if (EnableMemProfContextDisambiguation)
2206 MPM.addPass(Pass: MemProfContextDisambiguation(
2207 /*Summary=*/nullptr,
2208 PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
2209
2210 // Optimize globals again after we ran the inliner.
2211 MPM.addPass(Pass: GlobalOptPass());
2212
2213 // Run the OpenMPOpt pass again after global optimizations.
2214 MPM.addPass(Pass: OpenMPOptPass(ThinOrFullLTOPhase::FullLTOPostLink));
2215
2216 // Garbage collect dead functions.
2217 MPM.addPass(Pass: GlobalDCEPass(/*InLTOPostLink=*/true));
2218
2219 // If we didn't decide to inline a function, check to see if we can
2220 // transform it to pass arguments by value instead of by reference.
2221 CGSCCPassManager CGPM;
2222 CGPM.addPass(Pass: ArgumentPromotionPass());
2223 CGPM.addPass(Pass: CoroSplitPass(Level != OptimizationLevel::O0));
2224 CGPM.addPass(Pass: CoroAnnotationElidePass());
2225 invokeCGSCCOptimizerLateEPCallbacks(CGPM, Level);
2226 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: std::move(CGPM)));
2227
2228 FunctionPassManager FPM;
2229 // The IPO Passes may leave cruft around. Clean up after them.
2230 FPM.addPass(Pass: InstCombinePass());
2231 invokePeepholeEPCallbacks(FPM, Level);
2232
2233 if (EnableConstraintElimination)
2234 FPM.addPass(Pass: ConstraintEliminationPass());
2235
2236 FPM.addPass(Pass: JumpThreadingPass());
2237
2238 // Do a post inline PGO instrumentation and use pass. This is a context
2239 // sensitive PGO pass.
2240 if (PGOOpt) {
2241 if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
2242 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/true,
2243 /*IsCS=*/true, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate,
2244 ProfileFile: PGOOpt->CSProfileGenFile, ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
2245 else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
2246 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/false,
2247 /*IsCS=*/true, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate,
2248 ProfileFile: PGOOpt->ProfileFile, ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
2249 }
2250
2251 // Break up allocas
2252 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
2253
2254 // LTO provides additional opportunities for tailcall elimination due to
2255 // link-time inlining, and visibility of nocapture attribute.
2256 FPM.addPass(
2257 Pass: TailCallElimPass(/*UpdateFunctionEntryCount=*/isInstrumentedPGOUse()));
2258
2259 // Run a few AA driver optimizations here and now to cleanup the code.
2260 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM),
2261 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
2262
2263 MPM.addPass(
2264 Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: PostOrderFunctionAttrsPass()));
2265
2266 // Require the GlobalsAA analysis for the module so we can query it within
2267 // MainFPM.
2268 if (EnableGlobalAnalyses) {
2269 MPM.addPass(Pass: RequireAnalysisPass<GlobalsAA, Module>());
2270 // Invalidate AAManager so it can be recreated and pick up the newly
2271 // available GlobalsAA.
2272 MPM.addPass(
2273 Pass: createModuleToFunctionPassAdaptor(Pass: InvalidateAnalysisPass<AAManager>()));
2274 }
2275
2276 FunctionPassManager MainFPM;
2277 MainFPM.addPass(Pass: createFunctionToLoopPassAdaptor(
2278 Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
2279 /*AllowSpeculation=*/true),
2280 /*USeMemorySSA=*/UseMemorySSA: true));
2281
2282 if (RunNewGVN)
2283 MainFPM.addPass(Pass: NewGVNPass());
2284 else
2285 MainFPM.addPass(Pass: GVNPass());
2286
2287 // Remove dead memcpy()'s.
2288 MainFPM.addPass(Pass: MemCpyOptPass());
2289
2290 // Nuke dead stores.
2291 MainFPM.addPass(Pass: DSEPass());
2292 MainFPM.addPass(Pass: MoveAutoInitPass());
2293 MainFPM.addPass(Pass: MergedLoadStoreMotionPass());
2294
2295 invokeVectorizerStartEPCallbacks(FPM&: MainFPM, Level);
2296
2297 LoopPassManager LPM;
2298 if (EnableLoopFlatten && Level >= OptimizationLevel::O2)
2299 LPM.addPass(Pass: LoopFlattenPass());
2300 LPM.addPass(Pass: IndVarSimplifyPass());
2301 LPM.addPass(Pass: LoopDeletionPass());
2302 // FIXME: Add loop interchange.
2303
2304 // Unroll small loops and perform peeling.
2305 LPM.addPass(Pass: LoopFullUnrollPass(static_cast<int>(Level),
2306 /* OnlyWhenForced= */ !PTO.LoopUnrolling,
2307 PTO.ForgetAllSCEVInLoopUnroll));
2308 // The loop passes in LPM (LoopFullUnrollPass) do not preserve MemorySSA.
2309 // *All* loop passes must preserve it, in order to be able to use it.
2310 MainFPM.addPass(
2311 Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM), /*UseMemorySSA=*/false));
2312
2313 MainFPM.addPass(Pass: LoopDistributePass());
2314
2315 addVectorPasses(Level, FPM&: MainFPM, LTOPhase: ThinOrFullLTOPhase::FullLTOPostLink);
2316
2317 invokeVectorizerEndEPCallbacks(FPM&: MainFPM, Level);
2318
2319 // Run the OpenMPOpt CGSCC pass again late.
2320 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(
2321 Pass: OpenMPOptCGSCCPass(ThinOrFullLTOPhase::FullLTOPostLink)));
2322
2323 invokePeepholeEPCallbacks(FPM&: MainFPM, Level);
2324 MainFPM.addPass(Pass: JumpThreadingPass());
2325 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(MainFPM),
2326 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
2327
2328 // Lower type metadata and the type.test intrinsic. This pass supports
2329 // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
2330 // to be run at link time if CFI is enabled. This pass does nothing if
2331 // CFI is disabled.
2332 MPM.addPass(Pass: LowerTypeTestsPass(ExportSummary, nullptr));
2333 // Run a second time to clean up any type tests left behind by WPD for use
2334 // in ICP (which is performed earlier than this in the regular LTO pipeline).
2335 MPM.addPass(Pass: DropTypeTestsPass());
2336
2337 // Enable splitting late in the FullLTO post-link pipeline.
2338 if (EnableHotColdSplit)
2339 MPM.addPass(Pass: HotColdSplittingPass());
2340
2341 // Add late LTO optimization passes.
2342 FunctionPassManager LateFPM;
2343
2344 // LoopSink pass sinks instructions hoisted by LICM, which serves as a
2345 // canonicalization pass that enables other optimizations. As a result,
2346 // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
2347 // result too early.
2348 LateFPM.addPass(Pass: LoopSinkPass());
2349
2350 // This hoists/decomposes div/rem ops. It should run after other sink/hoist
2351 // passes to avoid re-sinking, but before SimplifyCFG because it can allow
2352 // flattening of blocks.
2353 LateFPM.addPass(Pass: DivRemPairsPass());
2354
2355 // Delete basic blocks, which optimization passes may have killed.
2356 LateFPM.addPass(Pass: SimplifyCFGPass(SimplifyCFGOptions()
2357 .convertSwitchRangeToICmp(B: true)
2358 .convertSwitchToArithmetic(B: true)
2359 .hoistCommonInsts(B: true)
2360 .speculateUnpredictables(B: true)));
2361 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(LateFPM)));
2362
2363 // Drop bodies of available eternally objects to improve GlobalDCE.
2364 MPM.addPass(Pass: EliminateAvailableExternallyPass());
2365
2366 // Now that we have optimized the program, discard unreachable functions.
2367 MPM.addPass(Pass: GlobalDCEPass(/*InLTOPostLink=*/true));
2368
2369 if (PTO.MergeFunctions)
2370 MPM.addPass(Pass: MergeFunctionsPass());
2371
2372 MPM.addPass(Pass: RelLookupTableConverterPass());
2373
2374 if (PTO.CallGraphProfile)
2375 MPM.addPass(Pass: CGProfilePass(/*InLTOPostLink=*/true));
2376
2377 MPM.addPass(Pass: CoroCleanupPass());
2378
2379 // AllocToken transforms heap allocation calls; this needs to run late after
2380 // other allocation call transformations (such as those in InstCombine).
2381 MPM.addPass(Pass: AllocTokenPass());
2382
2383 invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);
2384
2385 // Emit annotation remarks.
2386 addAnnotationRemarksPass(MPM);
2387
2388 instructionCountersPass(MPM, /* IsPreOptimization */ false);
2389
2390 return MPM;
2391}
2392
2393ModulePassManager
2394PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level,
2395 ThinOrFullLTOPhase Phase) {
2396 assert(Level == OptimizationLevel::O0 &&
2397 "buildO0DefaultPipeline should only be used with O0");
2398
2399 ModulePassManager MPM;
2400
2401 instructionCountersPass(MPM, /* IsPreOptimization */ true);
2402
2403 // Perform pseudo probe instrumentation in O0 mode. This is for the
2404 // consistency between different build modes. For example, a LTO build can be
2405 // mixed with an O0 prelink and an O2 postlink. Loading a sample profile in
2406 // the postlink will require pseudo probe instrumentation in the prelink.
2407 if (PGOOpt && PGOOpt->PseudoProbeForProfiling)
2408 MPM.addPass(Pass: SampleProfileProbePass(TM));
2409
2410 if (PGOOpt && (PGOOpt->Action == PGOOptions::IRInstr ||
2411 PGOOpt->Action == PGOOptions::IRUse))
2412 addPGOInstrPassesForO0(
2413 MPM,
2414 /*RunProfileGen=*/(PGOOpt->Action == PGOOptions::IRInstr),
2415 /*IsCS=*/false, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate, ProfileFile: PGOOpt->ProfileFile,
2416 ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
2417
2418 // Instrument function entry and exit before all inlining.
2419 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
2420 Pass: EntryExitInstrumenterPass(/*PostInlining=*/false)));
2421
2422 invokePipelineStartEPCallbacks(MPM, Level);
2423
2424 if (PGOOpt && PGOOpt->DebugInfoForProfiling)
2425 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: AddDiscriminatorsPass()));
2426
2427 if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
2428 // Explicitly disable sample loader inlining and use flattened profile in O0
2429 // pipeline.
2430 MPM.addPass(Pass: SampleProfileLoaderPass(PGOOpt->ProfileFile,
2431 PGOOpt->ProfileRemappingFile,
2432 ThinOrFullLTOPhase::None, FS,
2433 /*DisableSampleProfileInlining=*/true,
2434 /*UseFlattenedProfile=*/true));
2435 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
2436 // RequireAnalysisPass for PSI before subsequent non-module passes.
2437 MPM.addPass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
2438 }
2439
2440 invokePipelineEarlySimplificationEPCallbacks(MPM, Level, Phase);
2441
2442 // Build a minimal pipeline based on the semantics required by LLVM,
2443 // which is just that always inlining occurs. Further, disable generating
2444 // lifetime intrinsics to avoid enabling further optimizations during
2445 // code generation.
2446 MPM.addPass(Pass: AlwaysInlinerPass(
2447 /*InsertLifetimeIntrinsics=*/false));
2448
2449 if (PTO.MergeFunctions)
2450 MPM.addPass(Pass: MergeFunctionsPass());
2451
2452 if (EnableMatrix)
2453 MPM.addPass(
2454 Pass: createModuleToFunctionPassAdaptor(Pass: LowerMatrixIntrinsicsPass(true)));
2455
2456 if (!CGSCCOptimizerLateEPCallbacks.empty()) {
2457 CGSCCPassManager CGPM;
2458 invokeCGSCCOptimizerLateEPCallbacks(CGPM, Level);
2459 if (!CGPM.isEmpty())
2460 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: std::move(CGPM)));
2461 }
2462 if (!LateLoopOptimizationsEPCallbacks.empty()) {
2463 LoopPassManager LPM;
2464 invokeLateLoopOptimizationsEPCallbacks(LPM, Level);
2465 if (!LPM.isEmpty()) {
2466 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
2467 Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM))));
2468 }
2469 }
2470 if (!LoopOptimizerEndEPCallbacks.empty()) {
2471 LoopPassManager LPM;
2472 invokeLoopOptimizerEndEPCallbacks(LPM, Level);
2473 if (!LPM.isEmpty()) {
2474 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
2475 Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM))));
2476 }
2477 }
2478 if (!ScalarOptimizerLateEPCallbacks.empty()) {
2479 FunctionPassManager FPM;
2480 invokeScalarOptimizerLateEPCallbacks(FPM, Level);
2481 if (!FPM.isEmpty())
2482 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
2483 }
2484
2485 invokeOptimizerEarlyEPCallbacks(MPM, Level, Phase);
2486
2487 if (!VectorizerStartEPCallbacks.empty()) {
2488 FunctionPassManager FPM;
2489 invokeVectorizerStartEPCallbacks(FPM, Level);
2490 if (!FPM.isEmpty())
2491 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
2492 }
2493
2494 if (!VectorizerEndEPCallbacks.empty()) {
2495 FunctionPassManager FPM;
2496 invokeVectorizerEndEPCallbacks(FPM, Level);
2497 if (!FPM.isEmpty())
2498 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
2499 }
2500
2501 MPM.addPass(Pass: buildCoroWrapper(Phase));
2502
2503 // AllocToken transforms heap allocation calls; this needs to run late after
2504 // other allocation call transformations (such as those in InstCombine).
2505 if (!isLTOPreLink(Phase))
2506 MPM.addPass(Pass: AllocTokenPass());
2507
2508 invokeOptimizerLastEPCallbacks(MPM, Level, Phase);
2509
2510 if (EnableInstrumentor)
2511 MPM.addPass(Pass: InstrumentorPass(FS));
2512
2513 // Attach !implicit.ref metadata from all functions to copyright strings.
2514 MPM.addPass(Pass: LowerCommentStringPass());
2515
2516 if (isLTOPreLink(Phase))
2517 addRequiredLTOPreLinkPasses(MPM);
2518
2519 // Emit annotation remarks.
2520 addAnnotationRemarksPass(MPM);
2521
2522 instructionCountersPass(MPM, /* IsPreOptimization */ false);
2523
2524 return MPM;
2525}
2526
2527AAManager PassBuilder::buildDefaultAAPipeline() {
2528 AAManager AA;
2529
2530 // The order in which these are registered determines their priority when
2531 // being queried.
2532
2533 // Add any target-specific alias analyses that should be run early.
2534 if (TM)
2535 TM->registerEarlyDefaultAliasAnalyses(AA);
2536
2537 // First we register the basic alias analysis that provides the majority of
2538 // per-function local AA logic. This is a stateless, on-demand local set of
2539 // AA techniques.
2540 AA.registerFunctionAnalysis<BasicAA>();
2541
2542 // Next we query fast, specialized alias analyses that wrap IR-embedded
2543 // information about aliasing.
2544 AA.registerFunctionAnalysis<ScopedNoAliasAA>();
2545 AA.registerFunctionAnalysis<TypeBasedAA>();
2546
2547 // Add support for querying global aliasing information when available.
2548 // Because the `AAManager` is a function analysis and `GlobalsAA` is a module
2549 // analysis, all that the `AAManager` can do is query for any *cached*
2550 // results from `GlobalsAA` through a readonly proxy.
2551 if (EnableGlobalAnalyses)
2552 AA.registerModuleAnalysis<GlobalsAA>();
2553
2554 // Add target-specific alias analyses.
2555 if (TM)
2556 TM->registerDefaultAliasAnalyses(AA);
2557
2558 return AA;
2559}
2560
2561bool PassBuilder::isInstrumentedPGOUse() const {
2562 return (PGOOpt && PGOOpt->Action == PGOOptions::IRUse) ||
2563 !UseCtxProfile.empty();
2564}
2565