1//===- Construction of pass pipelines -------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9///
10/// This file provides the implementation of the PassBuilder based on our
11/// static pass registry as well as related functionality. It also provides
12/// helpers to aid in analyzing, debugging, and testing passes and pass
13/// pipelines.
14///
15//===----------------------------------------------------------------------===//
16
17#include "llvm/ADT/Statistic.h"
18#include "llvm/Analysis/AliasAnalysis.h"
19#include "llvm/Analysis/BasicAliasAnalysis.h"
20#include "llvm/Analysis/CGSCCPassManager.h"
21#include "llvm/Analysis/FunctionPropertiesAnalysis.h"
22#include "llvm/Analysis/GlobalsModRef.h"
23#include "llvm/Analysis/InlineAdvisor.h"
24#include "llvm/Analysis/InstCount.h"
25#include "llvm/Analysis/ProfileSummaryInfo.h"
26#include "llvm/Analysis/ScopedNoAliasAA.h"
27#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
28#include "llvm/IR/PassManager.h"
29#include "llvm/IR/Verifier.h"
30#include "llvm/Pass.h"
31#include "llvm/Passes/OptimizationLevel.h"
32#include "llvm/Passes/PassBuilder.h"
33#include "llvm/Passes/TriggerCrashPasses.h"
34#include "llvm/Support/CommandLine.h"
35#include "llvm/Support/ErrorHandling.h"
36#include "llvm/Support/PGOOptions.h"
37#include "llvm/Target/TargetMachine.h"
38#include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
39#include "llvm/Transforms/Coroutines/CoroAnnotationElide.h"
40#include "llvm/Transforms/Coroutines/CoroCleanup.h"
41#include "llvm/Transforms/Coroutines/CoroConditionalWrapper.h"
42#include "llvm/Transforms/Coroutines/CoroEarly.h"
43#include "llvm/Transforms/Coroutines/CoroElide.h"
44#include "llvm/Transforms/Coroutines/CoroSplit.h"
45#include "llvm/Transforms/HipStdPar/HipStdPar.h"
46#include "llvm/Transforms/IPO/AlwaysInliner.h"
47#include "llvm/Transforms/IPO/Annotation2Metadata.h"
48#include "llvm/Transforms/IPO/ArgumentPromotion.h"
49#include "llvm/Transforms/IPO/Attributor.h"
50#include "llvm/Transforms/IPO/CalledValuePropagation.h"
51#include "llvm/Transforms/IPO/ConstantMerge.h"
52#include "llvm/Transforms/IPO/CrossDSOCFI.h"
53#include "llvm/Transforms/IPO/DeadArgumentElimination.h"
54#include "llvm/Transforms/IPO/ElimAvailExtern.h"
55#include "llvm/Transforms/IPO/EmbedBitcodePass.h"
56#include "llvm/Transforms/IPO/ExpandVariadics.h"
57#include "llvm/Transforms/IPO/FatLTOCleanup.h"
58#include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
59#include "llvm/Transforms/IPO/FunctionAttrs.h"
60#include "llvm/Transforms/IPO/GlobalDCE.h"
61#include "llvm/Transforms/IPO/GlobalOpt.h"
62#include "llvm/Transforms/IPO/GlobalSplit.h"
63#include "llvm/Transforms/IPO/HotColdSplitting.h"
64#include "llvm/Transforms/IPO/InferFunctionAttrs.h"
65#include "llvm/Transforms/IPO/Inliner.h"
66#include "llvm/Transforms/IPO/Instrumentor.h"
67#include "llvm/Transforms/IPO/LowerTypeTests.h"
68#include "llvm/Transforms/IPO/MemProfContextDisambiguation.h"
69#include "llvm/Transforms/IPO/MergeFunctions.h"
70#include "llvm/Transforms/IPO/ModuleInliner.h"
71#include "llvm/Transforms/IPO/OpenMPOpt.h"
72#include "llvm/Transforms/IPO/PartialInlining.h"
73#include "llvm/Transforms/IPO/SCCP.h"
74#include "llvm/Transforms/IPO/SampleProfile.h"
75#include "llvm/Transforms/IPO/SampleProfileProbe.h"
76#include "llvm/Transforms/IPO/WholeProgramDevirt.h"
77#include "llvm/Transforms/InstCombine/InstCombine.h"
78#include "llvm/Transforms/Instrumentation/AllocToken.h"
79#include "llvm/Transforms/Instrumentation/CGProfile.h"
80#include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
81#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
82#include "llvm/Transforms/Instrumentation/MemProfInstrumentation.h"
83#include "llvm/Transforms/Instrumentation/MemProfUse.h"
84#include "llvm/Transforms/Instrumentation/PGOCtxProfFlattening.h"
85#include "llvm/Transforms/Instrumentation/PGOCtxProfLowering.h"
86#include "llvm/Transforms/Instrumentation/PGOForceFunctionAttrs.h"
87#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
88#include "llvm/Transforms/Scalar/ADCE.h"
89#include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
90#include "llvm/Transforms/Scalar/AnnotationRemarks.h"
91#include "llvm/Transforms/Scalar/BDCE.h"
92#include "llvm/Transforms/Scalar/CallSiteSplitting.h"
93#include "llvm/Transforms/Scalar/ConstraintElimination.h"
94#include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
95#include "llvm/Transforms/Scalar/DFAJumpThreading.h"
96#include "llvm/Transforms/Scalar/DeadStoreElimination.h"
97#include "llvm/Transforms/Scalar/DivRemPairs.h"
98#include "llvm/Transforms/Scalar/DropUnnecessaryAssumes.h"
99#include "llvm/Transforms/Scalar/EarlyCSE.h"
100#include "llvm/Transforms/Scalar/ExpandMemCmp.h"
101#include "llvm/Transforms/Scalar/Float2Int.h"
102#include "llvm/Transforms/Scalar/GVN.h"
103#include "llvm/Transforms/Scalar/IndVarSimplify.h"
104#include "llvm/Transforms/Scalar/InferAlignment.h"
105#include "llvm/Transforms/Scalar/InstSimplifyPass.h"
106#include "llvm/Transforms/Scalar/JumpTableToSwitch.h"
107#include "llvm/Transforms/Scalar/JumpThreading.h"
108#include "llvm/Transforms/Scalar/LICM.h"
109#include "llvm/Transforms/Scalar/LoopDeletion.h"
110#include "llvm/Transforms/Scalar/LoopDistribute.h"
111#include "llvm/Transforms/Scalar/LoopFlatten.h"
112#include "llvm/Transforms/Scalar/LoopFuse.h"
113#include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
114#include "llvm/Transforms/Scalar/LoopInstSimplify.h"
115#include "llvm/Transforms/Scalar/LoopInterchange.h"
116#include "llvm/Transforms/Scalar/LoopLoadElimination.h"
117#include "llvm/Transforms/Scalar/LoopPassManager.h"
118#include "llvm/Transforms/Scalar/LoopRotation.h"
119#include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
120#include "llvm/Transforms/Scalar/LoopSink.h"
121#include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
122#include "llvm/Transforms/Scalar/LoopUnrollPass.h"
123#include "llvm/Transforms/Scalar/LoopVersioningLICM.h"
124#include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
125#include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
126#include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h"
127#include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
128#include "llvm/Transforms/Scalar/MergeICmps.h"
129#include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"
130#include "llvm/Transforms/Scalar/NewGVN.h"
131#include "llvm/Transforms/Scalar/Reassociate.h"
132#include "llvm/Transforms/Scalar/SCCP.h"
133#include "llvm/Transforms/Scalar/SROA.h"
134#include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
135#include "llvm/Transforms/Scalar/SimplifyCFG.h"
136#include "llvm/Transforms/Scalar/SpeculativeExecution.h"
137#include "llvm/Transforms/Scalar/TailRecursionElimination.h"
138#include "llvm/Transforms/Scalar/WarnMissedTransforms.h"
139#include "llvm/Transforms/Utils/AddDiscriminators.h"
140#include "llvm/Transforms/Utils/AssignGUID.h"
141#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
142#include "llvm/Transforms/Utils/CanonicalizeAliases.h"
143#include "llvm/Transforms/Utils/CountVisits.h"
144#include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
145#include "llvm/Transforms/Utils/ExtraPassManager.h"
146#include "llvm/Transforms/Utils/InjectTLIMappings.h"
147#include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
148#include "llvm/Transforms/Utils/LowerCommentStringPass.h"
149#include "llvm/Transforms/Utils/Mem2Reg.h"
150#include "llvm/Transforms/Utils/MoveAutoInit.h"
151#include "llvm/Transforms/Utils/NameAnonGlobals.h"
152#include "llvm/Transforms/Utils/RelLookupTableConverter.h"
153#include "llvm/Transforms/Utils/SimplifyCFGOptions.h"
154#include "llvm/Transforms/Vectorize/LoopVectorize.h"
155#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
156#include "llvm/Transforms/Vectorize/VectorCombine.h"
157
158using namespace llvm;
159
160namespace llvm {
161
162static cl::opt<InliningAdvisorMode> UseInlineAdvisor(
163 "enable-ml-inliner", cl::init(Val: InliningAdvisorMode::Default), cl::Hidden,
164 cl::desc("Enable ML policy for inliner. Currently trained for -Oz only"),
165 cl::values(clEnumValN(InliningAdvisorMode::Default, "default",
166 "Heuristics-based inliner version"),
167 clEnumValN(InliningAdvisorMode::Development, "development",
168 "Use development mode (runtime-loadable model)"),
169 clEnumValN(InliningAdvisorMode::Release, "release",
170 "Use release mode (AOT-compiled model)")));
171
172/// Flag to enable inline deferral during PGO.
173static cl::opt<bool>
174 EnablePGOInlineDeferral("enable-npm-pgo-inline-deferral", cl::init(Val: true),
175 cl::Hidden,
176 cl::desc("Enable inline deferral during PGO"));
177
178static cl::opt<bool> EnableModuleInliner("enable-module-inliner",
179 cl::init(Val: false), cl::Hidden,
180 cl::desc("Enable module inliner"));
181
182static cl::opt<bool> PerformMandatoryInliningsFirst(
183 "mandatory-inlining-first", cl::init(Val: false), cl::Hidden,
184 cl::desc("Perform mandatory inlinings module-wide, before performing "
185 "inlining"));
186
187static cl::opt<bool> EnableEagerlyInvalidateAnalyses(
188 "eagerly-invalidate-analyses", cl::init(Val: true), cl::Hidden,
189 cl::desc("Eagerly invalidate more analyses in default pipelines"));
190
191static cl::opt<bool> EnableMergeFunctions(
192 "enable-merge-functions", cl::init(Val: false), cl::Hidden,
193 cl::desc("Enable function merging as part of the optimization pipeline"));
194
195static cl::opt<bool> EnablePostPGOLoopRotation(
196 "enable-post-pgo-loop-rotation", cl::init(Val: true), cl::Hidden,
197 cl::desc("Run the loop rotation transformation after PGO instrumentation"));
198
199static cl::opt<bool>
200 TriggerCrash("opt-pipeline-trigger-crash", cl::init(Val: false), cl::Hidden,
201 cl::desc("Trigger crash in optimization pipeline"));
202
203static cl::opt<bool> EnableGlobalAnalyses(
204 "enable-global-analyses", cl::init(Val: true), cl::Hidden,
205 cl::desc("Enable inter-procedural analyses"));
206
207static cl::opt<bool> RunPartialInlining("enable-partial-inlining",
208 cl::init(Val: false), cl::Hidden,
209 cl::desc("Run Partial inlining pass"));
210
211static cl::opt<bool> ExtraVectorizerPasses(
212 "extra-vectorizer-passes", cl::init(Val: false), cl::Hidden,
213 cl::desc("Run cleanup optimization passes after vectorization"));
214
215static cl::opt<bool> RunNewGVN("enable-newgvn", cl::init(Val: false), cl::Hidden,
216 cl::desc("Run the NewGVN pass"));
217
218static cl::opt<bool>
219 EnableLoopInterchange("enable-loopinterchange", cl::init(Val: true), cl::Hidden,
220 cl::desc("Enable the LoopInterchange Pass"));
221
222static cl::opt<bool> EnableUnrollAndJam("enable-unroll-and-jam",
223 cl::init(Val: false), cl::Hidden,
224 cl::desc("Enable Unroll And Jam Pass"));
225
226static cl::opt<bool> EnableLoopFlatten("enable-loop-flatten", cl::init(Val: false),
227 cl::Hidden,
228 cl::desc("Enable the LoopFlatten Pass"));
229
230static cl::opt<bool>
231 EnableInstrumentor("enable-instrumentor", cl::init(Val: false), cl::Hidden,
232 cl::desc("Enable the Instrumentor Pass"));
233
234static cl::opt<bool>
235 EnableDFAJumpThreading("enable-dfa-jump-thread",
236 cl::desc("Enable DFA jump threading"),
237 cl::init(Val: true), cl::Hidden);
238
239static cl::opt<bool>
240 EnableHotColdSplit("hot-cold-split",
241 cl::desc("Enable hot-cold splitting pass"));
242
243static cl::opt<bool>
244 DisablePreInliner("disable-preinline", cl::init(Val: false), cl::Hidden,
245 cl::desc("Disable pre-instrumentation inliner"));
246
247static cl::opt<int> PreInlineThreshold(
248 "preinline-threshold", cl::Hidden, cl::init(Val: 75),
249 cl::desc("Control the amount of inlining in pre-instrumentation inliner "
250 "(default = 75)"));
251
252static cl::opt<bool>
253 EnableGVNHoist("enable-gvn-hoist",
254 cl::desc("Enable the GVN hoisting pass (default = off)"));
255
256static cl::opt<bool>
257 EnableGVNSink("enable-gvn-sink",
258 cl::desc("Enable the GVN sinking pass (default = off)"));
259
260static cl::opt<bool> EnableJumpTableToSwitch(
261 "enable-jump-table-to-switch", cl::init(Val: true),
262 cl::desc("Enable JumpTableToSwitch pass (default = true)"));
263
264// This option is used in simplifying testing SampleFDO optimizations for
265// profile loading.
266static cl::opt<bool>
267 EnableCHR("enable-chr", cl::init(Val: true), cl::Hidden,
268 cl::desc("Enable control height reduction optimization (CHR)"));
269
270static cl::opt<bool> FlattenedProfileUsed(
271 "flattened-profile-used", cl::init(Val: false), cl::Hidden,
272 cl::desc("Indicate the sample profile being used is flattened, i.e., "
273 "no inline hierarchy exists in the profile"));
274
275static cl::opt<bool>
276 EnableMatrix("enable-matrix", cl::init(Val: false), cl::Hidden,
277 cl::desc("Enable lowering of the matrix intrinsics"));
278
279static cl::opt<bool> EnableMergeICmps(
280 "enable-mergeicmps", cl::init(Val: true), cl::Hidden,
281 cl::desc("Enable MergeICmps pass in the optimization pipeline"));
282
283static cl::opt<bool> EnableConstraintElimination(
284 "enable-constraint-elimination", cl::init(Val: true), cl::Hidden,
285 cl::desc(
286 "Enable pass to eliminate conditions based on linear constraints"));
287
288static cl::opt<AttributorRunOption> AttributorRun(
289 "attributor-enable", cl::Hidden, cl::init(Val: AttributorRunOption::NONE),
290 cl::desc("Enable the attributor inter-procedural deduction pass"),
291 cl::values(clEnumValN(AttributorRunOption::FULL, "full",
292 "enable all full attributor runs"),
293 clEnumValN(AttributorRunOption::LIGHT, "light",
294 "enable all attributor-light runs"),
295 clEnumValN(AttributorRunOption::MODULE, "module",
296 "enable module-wide attributor runs"),
297 clEnumValN(AttributorRunOption::MODULE_LIGHT, "module-light",
298 "enable module-wide attributor-light runs"),
299 clEnumValN(AttributorRunOption::CGSCC, "cgscc",
300 "enable call graph SCC attributor runs"),
301 clEnumValN(AttributorRunOption::CGSCC_LIGHT, "cgscc-light",
302 "enable call graph SCC attributor-light runs"),
303 clEnumValN(AttributorRunOption::NONE, "none",
304 "disable attributor runs")));
305
306static cl::opt<bool> EnableSampledInstr(
307 "enable-sampled-instrumentation", cl::init(Val: false), cl::Hidden,
308 cl::desc("Enable profile instrumentation sampling (default = off)"));
309static cl::opt<bool> UseLoopVersioningLICM(
310 "enable-loop-versioning-licm", cl::init(Val: false), cl::Hidden,
311 cl::desc("Enable the experimental Loop Versioning LICM pass"));
312
313static cl::opt<std::string> InstrumentColdFuncOnlyPath(
314 "instrument-cold-function-only-path", cl::init(Val: ""),
315 cl::desc("File path for cold function only instrumentation(requires use "
316 "with --pgo-instrument-cold-function-only)"),
317 cl::Hidden);
318
319// TODO: There is a similar flag in WPD pass, we should consolidate them by
320// parsing the option only once in PassBuilder and share it across both places.
321static cl::opt<bool> EnableDevirtualizeSpeculatively(
322 "enable-devirtualize-speculatively",
323 cl::desc("Enable speculative devirtualization optimization"),
324 cl::init(Val: false));
325
326extern cl::opt<std::string> UseCtxProfile;
327extern cl::opt<bool> PGOInstrumentColdFunctionOnly;
328
329extern cl::opt<bool> EnableMemProfContextDisambiguation;
330} // namespace llvm
331
332PipelineTuningOptions::PipelineTuningOptions() {
333 LoopInterleaving = true;
334 LoopVectorization = true;
335 SLPVectorization = false;
336 LoopUnrolling = true;
337 LoopInterchange = EnableLoopInterchange;
338 LoopFusion = false;
339 ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll;
340 LicmMssaOptCap = SetLicmMssaOptCap;
341 LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;
342 CallGraphProfile = true;
343 UnifiedLTO = false;
344 MergeFunctions = EnableMergeFunctions;
345 InlinerThreshold = -1;
346 EagerlyInvalidateAnalyses = EnableEagerlyInvalidateAnalyses;
347 DevirtualizeSpeculatively = EnableDevirtualizeSpeculatively;
348}
349
350namespace llvm {
351extern cl::opt<unsigned> MaxDevirtIterations;
352} // namespace llvm
353
354void PassBuilder::invokePeepholeEPCallbacks(FunctionPassManager &FPM,
355 OptimizationLevel Level) {
356 for (auto &C : PeepholeEPCallbacks)
357 C(FPM, Level);
358}
359void PassBuilder::invokeLateLoopOptimizationsEPCallbacks(
360 LoopPassManager &LPM, OptimizationLevel Level) {
361 for (auto &C : LateLoopOptimizationsEPCallbacks)
362 C(LPM, Level);
363}
364void PassBuilder::invokeLoopOptimizerEndEPCallbacks(LoopPassManager &LPM,
365 OptimizationLevel Level) {
366 for (auto &C : LoopOptimizerEndEPCallbacks)
367 C(LPM, Level);
368}
369void PassBuilder::invokeScalarOptimizerLateEPCallbacks(
370 FunctionPassManager &FPM, OptimizationLevel Level) {
371 for (auto &C : ScalarOptimizerLateEPCallbacks)
372 C(FPM, Level);
373}
374void PassBuilder::invokeCGSCCOptimizerLateEPCallbacks(CGSCCPassManager &CGPM,
375 OptimizationLevel Level) {
376 for (auto &C : CGSCCOptimizerLateEPCallbacks)
377 C(CGPM, Level);
378}
379void PassBuilder::invokeVectorizerStartEPCallbacks(FunctionPassManager &FPM,
380 OptimizationLevel Level) {
381 for (auto &C : VectorizerStartEPCallbacks)
382 C(FPM, Level);
383}
384void PassBuilder::invokeVectorizerEndEPCallbacks(FunctionPassManager &FPM,
385 OptimizationLevel Level) {
386 for (auto &C : VectorizerEndEPCallbacks)
387 C(FPM, Level);
388}
389void PassBuilder::invokeOptimizerEarlyEPCallbacks(ModulePassManager &MPM,
390 OptimizationLevel Level,
391 ThinOrFullLTOPhase Phase) {
392 for (auto &C : OptimizerEarlyEPCallbacks)
393 C(MPM, Level, Phase);
394}
395void PassBuilder::invokeOptimizerLastEPCallbacks(ModulePassManager &MPM,
396 OptimizationLevel Level,
397 ThinOrFullLTOPhase Phase) {
398 for (auto &C : OptimizerLastEPCallbacks)
399 C(MPM, Level, Phase);
400}
401void PassBuilder::invokeFullLinkTimeOptimizationEarlyEPCallbacks(
402 ModulePassManager &MPM, OptimizationLevel Level) {
403 for (auto &C : FullLinkTimeOptimizationEarlyEPCallbacks)
404 C(MPM, Level);
405}
406void PassBuilder::invokeFullLinkTimeOptimizationLastEPCallbacks(
407 ModulePassManager &MPM, OptimizationLevel Level) {
408 for (auto &C : FullLinkTimeOptimizationLastEPCallbacks)
409 C(MPM, Level);
410}
411void PassBuilder::invokeThinLinkTimeOptimizationEarlyEPCallbacks(
412 ModulePassManager &MPM, OptimizationLevel Level) {
413 for (auto &C : ThinLinkTimeOptimizationEarlyEPCallbacks)
414 C(MPM, Level);
415}
416void PassBuilder::invokeThinLinkTimeOptimizationLastEPCallbacks(
417 ModulePassManager &MPM, OptimizationLevel Level) {
418 for (auto &C : ThinLinkTimeOptimizationLastEPCallbacks)
419 C(MPM, Level);
420}
421void PassBuilder::invokePipelineStartEPCallbacks(ModulePassManager &MPM,
422 OptimizationLevel Level) {
423 for (auto &C : PipelineStartEPCallbacks)
424 C(MPM, Level);
425}
426void PassBuilder::invokePipelineEarlySimplificationEPCallbacks(
427 ModulePassManager &MPM, OptimizationLevel Level, ThinOrFullLTOPhase Phase) {
428 for (auto &C : PipelineEarlySimplificationEPCallbacks)
429 C(MPM, Level, Phase);
430}
431
432// Get IR stats with InstCount before/after the optimization pipeline
433static void instructionCountersPass(ModulePassManager &MPM,
434 bool IsPreOptimization) {
435 if (AreStatisticsEnabled()) {
436 MPM.addPass(
437 Pass: createModuleToFunctionPassAdaptor(Pass: InstCountPass(IsPreOptimization)));
438 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
439 Pass: FunctionPropertiesStatisticsPass(IsPreOptimization)));
440 }
441}
442
443// Helper to add AnnotationRemarksPass.
444static void addAnnotationRemarksPass(ModulePassManager &MPM) {
445 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: AnnotationRemarksPass()));
446}
447
448// Helper to check if the current compilation phase is preparing for LTO
449static bool isLTOPreLink(ThinOrFullLTOPhase Phase) {
450 return Phase == ThinOrFullLTOPhase::ThinLTOPreLink ||
451 Phase == ThinOrFullLTOPhase::FullLTOPreLink;
452}
453
454// Helper to check if the current compilation phase is preparing for FullLTO
455[[maybe_unused]] static bool isFullLTOPreLink(ThinOrFullLTOPhase Phase) {
456 return Phase == ThinOrFullLTOPhase::FullLTOPreLink;
457}
458
459// Helper to check if the current compilation phase is preparing for ThinLTO
460static bool isThinLTOPreLink(ThinOrFullLTOPhase Phase) {
461 return Phase == ThinOrFullLTOPhase::ThinLTOPreLink;
462}
463
464// Helper to check if the current compilation phase is LTO backend
465static bool isLTOPostLink(ThinOrFullLTOPhase Phase) {
466 return Phase == ThinOrFullLTOPhase::ThinLTOPostLink ||
467 Phase == ThinOrFullLTOPhase::FullLTOPostLink;
468}
469
470// Helper to check if the current compilation phase is FullLTO backend
471static bool isFullLTOPostLink(ThinOrFullLTOPhase Phase) {
472 return Phase == ThinOrFullLTOPhase::FullLTOPostLink;
473}
474
475// Helper to check if the current compilation phase is ThinLTO backend
476static bool isThinLTOPostLink(ThinOrFullLTOPhase Phase) {
477 return Phase == ThinOrFullLTOPhase::ThinLTOPostLink;
478}
479
480// Helper to wrap conditionally Coro passes.
481static CoroConditionalWrapper buildCoroWrapper(ThinOrFullLTOPhase Phase) {
482 // TODO: Skip passes according to Phase.
483 ModulePassManager CoroPM;
484 CoroPM.addPass(Pass: CoroEarlyPass());
485 CGSCCPassManager CGPM;
486 CGPM.addPass(Pass: CoroSplitPass());
487 CoroPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: std::move(CGPM)));
488 CoroPM.addPass(Pass: CoroCleanupPass());
489 CoroPM.addPass(Pass: GlobalDCEPass());
490 return CoroConditionalWrapper(std::move(CoroPM));
491}
492
493// TODO: Investigate the cost/benefit of tail call elimination on debugging.
494FunctionPassManager
495PassBuilder::buildO1FunctionSimplificationPipeline(OptimizationLevel Level,
496 ThinOrFullLTOPhase Phase) {
497
498 FunctionPassManager FPM;
499
500 if (AreStatisticsEnabled())
501 FPM.addPass(Pass: CountVisitsPass());
502
503 // Form SSA out of local memory accesses after breaking apart aggregates into
504 // scalars.
505 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
506
507 // Catch trivial redundancies
508 FPM.addPass(Pass: EarlyCSEPass(true /* Enable mem-ssa. */));
509
510 // Hoisting of scalars and load expressions.
511 FPM.addPass(
512 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
513 FPM.addPass(Pass: InstCombinePass());
514
515 FPM.addPass(Pass: LibCallsShrinkWrapPass());
516
517 invokePeepholeEPCallbacks(FPM, Level);
518
519 FPM.addPass(
520 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
521
522 // Form canonically associated expression trees, and simplify the trees using
523 // basic mathematical properties. For example, this will form (nearly)
524 // minimal multiplication trees.
525 FPM.addPass(Pass: ReassociatePass());
526
527 // Add the primary loop simplification pipeline.
528 // FIXME: Currently this is split into two loop pass pipelines because we run
529 // some function passes in between them. These can and should be removed
530 // and/or replaced by scheduling the loop pass equivalents in the correct
531 // positions. But those equivalent passes aren't powerful enough yet.
532 // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
533 // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
534 // fully replace `SimplifyCFGPass`, and the closest to the other we have is
535 // `LoopInstSimplify`.
536 LoopPassManager LPM1, LPM2;
537
538 // Simplify the loop body. We do this initially to clean up after other loop
539 // passes run, either when iterating on a loop or on inner loops with
540 // implications on the outer loop.
541 LPM1.addPass(Pass: LoopInstSimplifyPass());
542 LPM1.addPass(Pass: LoopSimplifyCFGPass());
543
544 // Try to remove as much code from the loop header as possible,
545 // to reduce amount of IR that will have to be duplicated. However,
546 // do not perform speculative hoisting the first time as LICM
547 // will destroy metadata that may not need to be destroyed if run
548 // after loop rotation.
549 // TODO: Investigate promotion cap for O1.
550 LPM1.addPass(Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
551 /*AllowSpeculation=*/false));
552
553 LPM1.addPass(
554 Pass: LoopRotatePass(/*EnableHeaderDuplication=*/true, isLTOPreLink(Phase)));
555 // TODO: Investigate promotion cap for O1.
556 LPM1.addPass(Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
557 /*AllowSpeculation=*/true));
558 LPM1.addPass(Pass: SimpleLoopUnswitchPass());
559 if (EnableLoopFlatten)
560 LPM1.addPass(Pass: LoopFlattenPass());
561
562 LPM2.addPass(Pass: LoopIdiomRecognizePass());
563 LPM2.addPass(Pass: IndVarSimplifyPass());
564
565 invokeLateLoopOptimizationsEPCallbacks(LPM&: LPM2, Level);
566
567 LPM2.addPass(Pass: LoopDeletionPass());
568
569 // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
570 // because it changes IR to makes profile annotation in back compile
571 // inaccurate. The normal unroller doesn't pay attention to forced full unroll
572 // attributes so we need to make sure and allow the full unroll pass to pay
573 // attention to it.
574 if (!isThinLTOPreLink(Phase) || !PGOOpt ||
575 PGOOpt->Action != PGOOptions::SampleUse)
576 LPM2.addPass(Pass: LoopFullUnrollPass(static_cast<int>(Level),
577 /* OnlyWhenForced= */ !PTO.LoopUnrolling,
578 PTO.ForgetAllSCEVInLoopUnroll));
579
580 invokeLoopOptimizerEndEPCallbacks(LPM&: LPM2, Level);
581
582 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM1),
583 /*UseMemorySSA=*/true));
584 FPM.addPass(
585 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
586 FPM.addPass(Pass: InstCombinePass());
587 // The loop passes in LPM2 (LoopFullUnrollPass) do not preserve MemorySSA.
588 // *All* loop passes must preserve it, in order to be able to use it.
589 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM2),
590 /*UseMemorySSA=*/false));
591
592 // Delete small array after loop unroll.
593 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
594
595 // Specially optimize memory movement as it doesn't look like dataflow in SSA.
596 FPM.addPass(Pass: MemCpyOptPass());
597
598 // Sparse conditional constant propagation.
599 // FIXME: It isn't clear why we do this *after* loop passes rather than
600 // before...
601 FPM.addPass(Pass: SCCPPass());
602
603 // Delete dead bit computations (instcombine runs after to fold away the dead
604 // computations, and then ADCE will run later to exploit any new DCE
605 // opportunities that creates).
606 FPM.addPass(Pass: BDCEPass());
607
608 // Run instcombine after redundancy and dead bit elimination to exploit
609 // opportunities opened up by them.
610 FPM.addPass(Pass: InstCombinePass());
611 invokePeepholeEPCallbacks(FPM, Level);
612
613 FPM.addPass(Pass: CoroElidePass());
614
615 invokeScalarOptimizerLateEPCallbacks(FPM, Level);
616
617 // Finally, do an expensive DCE pass to catch all the dead code exposed by
618 // the simplifications and basic cleanup after all the simplifications.
619 // TODO: Investigate if this is too expensive.
620 FPM.addPass(Pass: ADCEPass());
621 FPM.addPass(
622 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
623 FPM.addPass(Pass: InstCombinePass());
624 invokePeepholeEPCallbacks(FPM, Level);
625
626 return FPM;
627}
628
629FunctionPassManager
630PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
631 ThinOrFullLTOPhase Phase) {
632 assert(Level != OptimizationLevel::O0 && "Must request optimizations!");
633
634 // The O1 pipeline has a separate pipeline creation function to simplify
635 // construction readability.
636 if (Level == OptimizationLevel::O1)
637 return buildO1FunctionSimplificationPipeline(Level, Phase);
638
639 FunctionPassManager FPM;
640
641 if (AreStatisticsEnabled())
642 FPM.addPass(Pass: CountVisitsPass());
643
644 // Form SSA out of local memory accesses after breaking apart aggregates into
645 // scalars.
646 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
647
648 // Catch trivial redundancies
649 FPM.addPass(Pass: EarlyCSEPass(true /* Enable mem-ssa. */));
650 if (EnableKnowledgeRetention)
651 FPM.addPass(Pass: AssumeSimplifyPass());
652
653 // Hoisting of scalars and load expressions.
654 if (EnableGVNHoist)
655 FPM.addPass(Pass: GVNHoistPass());
656
657 // Global value numbering based sinking.
658 if (EnableGVNSink) {
659 FPM.addPass(Pass: GVNSinkPass());
660 FPM.addPass(
661 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
662 }
663
664 // Speculative execution if the target has divergent branches; otherwise nop.
665 FPM.addPass(Pass: SpeculativeExecutionPass(/* OnlyIfDivergentTarget =*/true));
666
667 // Optimize based on known information about branches, and cleanup afterward.
668 FPM.addPass(Pass: JumpThreadingPass());
669 FPM.addPass(Pass: CorrelatedValuePropagationPass());
670
671 // Jump table to switch conversion.
672 if (EnableJumpTableToSwitch)
673 FPM.addPass(Pass: JumpTableToSwitchPass(/*InLTO=*/isLTOPostLink(Phase)));
674
675 FPM.addPass(
676 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
677 FPM.addPass(Pass: InstCombinePass());
678 FPM.addPass(Pass: AggressiveInstCombinePass());
679 FPM.addPass(Pass: LibCallsShrinkWrapPass());
680
681 invokePeepholeEPCallbacks(FPM, Level);
682
683 // For PGO use pipeline, try to optimize memory intrinsics such as memcpy
684 // using the size value profile. Don't perform this when optimizing for size.
685 if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse)
686 FPM.addPass(Pass: PGOMemOPSizeOpt());
687
688 FPM.addPass(Pass: TailCallElimPass(/*UpdateFunctionEntryCount=*/
689 isInstrumentedPGOUse()));
690 FPM.addPass(
691 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
692
693 // Form canonically associated expression trees, and simplify the trees using
694 // basic mathematical properties. For example, this will form (nearly)
695 // minimal multiplication trees.
696 FPM.addPass(Pass: ReassociatePass());
697
698 if (EnableConstraintElimination)
699 FPM.addPass(Pass: ConstraintEliminationPass());
700
701 // Add the primary loop simplification pipeline.
702 // FIXME: Currently this is split into two loop pass pipelines because we run
703 // some function passes in between them. These can and should be removed
704 // and/or replaced by scheduling the loop pass equivalents in the correct
705 // positions. But those equivalent passes aren't powerful enough yet.
706 // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
707 // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
708 // fully replace `SimplifyCFGPass`, and the closest to the other we have is
709 // `LoopInstSimplify`.
710 LoopPassManager LPM1, LPM2;
711
712 // Simplify the loop body. We do this initially to clean up after other loop
713 // passes run, either when iterating on a loop or on inner loops with
714 // implications on the outer loop.
715 LPM1.addPass(Pass: LoopInstSimplifyPass());
716 LPM1.addPass(Pass: LoopSimplifyCFGPass());
717
718 // Try to remove as much code from the loop header as possible,
719 // to reduce amount of IR that will have to be duplicated. However,
720 // do not perform speculative hoisting the first time as LICM
721 // will destroy metadata that may not need to be destroyed if run
722 // after loop rotation.
723 // TODO: Investigate promotion cap for O1.
724 LPM1.addPass(Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
725 /*AllowSpeculation=*/false));
726
727 LPM1.addPass(
728 Pass: LoopRotatePass(/*EnableHeaderDuplication=*/true, isLTOPreLink(Phase)));
729 // TODO: Investigate promotion cap for O1.
730 LPM1.addPass(Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
731 /*AllowSpeculation=*/true));
732 LPM1.addPass(
733 Pass: SimpleLoopUnswitchPass(/* NonTrivial */ Level == OptimizationLevel::O3));
734 if (EnableLoopFlatten)
735 LPM1.addPass(Pass: LoopFlattenPass());
736
737 LPM2.addPass(Pass: LoopIdiomRecognizePass());
738 LPM2.addPass(Pass: IndVarSimplifyPass());
739
740 {
741 ExtraLoopPassManager<ShouldRunExtraSimpleLoopUnswitch> ExtraPasses;
742 ExtraPasses.addPass(Pass: SimpleLoopUnswitchPass(/* NonTrivial */ Level ==
743 OptimizationLevel::O3));
744 LPM2.addPass(Pass: std::move(ExtraPasses));
745 }
746
747 invokeLateLoopOptimizationsEPCallbacks(LPM&: LPM2, Level);
748
749 LPM2.addPass(Pass: LoopDeletionPass());
750
751 // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
752 // because it changes IR to makes profile annotation in back compile
753 // inaccurate. The normal unroller doesn't pay attention to forced full unroll
754 // attributes so we need to make sure and allow the full unroll pass to pay
755 // attention to it.
756 if (!isThinLTOPreLink(Phase) || !PGOOpt ||
757 PGOOpt->Action != PGOOptions::SampleUse)
758 LPM2.addPass(Pass: LoopFullUnrollPass(static_cast<int>(Level),
759 /* OnlyWhenForced= */ !PTO.LoopUnrolling,
760 PTO.ForgetAllSCEVInLoopUnroll));
761
762 invokeLoopOptimizerEndEPCallbacks(LPM&: LPM2, Level);
763
764 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM1),
765 /*UseMemorySSA=*/true));
766 FPM.addPass(
767 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
768 FPM.addPass(Pass: InstCombinePass());
769 // The loop passes in LPM2 (LoopIdiomRecognizePass, IndVarSimplifyPass,
770 // LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA.
771 // *All* loop passes must preserve it, in order to be able to use it.
772 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM2),
773 /*UseMemorySSA=*/false));
774
775 // Delete small array after loop unroll.
776 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
777
778 // Try vectorization/scalarization transforms that are both improvements
779 // themselves and can allow further folds with GVN and InstCombine.
780 FPM.addPass(Pass: VectorCombinePass(/*TryEarlyFoldsOnly=*/true));
781
782 // Eliminate redundancies.
783 FPM.addPass(Pass: MergedLoadStoreMotionPass());
784 if (RunNewGVN)
785 FPM.addPass(Pass: NewGVNPass());
786 else
787 FPM.addPass(Pass: GVNPass());
788
789 // Sparse conditional constant propagation.
790 // FIXME: It isn't clear why we do this *after* loop passes rather than
791 // before...
792 FPM.addPass(Pass: SCCPPass());
793
794 // Delete dead bit computations (instcombine runs after to fold away the dead
795 // computations, and then ADCE will run later to exploit any new DCE
796 // opportunities that creates).
797 FPM.addPass(Pass: BDCEPass());
798
799 // Run instcombine after redundancy and dead bit elimination to exploit
800 // opportunities opened up by them.
801 FPM.addPass(Pass: InstCombinePass());
802 invokePeepholeEPCallbacks(FPM, Level);
803
804 // Re-consider control flow based optimizations after redundancy elimination,
805 // redo DCE, etc.
806 if (EnableDFAJumpThreading)
807 FPM.addPass(Pass: DFAJumpThreadingPass());
808
809 FPM.addPass(Pass: JumpThreadingPass());
810 FPM.addPass(Pass: CorrelatedValuePropagationPass());
811
812 // Finally, do an expensive DCE pass to catch all the dead code exposed by
813 // the simplifications and basic cleanup after all the simplifications.
814 // TODO: Investigate if this is too expensive.
815 FPM.addPass(Pass: ADCEPass());
816
817 // Specially optimize memory movement as it doesn't look like dataflow in SSA.
818 FPM.addPass(Pass: MemCpyOptPass());
819
820 FPM.addPass(Pass: DSEPass());
821 FPM.addPass(Pass: MoveAutoInitPass());
822
823 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(
824 Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
825 /*AllowSpeculation=*/true),
826 /*UseMemorySSA=*/true));
827
828 FPM.addPass(Pass: CoroElidePass());
829
830 invokeScalarOptimizerLateEPCallbacks(FPM, Level);
831
832 FPM.addPass(Pass: SimplifyCFGPass(SimplifyCFGOptions()
833 .convertSwitchRangeToICmp(B: true)
834 .convertSwitchToArithmetic(B: true)
835 .hoistCommonInsts(B: true)
836 .sinkCommonInsts(B: true)));
837 FPM.addPass(Pass: InstCombinePass());
838 invokePeepholeEPCallbacks(FPM, Level);
839
840 return FPM;
841}
842
843void PassBuilder::addRequiredLTOPreLinkPasses(ModulePassManager &MPM) {
844 MPM.addPass(Pass: CanonicalizeAliasesPass());
845 MPM.addPass(Pass: NameAnonGlobalPass());
846 MPM.addPass(Pass: AssignGUIDPass());
847}
848
849void PassBuilder::addPreInlinerPasses(ModulePassManager &MPM,
850 OptimizationLevel Level,
851 ThinOrFullLTOPhase LTOPhase) {
852 assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");
853 if (DisablePreInliner)
854 return;
855 InlineParams IP;
856
857 IP.DefaultThreshold = PreInlineThreshold;
858
859 // FIXME: The hint threshold has the same value used by the regular inliner
860 // when not optimzing for size. This should probably be lowered after
861 // performance testing.
862 // FIXME: this comment is cargo culted from the old pass manager, revisit).
863 IP.HintThreshold = 325;
864 IP.OptSizeHintThreshold = PreInlineThreshold;
865 ModuleInlinerWrapperPass MIWP(
866 IP, /* MandatoryFirst */ true,
867 InlineContext{.LTOPhase: LTOPhase, .Pass: InlinePass::EarlyInliner});
868 CGSCCPassManager &CGPipeline = MIWP.getPM();
869
870 FunctionPassManager FPM;
871 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
872 FPM.addPass(Pass: EarlyCSEPass()); // Catch trivial redundancies.
873 FPM.addPass(Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(
874 B: true))); // Merge & remove basic blocks.
875 FPM.addPass(Pass: InstCombinePass()); // Combine silly sequences.
876 invokePeepholeEPCallbacks(FPM, Level);
877
878 CGPipeline.addPass(Pass: createCGSCCToFunctionPassAdaptor(
879 Pass: std::move(FPM), EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
880
881 MPM.addPass(Pass: std::move(MIWP));
882
883 // Delete anything that is now dead to make sure that we don't instrument
884 // dead code. Instrumentation can end up keeping dead code around and
885 // dramatically increase code size.
886 MPM.addPass(Pass: GlobalDCEPass());
887}
888
889void PassBuilder::addPostPGOLoopRotation(ModulePassManager &MPM,
890 OptimizationLevel Level) {
891 if (EnablePostPGOLoopRotation) {
892 // Disable header duplication in loop rotation at -Oz.
893 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
894 Pass: createFunctionToLoopPassAdaptor(Pass: LoopRotatePass(),
895 /*UseMemorySSA=*/false),
896 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
897 }
898}
899
900void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM,
901 OptimizationLevel Level, bool RunProfileGen,
902 bool IsCS, bool AtomicCounterUpdate,
903 std::string ProfileFile,
904 std::string ProfileRemappingFile) {
905 assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");
906
907 if (!RunProfileGen) {
908 assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
909 MPM.addPass(
910 Pass: PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS, FS));
911 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
912 // RequireAnalysisPass for PSI before subsequent non-module passes.
913 MPM.addPass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
914 return;
915 }
916
917 // Perform PGO instrumentation.
918 MPM.addPass(Pass: PGOInstrumentationGen(IsCS ? PGOInstrumentationType::CSFDO
919 : PGOInstrumentationType::FDO));
920
921 addPostPGOLoopRotation(MPM, Level);
922 // Add the profile lowering pass.
923 InstrProfOptions Options;
924 if (!ProfileFile.empty())
925 Options.InstrProfileOutput = ProfileFile;
926 // Do counter promotion at Level greater than O0.
927 Options.DoCounterPromotion = true;
928 Options.UseBFIInPromotion = IsCS;
929 if (EnableSampledInstr) {
930 Options.Sampling = true;
931 // With sampling, there is little beneifit to enable counter promotion.
932 // But note that sampling does work with counter promotion.
933 Options.DoCounterPromotion = false;
934 }
935 Options.Atomic = AtomicCounterUpdate;
936 MPM.addPass(Pass: InstrProfilingLoweringPass(Options, IsCS));
937}
938
939void PassBuilder::addPGOInstrPassesForO0(ModulePassManager &MPM,
940 bool RunProfileGen, bool IsCS,
941 bool AtomicCounterUpdate,
942 std::string ProfileFile,
943 std::string ProfileRemappingFile) {
944 if (!RunProfileGen) {
945 assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
946 MPM.addPass(
947 Pass: PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS, FS));
948 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
949 // RequireAnalysisPass for PSI before subsequent non-module passes.
950 MPM.addPass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
951 return;
952 }
953
954 // Perform PGO instrumentation.
955 MPM.addPass(Pass: PGOInstrumentationGen(IsCS ? PGOInstrumentationType::CSFDO
956 : PGOInstrumentationType::FDO));
957 // Add the profile lowering pass.
958 InstrProfOptions Options;
959 if (!ProfileFile.empty())
960 Options.InstrProfileOutput = ProfileFile;
961 // Do not do counter promotion at O0.
962 Options.DoCounterPromotion = false;
963 Options.UseBFIInPromotion = IsCS;
964 Options.Atomic = AtomicCounterUpdate;
965 MPM.addPass(Pass: InstrProfilingLoweringPass(Options, IsCS));
966}
967
968static InlineParams getInlineParamsFromOptLevel(OptimizationLevel Level) {
969 return getInlineParamsFromOptLevel(OptLevel: static_cast<unsigned>(Level));
970}
971
972ModuleInlinerWrapperPass
973PassBuilder::buildInlinerPipeline(OptimizationLevel Level,
974 ThinOrFullLTOPhase Phase) {
975 InlineParams IP;
976 if (PTO.InlinerThreshold == -1)
977 IP = ::getInlineParamsFromOptLevel(Level);
978 else
979 IP = getInlineParams(Threshold: PTO.InlinerThreshold);
980 // For PreLinkThinLTO + SamplePGO or PreLinkFullLTO + SamplePGO,
981 // set hot-caller threshold to 0 to disable hot
982 // callsite inline (as much as possible [1]) because it makes
983 // profile annotation in the backend inaccurate.
984 //
985 // [1] Note the cost of a function could be below zero due to erased
986 // prologue / epilogue.
987 if (isLTOPreLink(Phase) && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse)
988 IP.HotCallSiteThreshold = 0;
989
990 if (PGOOpt)
991 IP.EnableDeferral = EnablePGOInlineDeferral;
992
993 ModuleInlinerWrapperPass MIWP(IP, PerformMandatoryInliningsFirst,
994 InlineContext{.LTOPhase: Phase, .Pass: InlinePass::CGSCCInliner},
995 UseInlineAdvisor, MaxDevirtIterations);
996
997 // Require the GlobalsAA analysis for the module so we can query it within
998 // the CGSCC pipeline.
999 if (EnableGlobalAnalyses) {
1000 MIWP.addModulePass(Pass: RequireAnalysisPass<GlobalsAA, Module>());
1001 // Invalidate AAManager so it can be recreated and pick up the newly
1002 // available GlobalsAA.
1003 MIWP.addModulePass(
1004 Pass: createModuleToFunctionPassAdaptor(Pass: InvalidateAnalysisPass<AAManager>()));
1005 }
1006
1007 // Require the ProfileSummaryAnalysis for the module so we can query it within
1008 // the inliner pass.
1009 MIWP.addModulePass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
1010
1011 // Now begin the main postorder CGSCC pipeline.
1012 // FIXME: The current CGSCC pipeline has its origins in the legacy pass
1013 // manager and trying to emulate its precise behavior. Much of this doesn't
1014 // make a lot of sense and we should revisit the core CGSCC structure.
1015 CGSCCPassManager &MainCGPipeline = MIWP.getPM();
1016
1017 // Note: historically, the PruneEH pass was run first to deduce nounwind and
1018 // generally clean up exception handling overhead. It isn't clear this is
1019 // valuable as the inliner doesn't currently care whether it is inlining an
1020 // invoke or a call.
1021
1022 if (AttributorRun & AttributorRunOption::CGSCC)
1023 MainCGPipeline.addPass(Pass: AttributorCGSCCPass());
1024 else if (AttributorRun & AttributorRunOption::CGSCC_LIGHT)
1025 MainCGPipeline.addPass(Pass: AttributorLightCGSCCPass());
1026
1027 // Deduce function attributes. We do another run of this after the function
1028 // simplification pipeline, so this only needs to run when it could affect the
1029 // function simplification pipeline, which is only the case with recursive
1030 // functions.
1031 MainCGPipeline.addPass(Pass: PostOrderFunctionAttrsPass(/*SkipNonRecursive*/ true));
1032
1033 // When at O3 add argument promotion to the pass pipeline.
1034 // FIXME: It isn't at all clear why this should be limited to O3.
1035 if (Level == OptimizationLevel::O3)
1036 MainCGPipeline.addPass(Pass: ArgumentPromotionPass());
1037
1038 // Try to perform OpenMP specific optimizations. This is a (quick!) no-op if
1039 // there are no OpenMP runtime calls present in the module.
1040 if (Level == OptimizationLevel::O2 || Level == OptimizationLevel::O3)
1041 MainCGPipeline.addPass(Pass: OpenMPOptCGSCCPass(Phase));
1042
1043 invokeCGSCCOptimizerLateEPCallbacks(CGPM&: MainCGPipeline, Level);
1044
1045 // Add the core function simplification pipeline nested inside the
1046 // CGSCC walk.
1047 MainCGPipeline.addPass(Pass: createCGSCCToFunctionPassAdaptor(
1048 Pass: buildFunctionSimplificationPipeline(Level, Phase),
1049 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses, /*NoRerun=*/true));
1050
1051 // Finally, deduce any function attributes based on the fully simplified
1052 // function.
1053 MainCGPipeline.addPass(Pass: PostOrderFunctionAttrsPass());
1054
1055 // Mark that the function is fully simplified and that it shouldn't be
1056 // simplified again if we somehow revisit it due to CGSCC mutations unless
1057 // it's been modified since.
1058 MainCGPipeline.addPass(Pass: createCGSCCToFunctionPassAdaptor(
1059 Pass: RequireAnalysisPass<ShouldNotRunFunctionPassesAnalysis, Function>()));
1060
1061 if (!isThinLTOPreLink(Phase)) {
1062 MainCGPipeline.addPass(Pass: CoroSplitPass(Level != OptimizationLevel::O0));
1063 MainCGPipeline.addPass(Pass: CoroAnnotationElidePass());
1064 }
1065
1066 // Make sure we don't affect potential future NoRerun CGSCC adaptors.
1067 MIWP.addLateModulePass(Pass: createModuleToFunctionPassAdaptor(
1068 Pass: InvalidateAnalysisPass<ShouldNotRunFunctionPassesAnalysis>()));
1069
1070 return MIWP;
1071}
1072
1073ModulePassManager
1074PassBuilder::buildModuleInlinerPipeline(OptimizationLevel Level,
1075 ThinOrFullLTOPhase Phase) {
1076 ModulePassManager MPM;
1077
1078 InlineParams IP = ::getInlineParamsFromOptLevel(Level);
1079 // For PreLinkThinLTO + SamplePGO or PreLinkFullLTO + SamplePGO,
1080 // set hot-caller threshold to 0 to disable hot
1081 // callsite inline (as much as possible [1]) because it makes
1082 // profile annotation in the backend inaccurate.
1083 //
1084 // [1] Note the cost of a function could be below zero due to erased
1085 // prologue / epilogue.
1086 if (isLTOPreLink(Phase) && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse)
1087 IP.HotCallSiteThreshold = 0;
1088
1089 if (PGOOpt)
1090 IP.EnableDeferral = EnablePGOInlineDeferral;
1091
1092 // The inline deferral logic is used to avoid losing some
1093 // inlining chance in future. It is helpful in SCC inliner, in which
1094 // inlining is processed in bottom-up order.
1095 // While in module inliner, the inlining order is a priority-based order
1096 // by default. The inline deferral is unnecessary there. So we disable the
1097 // inline deferral logic in module inliner.
1098 IP.EnableDeferral = false;
1099
1100 MPM.addPass(Pass: ModuleInlinerPass(IP, UseInlineAdvisor, Phase));
1101 if (!UseCtxProfile.empty() && Phase == ThinOrFullLTOPhase::ThinLTOPostLink) {
1102 MPM.addPass(Pass: GlobalOptPass());
1103 MPM.addPass(Pass: GlobalDCEPass());
1104 MPM.addPass(Pass: AssignGUIDPass());
1105 MPM.addPass(Pass: PGOCtxProfFlatteningPass(/*IsPreThinlink=*/false));
1106 }
1107
1108 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
1109 Pass: buildFunctionSimplificationPipeline(Level, Phase),
1110 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
1111
1112 if (!isThinLTOPreLink(Phase)) {
1113 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(
1114 Pass: CoroSplitPass(Level != OptimizationLevel::O0)));
1115 MPM.addPass(
1116 Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: CoroAnnotationElidePass()));
1117 }
1118
1119 return MPM;
1120}
1121
1122ModulePassManager
1123PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
1124 ThinOrFullLTOPhase Phase) {
1125 assert(Level != OptimizationLevel::O0 &&
1126 "Should not be used for O0 pipeline");
1127
1128 assert(!isFullLTOPostLink(Phase) &&
1129 "FullLTOPostLink shouldn't call buildModuleSimplificationPipeline!");
1130
1131 ModulePassManager MPM;
1132
1133 // Place pseudo probe instrumentation as the first pass of the pipeline to
1134 // minimize the impact of optimization changes.
1135 if (PGOOpt && PGOOpt->PseudoProbeForProfiling && !isThinLTOPostLink(Phase))
1136 MPM.addPass(Pass: SampleProfileProbePass(TM));
1137
1138 bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse);
1139
1140 // In ThinLTO mode, when flattened profile is used, all the available
1141 // profile information will be annotated in PreLink phase so there is
1142 // no need to load the profile again in PostLink.
1143 bool LoadSampleProfile =
1144 HasSampleProfile && !(FlattenedProfileUsed && isThinLTOPostLink(Phase));
1145
1146 // During the ThinLTO backend phase we perform early indirect call promotion
1147 // here, before globalopt. Otherwise imported available_externally functions
1148 // look unreferenced and are removed. If we are going to load the sample
1149 // profile then defer until later.
1150 // TODO: See if we can move later and consolidate with the location where
1151 // we perform ICP when we are loading a sample profile.
1152 // TODO: We pass HasSampleProfile (whether there was a sample profile file
1153 // passed to the compile) to the SamplePGO flag of ICP. This is used to
1154 // determine whether the new direct calls are annotated with prof metadata.
1155 // Ideally this should be determined from whether the IR is annotated with
1156 // sample profile, and not whether the a sample profile was provided on the
1157 // command line. E.g. for flattened profiles where we will not be reloading
1158 // the sample profile in the ThinLTO backend, we ideally shouldn't have to
1159 // provide the sample profile file.
1160 if (isThinLTOPostLink(Phase) && !LoadSampleProfile)
1161 MPM.addPass(Pass: PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile));
1162
1163 // Create an early function pass manager to cleanup the output of the
1164 // frontend. Not necessary with LTO post link pipelines since the pre link
1165 // pipeline already cleaned up the frontend output.
1166 if (!isThinLTOPostLink(Phase)) {
1167 // Do basic inference of function attributes from known properties of system
1168 // libraries and other oracles.
1169 MPM.addPass(Pass: InferFunctionAttrsPass());
1170 MPM.addPass(Pass: CoroEarlyPass());
1171
1172 FunctionPassManager EarlyFPM;
1173 EarlyFPM.addPass(Pass: EntryExitInstrumenterPass(/*PostInlining=*/false));
1174 // Lower llvm.expect to metadata before attempting transforms.
1175 // Compare/branch metadata may alter the behavior of passes like
1176 // SimplifyCFG.
1177 EarlyFPM.addPass(Pass: LowerExpectIntrinsicPass());
1178 EarlyFPM.addPass(Pass: SimplifyCFGPass());
1179 EarlyFPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
1180 EarlyFPM.addPass(Pass: EarlyCSEPass());
1181 if (Level == OptimizationLevel::O3)
1182 EarlyFPM.addPass(Pass: CallSiteSplittingPass());
1183 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
1184 Pass: std::move(EarlyFPM), EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
1185 }
1186
1187 if (LoadSampleProfile) {
1188 // Annotate sample profile right after early FPM to ensure freshness of
1189 // the debug info.
1190 MPM.addPass(Pass: SampleProfileLoaderPass(
1191 PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile, Phase, FS));
1192 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1193 // RequireAnalysisPass for PSI before subsequent non-module passes.
1194 MPM.addPass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
1195 // Do not invoke ICP in the LTOPrelink phase as it makes it hard
1196 // for the profile annotation to be accurate in the LTO backend.
1197 if (!isLTOPreLink(Phase))
1198 // We perform early indirect call promotion here, before globalopt.
1199 // This is important for the ThinLTO backend phase because otherwise
1200 // imported available_externally functions look unreferenced and are
1201 // removed.
1202 MPM.addPass(
1203 Pass: PGOIndirectCallPromotion(true /* IsInLTO */, true /* SamplePGO */));
1204 }
1205
1206 // Try to perform OpenMP specific optimizations on the module. This is a
1207 // (quick!) no-op if there are no OpenMP runtime calls present in the module.
1208 MPM.addPass(Pass: OpenMPOptPass(Phase));
1209
1210 if (AttributorRun & AttributorRunOption::MODULE)
1211 MPM.addPass(Pass: AttributorPass());
1212 else if (AttributorRun & AttributorRunOption::MODULE_LIGHT)
1213 MPM.addPass(Pass: AttributorLightPass());
1214
1215 // Lower type metadata and the type.test intrinsic in the ThinLTO
1216 // post link pipeline after ICP. This is to enable usage of the type
1217 // tests in ICP sequences.
1218 if (isThinLTOPostLink(Phase))
1219 MPM.addPass(Pass: DropTypeTestsPass());
1220
1221 invokePipelineEarlySimplificationEPCallbacks(MPM, Level, Phase);
1222
1223 // Interprocedural constant propagation now that basic cleanup has occurred
1224 // and prior to optimizing globals.
1225 // FIXME: This position in the pipeline hasn't been carefully considered in
1226 // years, it should be re-analyzed.
1227 MPM.addPass(
1228 Pass: IPSCCPPass(IPSCCPOptions(/*AllowFuncSpec=*/!isLTOPreLink(Phase))));
1229
1230 // Attach metadata to indirect call sites indicating the set of functions
1231 // they may target at run-time. This should follow IPSCCP.
1232 MPM.addPass(Pass: CalledValuePropagationPass());
1233
1234 // Optimize globals to try and fold them into constants.
1235 MPM.addPass(Pass: GlobalOptPass());
1236
1237 // Create a small function pass pipeline to cleanup after all the global
1238 // optimizations.
1239 FunctionPassManager GlobalCleanupPM;
1240 // FIXME: Should this instead by a run of SROA?
1241 GlobalCleanupPM.addPass(Pass: PromotePass());
1242 GlobalCleanupPM.addPass(Pass: InstCombinePass());
1243 invokePeepholeEPCallbacks(FPM&: GlobalCleanupPM, Level);
1244 GlobalCleanupPM.addPass(
1245 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
1246 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(GlobalCleanupPM),
1247 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
1248
1249 // We already asserted this happens in non-FullLTOPostLink earlier.
1250 const bool IsPreLink = !isThinLTOPostLink(Phase);
1251 // Enable contextual profiling instrumentation.
1252 const bool IsCtxProfGen =
1253 IsPreLink && PGOCtxProfLoweringPass::isCtxIRPGOInstrEnabled();
1254 const bool IsPGOPreLink = !IsCtxProfGen && PGOOpt && IsPreLink;
1255 const bool IsPGOInstrGen =
1256 IsPGOPreLink && PGOOpt->Action == PGOOptions::IRInstr;
1257 const bool IsPGOInstrUse =
1258 IsPGOPreLink && PGOOpt->Action == PGOOptions::IRUse;
1259 const bool IsMemprofUse = IsPGOPreLink && !PGOOpt->MemoryProfile.empty();
1260 // We don't want to mix pgo ctx gen and pgo gen; we also don't currently
1261 // enable ctx profiling from the frontend.
1262 assert(!(IsPGOInstrGen && PGOCtxProfLoweringPass::isCtxIRPGOInstrEnabled()) &&
1263 "Enabling both instrumented PGO and contextual instrumentation is not "
1264 "supported.");
1265 const bool IsCtxProfUse = !UseCtxProfile.empty() && isThinLTOPreLink(Phase);
1266
1267 assert(
1268 (InstrumentColdFuncOnlyPath.empty() || PGOInstrumentColdFunctionOnly) &&
1269 "--instrument-cold-function-only-path is provided but "
1270 "--pgo-instrument-cold-function-only is not enabled");
1271 const bool IsColdFuncOnlyInstrGen = PGOInstrumentColdFunctionOnly &&
1272 IsPGOPreLink &&
1273 !InstrumentColdFuncOnlyPath.empty();
1274
1275 if (IsPGOInstrGen || IsPGOInstrUse || IsMemprofUse || IsCtxProfGen ||
1276 IsCtxProfUse || IsColdFuncOnlyInstrGen)
1277 addPreInlinerPasses(MPM, Level, LTOPhase: Phase);
1278
1279 // Add all the requested passes for instrumentation PGO, if requested.
1280 if (IsPGOInstrGen || IsPGOInstrUse) {
1281 addPGOInstrPasses(MPM, Level,
1282 /*RunProfileGen=*/IsPGOInstrGen,
1283 /*IsCS=*/false, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate,
1284 ProfileFile: PGOOpt->ProfileFile, ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
1285 } else if (IsCtxProfGen || IsCtxProfUse) {
1286 MPM.addPass(Pass: PGOInstrumentationGen(PGOInstrumentationType::CTXPROF));
1287 // In pre-link, we just want the instrumented IR. We use the contextual
1288 // profile in the post-thinlink phase.
1289 // The instrumentation will be removed in post-thinlink after IPO.
1290 if (IsCtxProfUse) {
1291 MPM.addPass(Pass: AssignGUIDPass());
1292 MPM.addPass(Pass: PGOCtxProfFlatteningPass(/*IsPreThinlink=*/true));
1293 return MPM;
1294 }
1295 // Block further inlining in the instrumented ctxprof case. This avoids
1296 // confusingly collecting profiles for the same GUID corresponding to
1297 // different variants of the function. We could do like PGO and identify
1298 // functions by a (GUID, Hash) tuple, but since the ctxprof "use" waits for
1299 // thinlto to happen before performing any further optimizations, it's
1300 // unnecessary to collect profiles for non-prevailing copies.
1301 MPM.addPass(Pass: NoinlineNonPrevailing());
1302 addPostPGOLoopRotation(MPM, Level);
1303 MPM.addPass(Pass: AssignGUIDPass());
1304 MPM.addPass(Pass: PGOCtxProfLoweringPass());
1305 } else if (IsColdFuncOnlyInstrGen) {
1306 addPGOInstrPasses(MPM, Level, /* RunProfileGen */ true, /* IsCS */ false,
1307 /* AtomicCounterUpdate */ false,
1308 ProfileFile: InstrumentColdFuncOnlyPath,
1309 /* ProfileRemappingFile */ "");
1310 }
1311
1312 if (IsPGOInstrGen || IsPGOInstrUse || IsCtxProfGen)
1313 MPM.addPass(Pass: PGOIndirectCallPromotion(false, false));
1314
1315 if (IsPGOPreLink && PGOOpt->CSAction == PGOOptions::CSIRInstr)
1316 MPM.addPass(Pass: PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile,
1317 EnableSampledInstr));
1318
1319 if (IsMemprofUse)
1320 MPM.addPass(Pass: MemProfUsePass(PGOOpt->MemoryProfile, FS));
1321
1322 if (PGOOpt && (PGOOpt->Action == PGOOptions::IRUse ||
1323 PGOOpt->Action == PGOOptions::SampleUse))
1324 MPM.addPass(Pass: PGOForceFunctionAttrsPass(PGOOpt->ColdOptType));
1325
1326 MPM.addPass(Pass: AlwaysInlinerPass(/*InsertLifetimeIntrinsics=*/true));
1327
1328 if (EnableModuleInliner)
1329 MPM.addPass(Pass: buildModuleInlinerPipeline(Level, Phase));
1330 else
1331 MPM.addPass(Pass: buildInlinerPipeline(Level, Phase));
1332
1333 // Remove any dead arguments exposed by cleanups, constant folding globals,
1334 // and argument promotion.
1335 MPM.addPass(Pass: DeadArgumentEliminationPass());
1336
1337 if (isThinLTOPostLink(Phase))
1338 MPM.addPass(Pass: SimplifyTypeTestsPass());
1339
1340 if (!isThinLTOPreLink(Phase))
1341 MPM.addPass(Pass: CoroCleanupPass());
1342
1343 // Optimize globals now that functions are fully simplified.
1344 MPM.addPass(Pass: GlobalOptPass());
1345 MPM.addPass(Pass: GlobalDCEPass());
1346
1347 return MPM;
1348}
1349
1350/// TODO: Should LTO cause any differences to this set of passes?
1351void PassBuilder::addVectorPasses(OptimizationLevel Level,
1352 FunctionPassManager &FPM,
1353 ThinOrFullLTOPhase LTOPhase) {
1354 FPM.addPass(Pass: LoopVectorizePass(
1355 LoopVectorizeOptions(!PTO.LoopInterleaving, !PTO.LoopVectorization)));
1356
1357 // Drop dereferenceable assumes after vectorization, as they are no longer
1358 // needed and can inhibit further optimization.
1359 if (!isLTOPreLink(Phase: LTOPhase))
1360 FPM.addPass(Pass: DropUnnecessaryAssumesPass(/*DropDereferenceable=*/true));
1361
1362 FPM.addPass(Pass: InferAlignmentPass());
1363 if (isFullLTOPostLink(Phase: LTOPhase)) {
1364 // The vectorizer may have significantly shortened a loop body; unroll
1365 // again. Unroll small loops to hide loop backedge latency and saturate any
1366 // parallel execution resources of an out-of-order processor. We also then
1367 // need to clean up redundancies and loop invariant code.
1368 // FIXME: It would be really good to use a loop-integrated instruction
1369 // combiner for cleanup here so that the unrolling and LICM can be pipelined
1370 // across the loop nests.
1371 // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1372 if (EnableUnrollAndJam && PTO.LoopUnrolling)
1373 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(
1374 Pass: LoopUnrollAndJamPass(static_cast<int>(Level))));
1375 FPM.addPass(Pass: LoopUnrollPass(LoopUnrollOptions(
1376 static_cast<int>(Level), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1377 PTO.ForgetAllSCEVInLoopUnroll)));
1378 FPM.addPass(Pass: WarnMissedTransformationsPass());
1379 // Now that we are done with loop unrolling, be it either by LoopVectorizer,
1380 // or LoopUnroll passes, some variable-offset GEP's into alloca's could have
1381 // become constant-offset, thus enabling SROA and alloca promotion. Do so.
1382 // NOTE: we are very late in the pipeline, and we don't have any LICM
1383 // or SimplifyCFG passes scheduled after us, that would cleanup
1384 // the CFG mess this may created if allowed to modify CFG, so forbid that.
1385
1386 // We also turn on struct to vector canonicalization here, which allows
1387 // converting allocas of homogeneous structs into vector allocas when the
1388 // allocas' users are all memory intrinsics. This allows promotion in some
1389 // cases because structs cannot promote to SSA values, but vectors can. We
1390 // only turn this on after memcpyopt runs because this might hinder
1391 // memcpyopt's optimizations if done before. Look at the documentation for
1392 // `tryCanonicalizeStructToVector` in SROA.cpp to see why.
1393 FPM.addPass(Pass: SROAPass(SROAOptions(SROAOptions::PreserveCFG,
1394 /*AggregateToVector=*/true)));
1395 }
1396
1397 if (!isFullLTOPostLink(Phase: LTOPhase)) {
1398 // Eliminate loads by forwarding stores from the previous iteration to loads
1399 // of the current iteration.
1400 FPM.addPass(Pass: LoopLoadEliminationPass());
1401 }
1402 // Cleanup after the loop optimization passes.
1403 FPM.addPass(Pass: InstCombinePass());
1404
1405 if (Level > OptimizationLevel::O1 && ExtraVectorizerPasses) {
1406 ExtraFunctionPassManager<ShouldRunExtraVectorPasses> ExtraPasses;
1407 // At higher optimization levels, try to clean up any runtime overlap and
1408 // alignment checks inserted by the vectorizer. We want to track correlated
1409 // runtime checks for two inner loops in the same outer loop, fold any
1410 // common computations, hoist loop-invariant aspects out of any outer loop,
1411 // and unswitch the runtime checks if possible. Once hoisted, we may have
1412 // dead (or speculatable) control flows or more combining opportunities.
1413 ExtraPasses.addPass(Pass: EarlyCSEPass());
1414 ExtraPasses.addPass(Pass: CorrelatedValuePropagationPass());
1415 ExtraPasses.addPass(Pass: InstCombinePass());
1416 LoopPassManager LPM;
1417 LPM.addPass(Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
1418 /*AllowSpeculation=*/true));
1419 LPM.addPass(Pass: SimpleLoopUnswitchPass(/* NonTrivial */ Level ==
1420 OptimizationLevel::O3));
1421 ExtraPasses.addPass(
1422 Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM), /*UseMemorySSA=*/true));
1423 ExtraPasses.addPass(
1424 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
1425 ExtraPasses.addPass(Pass: InstCombinePass());
1426 FPM.addPass(Pass: std::move(ExtraPasses));
1427 }
1428
1429 // Now that we've formed fast to execute loop structures, we do further
1430 // optimizations. These are run afterward as they might block doing complex
1431 // analyses and transforms such as what are needed for loop vectorization.
1432
1433 // Cleanup after loop vectorization, etc. Simplification passes like CVP and
1434 // GVN, loop transforms, and others have already run, so it's now better to
1435 // convert to more optimized IR using more aggressive simplify CFG options.
1436 // The extra sinking transform can create larger basic blocks, so do this
1437 // before SLP vectorization.
1438 FPM.addPass(Pass: SimplifyCFGPass(SimplifyCFGOptions()
1439 .forwardSwitchCondToPhi(B: true)
1440 .convertSwitchRangeToICmp(B: true)
1441 .convertSwitchToArithmetic(B: true)
1442 .convertSwitchToLookupTable(B: true)
1443 .needCanonicalLoops(B: false)
1444 .hoistCommonInsts(B: true)
1445 .sinkCommonInsts(B: true)));
1446
1447 if (isFullLTOPostLink(Phase: LTOPhase)) {
1448 FPM.addPass(Pass: SCCPPass());
1449 FPM.addPass(Pass: InstCombinePass());
1450 FPM.addPass(Pass: BDCEPass());
1451 }
1452
1453 // Optimize parallel scalar instruction chains into SIMD instructions.
1454 if (PTO.SLPVectorization) {
1455 FPM.addPass(Pass: SLPVectorizerPass());
1456 if (Level >= OptimizationLevel::O2 && ExtraVectorizerPasses) {
1457 FPM.addPass(Pass: EarlyCSEPass());
1458 }
1459 }
1460 // Enhance/cleanup vector code.
1461 FPM.addPass(Pass: VectorCombinePass());
1462
1463 if (!isFullLTOPostLink(Phase: LTOPhase)) {
1464 FPM.addPass(Pass: InstCombinePass());
1465 // Unroll small loops to hide loop backedge latency and saturate any
1466 // parallel execution resources of an out-of-order processor. We also then
1467 // need to clean up redundancies and loop invariant code.
1468 // FIXME: It would be really good to use a loop-integrated instruction
1469 // combiner for cleanup here so that the unrolling and LICM can be pipelined
1470 // across the loop nests.
1471 // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1472 if (EnableUnrollAndJam && PTO.LoopUnrolling) {
1473 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(
1474 Pass: LoopUnrollAndJamPass(static_cast<int>(Level))));
1475 }
1476 FPM.addPass(Pass: LoopUnrollPass(LoopUnrollOptions(
1477 static_cast<int>(Level), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1478 PTO.ForgetAllSCEVInLoopUnroll)));
1479 FPM.addPass(Pass: WarnMissedTransformationsPass());
1480 // Now that we are done with loop unrolling, be it either by LoopVectorizer,
1481 // or LoopUnroll passes, some variable-offset GEP's into alloca's could have
1482 // become constant-offset, thus enabling SROA and alloca promotion. Do so.
1483 // NOTE: we are very late in the pipeline, and we don't have any LICM
1484 // or SimplifyCFG passes scheduled after us, that would cleanup
1485 // the CFG mess this may created if allowed to modify CFG, so forbid that.
1486
1487 // We also turn on struct to vector canonicalization here, which allows
1488 // converting allocas of homogeneous structs into vector allocas when the
1489 // allocas' users are all memory intrinsics. This allows promotion in some
1490 // cases because structs cannot promote to SSA values, but vectors can. We
1491 // only turn this on after memcpyopt runs because this might hinder
1492 // memcpyopt's optimizations if done before. Look at the documentation for
1493 // `tryCanonicalizeStructToVector` in SROA.cpp to see why.
1494 FPM.addPass(Pass: SROAPass(SROAOptions(SROAOptions::PreserveCFG,
1495 /*AggregateToVector=*/true)));
1496 }
1497
1498 FPM.addPass(Pass: InferAlignmentPass());
1499 FPM.addPass(Pass: InstCombinePass());
1500
1501 // This is needed for two reasons:
1502 // 1. It works around problems that instcombine introduces, such as sinking
1503 // expensive FP divides into loops containing multiplications using the
1504 // divide result.
1505 // 2. It helps to clean up some loop-invariant code created by the loop
1506 // unroll pass when IsFullLTO=false.
1507 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(
1508 Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
1509 /*AllowSpeculation=*/true),
1510 /*UseMemorySSA=*/true));
1511
1512 // Now that we've vectorized and unrolled loops, we may have more refined
1513 // alignment information, try to re-derive it here.
1514 FPM.addPass(Pass: AlignmentFromAssumptionsPass());
1515}
1516
1517ModulePassManager
1518PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
1519 ThinOrFullLTOPhase LTOPhase) {
1520 ModulePassManager MPM;
1521
1522 // Run partial inlining pass to partially inline functions that have
1523 // large bodies.
1524 if (RunPartialInlining)
1525 MPM.addPass(Pass: PartialInlinerPass());
1526
1527 // Remove avail extern fns and globals definitions since we aren't compiling
1528 // an object file for later LTO. For LTO we want to preserve these so they
1529 // are eligible for inlining at link-time. Note if they are unreferenced they
1530 // will be removed by GlobalDCE later, so this only impacts referenced
1531 // available externally globals. Eventually they will be suppressed during
1532 // codegen, but eliminating here enables more opportunity for GlobalDCE as it
1533 // may make globals referenced by available external functions dead and saves
1534 // running remaining passes on the eliminated functions. These should be
1535 // preserved during prelinking for link-time inlining decisions.
1536 if (!isLTOPreLink(Phase: LTOPhase))
1537 MPM.addPass(Pass: EliminateAvailableExternallyPass());
1538
1539 // Do RPO function attribute inference across the module to forward-propagate
1540 // attributes where applicable.
1541 // FIXME: Is this really an optimization rather than a canonicalization?
1542 MPM.addPass(Pass: ReversePostOrderFunctionAttrsPass());
1543
1544 // Do a post inline PGO instrumentation and use pass. This is a context
1545 // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as
1546 // cross-module inline has not been done yet. The context sensitive
1547 // instrumentation is after all the inlines are done.
1548 if (!isLTOPreLink(Phase: LTOPhase) && PGOOpt) {
1549 if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1550 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/true,
1551 /*IsCS=*/true, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate,
1552 ProfileFile: PGOOpt->CSProfileGenFile, ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
1553 else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1554 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/false,
1555 /*IsCS=*/true, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate,
1556 ProfileFile: PGOOpt->ProfileFile, ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
1557 }
1558
1559 // Re-compute GlobalsAA here prior to function passes. This is particularly
1560 // useful as the above will have inlined, DCE'ed, and function-attr
1561 // propagated everything. We should at this point have a reasonably minimal
1562 // and richly annotated call graph. By computing aliasing and mod/ref
1563 // information for all local globals here, the late loop passes and notably
1564 // the vectorizer will be able to use them to help recognize vectorizable
1565 // memory operations.
1566 if (EnableGlobalAnalyses)
1567 MPM.addPass(Pass: RecomputeGlobalsAAPass());
1568
1569 invokeOptimizerEarlyEPCallbacks(MPM, Level, Phase: LTOPhase);
1570
1571 FunctionPassManager OptimizePM;
1572
1573 // Only drop unnecessary assumes post-inline and post-link, as otherwise
1574 // additional uses of the affected value may be introduced through inlining
1575 // and CSE.
1576 if (!isLTOPreLink(Phase: LTOPhase))
1577 OptimizePM.addPass(Pass: DropUnnecessaryAssumesPass());
1578
1579 // Scheduling LoopVersioningLICM when inlining is over, because after that
1580 // we may see more accurate aliasing. Reason to run this late is that too
1581 // early versioning may prevent further inlining due to increase of code
1582 // size. Other optimizations which runs later might get benefit of no-alias
1583 // assumption in clone loop.
1584 if (UseLoopVersioningLICM) {
1585 OptimizePM.addPass(
1586 Pass: createFunctionToLoopPassAdaptor(Pass: LoopVersioningLICMPass()));
1587 // LoopVersioningLICM pass might increase new LICM opportunities.
1588 OptimizePM.addPass(Pass: createFunctionToLoopPassAdaptor(
1589 Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
1590 /*AllowSpeculation=*/true),
1591 /*USeMemorySSA=*/UseMemorySSA: true));
1592 }
1593
1594 OptimizePM.addPass(Pass: Float2IntPass());
1595 // Defer until LTO post-link where some constants may become known.
1596 if (!isLTOPreLink(Phase: LTOPhase))
1597 OptimizePM.addPass(Pass: LowerConstantIntrinsicsPass());
1598
1599 if (EnableMatrix) {
1600 OptimizePM.addPass(Pass: LowerMatrixIntrinsicsPass());
1601 OptimizePM.addPass(Pass: EarlyCSEPass());
1602 }
1603
1604 // CHR pass should only be applied with the profile information.
1605 // The check is to check the profile summary information in CHR.
1606 if (EnableCHR && Level == OptimizationLevel::O3)
1607 OptimizePM.addPass(Pass: ControlHeightReductionPass());
1608
1609 // FIXME: We need to run some loop optimizations to re-rotate loops after
1610 // simplifycfg and others undo their rotation.
1611
1612 // Optimize the loop execution. These passes operate on entire loop nests
1613 // rather than on each loop in an inside-out manner, and so they are actually
1614 // function passes.
1615
1616 invokeVectorizerStartEPCallbacks(FPM&: OptimizePM, Level);
1617
1618 LoopPassManager LPM;
1619 // First rotate loops that may have been un-rotated by prior passes.
1620 // Disable header duplication at -Oz.
1621 LPM.addPass(Pass: LoopRotatePass(/*EnableLoopHeaderDuplication=*/true,
1622 isLTOPreLink(Phase: LTOPhase),
1623 /*CheckExitCount=*/true));
1624 // Some loops may have become dead by now. Try to delete them.
1625 // FIXME: see discussion in https://reviews.llvm.org/D112851,
1626 // this may need to be revisited once we run GVN before loop deletion
1627 // in the simplification pipeline.
1628 LPM.addPass(Pass: LoopDeletionPass());
1629
1630 if (PTO.LoopInterchange)
1631 LPM.addPass(Pass: LoopInterchangePass());
1632
1633 OptimizePM.addPass(
1634 Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM), /*UseMemorySSA=*/false));
1635
1636 // FIXME: This may not be the right place in the pipeline.
1637 // We need to have the data to support the right place.
1638 if (PTO.LoopFusion)
1639 OptimizePM.addPass(Pass: LoopFusePass());
1640
1641 // Distribute loops to allow partial vectorization. I.e. isolate dependences
1642 // into separate loop that would otherwise inhibit vectorization. This is
1643 // currently only performed for loops marked with the metadata
1644 // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
1645 OptimizePM.addPass(Pass: LoopDistributePass());
1646
1647 // Populates the VFABI attribute with the scalar-to-vector mappings
1648 // from the TargetLibraryInfo.
1649 OptimizePM.addPass(Pass: InjectTLIMappings());
1650
1651 addVectorPasses(Level, FPM&: OptimizePM, LTOPhase);
1652
1653 invokeVectorizerEndEPCallbacks(FPM&: OptimizePM, Level);
1654
1655 // LoopSink pass sinks instructions hoisted by LICM, which serves as a
1656 // canonicalization pass that enables other optimizations. As a result,
1657 // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
1658 // result too early.
1659 OptimizePM.addPass(Pass: LoopSinkPass());
1660
1661 // And finally clean up LCSSA form before generating code.
1662 OptimizePM.addPass(Pass: InstSimplifyPass());
1663
1664 // This hoists/decomposes div/rem ops. It should run after other sink/hoist
1665 // passes to avoid re-sinking, but before SimplifyCFG because it can allow
1666 // flattening of blocks.
1667 OptimizePM.addPass(Pass: DivRemPairsPass());
1668
1669 // Merge adjacent icmps into memcmp, then expand memcmp to loads/compares.
1670 // TODO: move this furter up so that it can be optimized by GVN, etc.
1671 if (EnableMergeICmps)
1672 OptimizePM.addPass(Pass: MergeICmpsPass());
1673 OptimizePM.addPass(Pass: ExpandMemCmpPass());
1674
1675 // Try to annotate calls that were created during optimization.
1676 OptimizePM.addPass(
1677 Pass: TailCallElimPass(/*UpdateFunctionEntryCount=*/isInstrumentedPGOUse()));
1678
1679 // LoopSink (and other loop passes since the last simplifyCFG) might have
1680 // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
1681 OptimizePM.addPass(
1682 Pass: SimplifyCFGPass(SimplifyCFGOptions()
1683 .convertSwitchRangeToICmp(B: true)
1684 .convertSwitchToArithmetic(B: true)
1685 .speculateUnpredictables(B: true)
1686 .hoistLoadsStoresWithCondFaulting(B: true)));
1687
1688 // Add the core optimizing pipeline.
1689 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(OptimizePM),
1690 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
1691
1692 // AllocToken transforms heap allocation calls; this needs to run late after
1693 // other allocation call transformations (such as those in InstCombine).
1694 if (!isLTOPreLink(Phase: LTOPhase))
1695 MPM.addPass(Pass: AllocTokenPass());
1696
1697 invokeOptimizerLastEPCallbacks(MPM, Level, Phase: LTOPhase);
1698
1699 // Run the Instrumentor pass late.
1700 if (EnableInstrumentor)
1701 MPM.addPass(Pass: InstrumentorPass(FS));
1702
1703 // Split out cold code. Splitting is done late to avoid hiding context from
1704 // other optimizations and inadvertently regressing performance. The tradeoff
1705 // is that this has a higher code size cost than splitting early.
1706 if (EnableHotColdSplit && !isLTOPreLink(Phase: LTOPhase))
1707 MPM.addPass(Pass: HotColdSplittingPass());
1708
1709 // Now we need to do some global optimization transforms.
1710 // FIXME: It would seem like these should come first in the optimization
1711 // pipeline and maybe be the bottom of the canonicalization pipeline? Weird
1712 // ordering here.
1713 MPM.addPass(Pass: GlobalDCEPass());
1714 MPM.addPass(Pass: ConstantMergePass());
1715
1716 // Merge functions if requested. It has a better chance to merge functions
1717 // after ConstantMerge folded jump tables.
1718 if (PTO.MergeFunctions)
1719 MPM.addPass(Pass: MergeFunctionsPass());
1720
1721 if (PTO.CallGraphProfile && !isLTOPreLink(Phase: LTOPhase))
1722 MPM.addPass(Pass: CGProfilePass(isLTOPostLink(Phase: LTOPhase)));
1723
1724 // RelLookupTableConverterPass runs later in LTO post-link pipeline.
1725 if (!isLTOPreLink(Phase: LTOPhase))
1726 MPM.addPass(Pass: RelLookupTableConverterPass());
1727
1728 // Add devirtualization pass only when LTO is not enabled, as otherwise
1729 // the pass is already enabled in the LTO pipeline.
1730 if (PTO.DevirtualizeSpeculatively && LTOPhase == ThinOrFullLTOPhase::None) {
1731 // TODO: explore a better pipeline configuration that can improve
1732 // compilation time overhead.
1733 // FIXME: move this earlier (lots of pass ordering tests will need fixing)
1734 MPM.addPass(Pass: AssignGUIDPass());
1735 MPM.addPass(Pass: WholeProgramDevirtPass(
1736 /*ExportSummary*/ nullptr,
1737 /*ImportSummary*/ nullptr,
1738 /*DevirtSpeculatively*/ PTO.DevirtualizeSpeculatively));
1739 MPM.addPass(Pass: DropTypeTestsPass());
1740 // Given that the devirtualization creates more opportunities for inlining,
1741 // we run the Inliner again here to maximize the optimization gain we
1742 // get from devirtualization.
1743 // Also, we can't run devirtualization before inlining because the
1744 // devirtualization depends on the passes optimizing/eliminating vtable GVs
1745 // and those passes are only effective after inlining.
1746 if (EnableModuleInliner) {
1747 MPM.addPass(Pass: ModuleInlinerPass(::getInlineParamsFromOptLevel(Level),
1748 UseInlineAdvisor,
1749 ThinOrFullLTOPhase::None));
1750 } else {
1751 MPM.addPass(Pass: ModuleInlinerWrapperPass(
1752 ::getInlineParamsFromOptLevel(Level),
1753 /* MandatoryFirst */ true,
1754 InlineContext{.LTOPhase: ThinOrFullLTOPhase::None, .Pass: InlinePass::CGSCCInliner}));
1755 }
1756 }
1757
1758 // Attach !implicit.ref metadata from all functions to copyright strings.
1759 MPM.addPass(Pass: LowerCommentStringPass());
1760
1761 return MPM;
1762}
1763
1764ModulePassManager
1765PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level,
1766 ThinOrFullLTOPhase Phase) {
1767 if (Level == OptimizationLevel::O0)
1768 return buildO0DefaultPipeline(Level, Phase);
1769
1770 ModulePassManager MPM;
1771 instructionCountersPass(MPM, /* IsPreOptimization */ true);
1772 // Currently this pipeline is only invoked in an LTO pre link pass or when we
1773 // are not running LTO. If that changes the below checks may need updating.
1774 assert(isLTOPreLink(Phase) || Phase == ThinOrFullLTOPhase::None);
1775
1776 // If we are invoking this in non-LTO mode, remove any MemProf related
1777 // attributes and metadata, as we don't know whether we are linking with
1778 // a library containing the necessary interfaces.
1779 if (Phase == ThinOrFullLTOPhase::None)
1780 MPM.addPass(Pass: MemProfRemoveInfo());
1781
1782 // Convert @llvm.global.annotations to !annotation metadata.
1783 MPM.addPass(Pass: Annotation2MetadataPass());
1784
1785 // Force any function attributes we want the rest of the pipeline to observe.
1786 MPM.addPass(Pass: ForceFunctionAttrsPass());
1787
1788 if (TriggerCrash)
1789 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: TriggerCrashFunctionPass()));
1790
1791 if (PGOOpt && PGOOpt->DebugInfoForProfiling)
1792 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: AddDiscriminatorsPass()));
1793
1794 // Apply module pipeline start EP callback.
1795 invokePipelineStartEPCallbacks(MPM, Level);
1796
1797 // Add the core simplification pipeline.
1798 MPM.addPass(Pass: buildModuleSimplificationPipeline(Level, Phase));
1799
1800 // Now add the optimization pipeline.
1801 MPM.addPass(Pass: buildModuleOptimizationPipeline(Level, LTOPhase: Phase));
1802
1803 if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&
1804 PGOOpt->Action == PGOOptions::SampleUse)
1805 MPM.addPass(Pass: PseudoProbeUpdatePass());
1806
1807 // Emit annotation remarks.
1808 addAnnotationRemarksPass(MPM);
1809
1810 if (isLTOPreLink(Phase))
1811 addRequiredLTOPreLinkPasses(MPM);
1812
1813 instructionCountersPass(MPM, /* IsPreOptimization */ false);
1814 return MPM;
1815}
1816
1817ModulePassManager
1818PassBuilder::buildFatLTODefaultPipeline(OptimizationLevel Level, bool ThinLTO,
1819 bool EmitSummary, bool Verify) {
1820 ModulePassManager MPM;
1821
1822 instructionCountersPass(MPM, /* IsPreOptimization */ true);
1823
1824 if (ThinLTO)
1825 MPM.addPass(Pass: buildThinLTOPreLinkDefaultPipeline(Level));
1826 else
1827 MPM.addPass(Pass: buildLTOPreLinkDefaultPipeline(Level));
1828 // AssignGUIDPass attaches !guid metadata (MD_unique_id) to global objects,
1829 // triggering the bitcode writer to emit a METADATA_KIND_BLOCK. Standard LTO
1830 // bitcode emission runs VerifierPass by default, which registers metadata
1831 // kind IDs in LLVMContext. Running VerifierPass here before EmbedBitcodePass
1832 // to get the same behavior.
1833 if (Verify)
1834 MPM.addPass(Pass: VerifierPass());
1835 MPM.addPass(Pass: EmbedBitcodePass(ThinLTO, EmitSummary));
1836
1837 // Perform any cleanups to the IR that aren't suitable for per TU compilation,
1838 // like removing CFI/WPD related instructions. Note, we reuse
1839 // DropTypeTestsPass to clean up type tests rather than duplicate that logic
1840 // in FatLtoCleanup.
1841 MPM.addPass(Pass: FatLtoCleanup());
1842
1843 // If we're doing FatLTO w/ CFI enabled, we don't want the type tests in the
1844 // object code, only in the bitcode section, so drop it before we run
1845 // module optimization and generate machine code. If llvm.type.test() isn't in
1846 // the IR, this won't do anything.
1847 MPM.addPass(Pass: DropTypeTestsPass(lowertypetests::DropTestKind::All));
1848
1849 // Use the ThinLTO post-link pipeline with sample profiling
1850 if (ThinLTO && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse)
1851 MPM.addPass(Pass: buildThinLTODefaultPipeline(Level, /*ImportSummary=*/nullptr));
1852 else {
1853 // ModuleSimplification does not run the coroutine passes for
1854 // ThinLTOPreLink, so we need the coroutine passes to run for ThinLTO
1855 // builds, otherwise they will miscompile.
1856 if (ThinLTO) {
1857 // TODO: replace w/ buildCoroWrapper() when it takes phase and level into
1858 // consideration.
1859 CGSCCPassManager CGPM;
1860 CGPM.addPass(Pass: CoroSplitPass(Level != OptimizationLevel::O0));
1861 CGPM.addPass(Pass: CoroAnnotationElidePass());
1862 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: std::move(CGPM)));
1863 MPM.addPass(Pass: CoroCleanupPass());
1864 }
1865
1866 // otherwise, just use module optimization
1867 MPM.addPass(
1868 Pass: buildModuleOptimizationPipeline(Level, LTOPhase: ThinOrFullLTOPhase::None));
1869 // Emit annotation remarks.
1870 addAnnotationRemarksPass(MPM);
1871 }
1872
1873 instructionCountersPass(MPM, /* IsPreOptimization */ false);
1874
1875 return MPM;
1876}
1877
1878ModulePassManager
1879PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level) {
1880 if (Level == OptimizationLevel::O0)
1881 return buildO0DefaultPipeline(Level, Phase: ThinOrFullLTOPhase::ThinLTOPreLink);
1882
1883 ModulePassManager MPM;
1884
1885 instructionCountersPass(MPM, /* IsPreOptimization */ true);
1886
1887 // Convert @llvm.global.annotations to !annotation metadata.
1888 MPM.addPass(Pass: Annotation2MetadataPass());
1889
1890 // Force any function attributes we want the rest of the pipeline to observe.
1891 MPM.addPass(Pass: ForceFunctionAttrsPass());
1892
1893 if (PGOOpt && PGOOpt->DebugInfoForProfiling)
1894 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: AddDiscriminatorsPass()));
1895
1896 // Apply module pipeline start EP callback.
1897 invokePipelineStartEPCallbacks(MPM, Level);
1898
1899 // If we are planning to perform ThinLTO later, we don't bloat the code with
1900 // unrolling/vectorization/... now. Just simplify the module as much as we
1901 // can.
1902 MPM.addPass(Pass: buildModuleSimplificationPipeline(
1903 Level, Phase: ThinOrFullLTOPhase::ThinLTOPreLink));
1904 // In pre-link, for ctx prof use, we stop here with an instrumented IR. We let
1905 // thinlto use the contextual info to perform imports; then use the contextual
1906 // profile in the post-thinlink phase.
1907 if (!UseCtxProfile.empty()) {
1908 addRequiredLTOPreLinkPasses(MPM);
1909 return MPM;
1910 }
1911
1912 // Run partial inlining pass to partially inline functions that have
1913 // large bodies.
1914 // FIXME: It isn't clear whether this is really the right place to run this
1915 // in ThinLTO. Because there is another canonicalization and simplification
1916 // phase that will run after the thin link, running this here ends up with
1917 // less information than will be available later and it may grow functions in
1918 // ways that aren't beneficial.
1919 if (RunPartialInlining)
1920 MPM.addPass(Pass: PartialInlinerPass());
1921
1922 if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&
1923 PGOOpt->Action == PGOOptions::SampleUse)
1924 MPM.addPass(Pass: PseudoProbeUpdatePass());
1925
1926 // Handle Optimizer{Early,Last}EPCallbacks added by clang on PreLink. Actual
1927 // optimization is going to be done in PostLink stage, but clang can't add
1928 // callbacks there in case of in-process ThinLTO called by linker.
1929 invokeOptimizerEarlyEPCallbacks(MPM, Level,
1930 /*Phase=*/ThinOrFullLTOPhase::ThinLTOPreLink);
1931 invokeOptimizerLastEPCallbacks(MPM, Level,
1932 /*Phase=*/ThinOrFullLTOPhase::ThinLTOPreLink);
1933
1934 // Emit annotation remarks.
1935 addAnnotationRemarksPass(MPM);
1936
1937 // Attach !implicit.ref metadata from all functions to copyright strings.
1938 MPM.addPass(Pass: LowerCommentStringPass());
1939
1940 addRequiredLTOPreLinkPasses(MPM);
1941
1942 instructionCountersPass(MPM, /* IsPreOptimization */ false);
1943
1944 return MPM;
1945}
1946
1947ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
1948 OptimizationLevel Level, const ModuleSummaryIndex *ImportSummary) {
1949 ModulePassManager MPM;
1950
1951 instructionCountersPass(MPM, /* IsPreOptimization */ true);
1952
1953 invokeThinLinkTimeOptimizationEarlyEPCallbacks(MPM, Level);
1954
1955 // If we are invoking this without a summary index noting that we are linking
1956 // with a library containing the necessary APIs, remove any MemProf related
1957 // attributes and metadata.
1958 if (!ImportSummary || !ImportSummary->withSupportsHotColdNew())
1959 MPM.addPass(Pass: MemProfRemoveInfo());
1960
1961 if (ImportSummary) {
1962 // For ThinLTO we must apply the context disambiguation decisions early, to
1963 // ensure we can correctly match the callsites to summary data.
1964 if (EnableMemProfContextDisambiguation)
1965 MPM.addPass(Pass: MemProfContextDisambiguation(
1966 ImportSummary, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
1967
1968 // These passes import type identifier resolutions for whole-program
1969 // devirtualization and CFI. They must run early because other passes may
1970 // disturb the specific instruction patterns that these passes look for,
1971 // creating dependencies on resolutions that may not appear in the summary.
1972 //
1973 // For example, GVN may transform the pattern assume(type.test) appearing in
1974 // two basic blocks into assume(phi(type.test, type.test)), which would
1975 // transform a dependency on a WPD resolution into a dependency on a type
1976 // identifier resolution for CFI.
1977 //
1978 // Also, WPD has access to more precise information than ICP and can
1979 // devirtualize more effectively, so it should operate on the IR first.
1980 //
1981 // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1982 // metadata and intrinsics.
1983 MPM.addPass(Pass: WholeProgramDevirtPass(nullptr, ImportSummary));
1984 MPM.addPass(Pass: LowerTypeTestsPass(nullptr, ImportSummary));
1985 }
1986
1987 if (Level == OptimizationLevel::O0) {
1988 // Run a second time to clean up any type tests left behind by WPD for use
1989 // in ICP.
1990 MPM.addPass(Pass: DropTypeTestsPass());
1991 MPM.addPass(Pass: buildCoroWrapper(Phase: ThinOrFullLTOPhase::ThinLTOPostLink));
1992
1993 // AllocToken transforms heap allocation calls; this needs to run late after
1994 // other allocation call transformations (such as those in InstCombine).
1995 MPM.addPass(Pass: AllocTokenPass());
1996
1997 // Drop available_externally and unreferenced globals. This is necessary
1998 // with ThinLTO in order to avoid leaving undefined references to dead
1999 // globals in the object file.
2000 MPM.addPass(Pass: EliminateAvailableExternallyPass());
2001 MPM.addPass(Pass: GlobalDCEPass());
2002
2003 invokeThinLinkTimeOptimizationLastEPCallbacks(MPM, Level);
2004
2005 return MPM;
2006 }
2007 if (!UseCtxProfile.empty()) {
2008 MPM.addPass(
2009 Pass: buildModuleInlinerPipeline(Level, Phase: ThinOrFullLTOPhase::ThinLTOPostLink));
2010 } else {
2011 // Add the core simplification pipeline.
2012 MPM.addPass(Pass: buildModuleSimplificationPipeline(
2013 Level, Phase: ThinOrFullLTOPhase::ThinLTOPostLink));
2014 }
2015 // Now add the optimization pipeline.
2016 MPM.addPass(Pass: buildModuleOptimizationPipeline(
2017 Level, LTOPhase: ThinOrFullLTOPhase::ThinLTOPostLink));
2018
2019 invokeThinLinkTimeOptimizationLastEPCallbacks(MPM, Level);
2020
2021 // Emit annotation remarks.
2022 addAnnotationRemarksPass(MPM);
2023
2024 instructionCountersPass(MPM, /* IsPreOptimization */ false);
2025
2026 return MPM;
2027}
2028
2029ModulePassManager
2030PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level) {
2031 // FIXME: We should use a customized pre-link pipeline!
2032 return buildPerModuleDefaultPipeline(Level,
2033 Phase: ThinOrFullLTOPhase::FullLTOPreLink);
2034}
2035
2036ModulePassManager
2037PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
2038 ModuleSummaryIndex *ExportSummary) {
2039 ModulePassManager MPM;
2040
2041 instructionCountersPass(MPM, /* IsPreOptimization */ true);
2042
2043 invokeFullLinkTimeOptimizationEarlyEPCallbacks(MPM, Level);
2044
2045 // If we are invoking this without a summary index noting that we are linking
2046 // with a library containing the necessary APIs, remove any MemProf related
2047 // attributes and metadata.
2048 if (!ExportSummary || !ExportSummary->withSupportsHotColdNew())
2049 MPM.addPass(Pass: MemProfRemoveInfo());
2050
2051 // Create a function that performs CFI checks for cross-DSO calls with targets
2052 // in the current module.
2053 MPM.addPass(Pass: CrossDSOCFIPass());
2054
2055 if (Level == OptimizationLevel::O0) {
2056 // The WPD and LowerTypeTest passes need to run at -O0 to lower type
2057 // metadata and intrinsics.
2058 MPM.addPass(Pass: WholeProgramDevirtPass(ExportSummary, nullptr));
2059 MPM.addPass(Pass: LowerTypeTestsPass(ExportSummary, nullptr));
2060 // Run a second time to clean up any type tests left behind by WPD for use
2061 // in ICP.
2062 MPM.addPass(Pass: DropTypeTestsPass());
2063
2064 MPM.addPass(Pass: buildCoroWrapper(Phase: ThinOrFullLTOPhase::FullLTOPostLink));
2065
2066 // AllocToken transforms heap allocation calls; this needs to run late after
2067 // other allocation call transformations (such as those in InstCombine).
2068 MPM.addPass(Pass: AllocTokenPass());
2069
2070 invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);
2071
2072 // Emit annotation remarks.
2073 addAnnotationRemarksPass(MPM);
2074
2075 return MPM;
2076 }
2077
2078 if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
2079 // Load sample profile before running the LTO optimization pipeline.
2080 MPM.addPass(Pass: SampleProfileLoaderPass(PGOOpt->ProfileFile,
2081 PGOOpt->ProfileRemappingFile,
2082 ThinOrFullLTOPhase::FullLTOPostLink));
2083 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
2084 // RequireAnalysisPass for PSI before subsequent non-module passes.
2085 MPM.addPass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
2086 }
2087
2088 // Try to run OpenMP optimizations, quick no-op if no OpenMP metadata present.
2089 MPM.addPass(Pass: OpenMPOptPass(ThinOrFullLTOPhase::FullLTOPostLink));
2090
2091 // Remove unused virtual tables to improve the quality of code generated by
2092 // whole-program devirtualization and bitset lowering.
2093 MPM.addPass(Pass: GlobalDCEPass(/*InLTOPostLink=*/true));
2094
2095 // Do basic inference of function attributes from known properties of system
2096 // libraries and other oracles.
2097 MPM.addPass(Pass: InferFunctionAttrsPass());
2098
2099 if (Level >= OptimizationLevel::O2) {
2100 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
2101 Pass: CallSiteSplittingPass(), EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
2102
2103 // Indirect call promotion. This should promote all the targets that are
2104 // left by the earlier promotion pass that promotes intra-module targets.
2105 // This two-step promotion is to save the compile time. For LTO, it should
2106 // produce the same result as if we only do promotion here.
2107 MPM.addPass(Pass: PGOIndirectCallPromotion(
2108 true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
2109
2110 // Promoting by-reference arguments to by-value exposes more constants to
2111 // IPSCCP.
2112 CGSCCPassManager CGPM;
2113 CGPM.addPass(Pass: PostOrderFunctionAttrsPass());
2114 CGPM.addPass(Pass: ArgumentPromotionPass());
2115 CGPM.addPass(
2116 Pass: createCGSCCToFunctionPassAdaptor(Pass: SROAPass(SROAOptions::ModifyCFG)));
2117 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: std::move(CGPM)));
2118
2119 // Propagate constants at call sites into the functions they call. This
2120 // opens opportunities for globalopt (and inlining) by substituting function
2121 // pointers passed as arguments to direct uses of functions.
2122 MPM.addPass(Pass: IPSCCPPass(IPSCCPOptions(/*AllowFuncSpec=*/true)));
2123
2124 // Attach metadata to indirect call sites indicating the set of functions
2125 // they may target at run-time. This should follow IPSCCP.
2126 MPM.addPass(Pass: CalledValuePropagationPass());
2127 }
2128
2129 // Do RPO function attribute inference across the module to forward-propagate
2130 // attributes where applicable.
2131 // FIXME: Is this really an optimization rather than a canonicalization?
2132 MPM.addPass(Pass: ReversePostOrderFunctionAttrsPass());
2133
2134 // Use in-range annotations on GEP indices to split globals where beneficial.
2135 MPM.addPass(Pass: GlobalSplitPass());
2136
2137 // Run whole program optimization of virtual call when the list of callees
2138 // is fixed.
2139 MPM.addPass(Pass: WholeProgramDevirtPass(ExportSummary, nullptr));
2140
2141 MPM.addPass(Pass: NoRecurseLTOInferencePass());
2142 // Stop here at -O1.
2143 if (Level == OptimizationLevel::O1) {
2144 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
2145 Pass: LowerConstantIntrinsicsPass(), EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
2146
2147 // The LowerTypeTestsPass needs to run to lower type metadata and the
2148 // type.test intrinsics. The pass does nothing if CFI is disabled.
2149 MPM.addPass(Pass: LowerTypeTestsPass(ExportSummary, nullptr));
2150 // Run a second time to clean up any type tests left behind by WPD for use
2151 // in ICP (which is performed earlier than this in the regular LTO
2152 // pipeline).
2153 MPM.addPass(Pass: DropTypeTestsPass());
2154
2155 MPM.addPass(Pass: buildCoroWrapper(Phase: ThinOrFullLTOPhase::FullLTOPostLink));
2156
2157 // AllocToken transforms heap allocation calls; this needs to run late after
2158 // other allocation call transformations (such as those in InstCombine).
2159 MPM.addPass(Pass: AllocTokenPass());
2160
2161 invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);
2162
2163 // Emit annotation remarks.
2164 addAnnotationRemarksPass(MPM);
2165
2166 instructionCountersPass(MPM, /* IsPreOptimization */ false);
2167
2168 return MPM;
2169 }
2170
2171 // TODO: Skip to match buildCoroWrapper.
2172 MPM.addPass(Pass: CoroEarlyPass());
2173
2174 // Optimize globals to try and fold them into constants.
2175 MPM.addPass(Pass: GlobalOptPass());
2176
2177 // Promote any localized globals to SSA registers.
2178 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: PromotePass()));
2179
2180 // Linking modules together can lead to duplicate global constant, only
2181 // keep one copy of each constant.
2182 MPM.addPass(Pass: ConstantMergePass());
2183
2184 // Remove unused arguments from functions.
2185 MPM.addPass(Pass: DeadArgumentEliminationPass());
2186
2187 // Reduce the code after globalopt and ipsccp. Both can open up significant
2188 // simplification opportunities, and both can propagate functions through
2189 // function pointers. When this happens, we often have to resolve varargs
2190 // calls, etc, so let instcombine do this.
2191 FunctionPassManager PeepholeFPM;
2192 PeepholeFPM.addPass(Pass: InstCombinePass());
2193 if (Level >= OptimizationLevel::O2)
2194 PeepholeFPM.addPass(Pass: AggressiveInstCombinePass());
2195 invokePeepholeEPCallbacks(FPM&: PeepholeFPM, Level);
2196
2197 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(PeepholeFPM),
2198 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
2199
2200 // Lower variadic functions for supported targets prior to inlining.
2201 MPM.addPass(Pass: ExpandVariadicsPass(ExpandVariadicsMode::Optimize));
2202
2203 // Note: historically, the PruneEH pass was run first to deduce nounwind and
2204 // generally clean up exception handling overhead. It isn't clear this is
2205 // valuable as the inliner doesn't currently care whether it is inlining an
2206 // invoke or a call.
2207 // Run the inliner now.
2208 if (EnableModuleInliner) {
2209 MPM.addPass(Pass: ModuleInlinerPass(::getInlineParamsFromOptLevel(Level),
2210 UseInlineAdvisor,
2211 ThinOrFullLTOPhase::FullLTOPostLink));
2212 } else {
2213 MPM.addPass(Pass: ModuleInlinerWrapperPass(
2214 ::getInlineParamsFromOptLevel(Level),
2215 /* MandatoryFirst */ true,
2216 InlineContext{.LTOPhase: ThinOrFullLTOPhase::FullLTOPostLink,
2217 .Pass: InlinePass::CGSCCInliner}));
2218 }
2219
2220 // Perform context disambiguation after inlining, since that would reduce the
2221 // amount of additional cloning required to distinguish the allocation
2222 // contexts.
2223 if (EnableMemProfContextDisambiguation)
2224 MPM.addPass(Pass: MemProfContextDisambiguation(
2225 /*Summary=*/nullptr,
2226 PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
2227
2228 // Optimize globals again after we ran the inliner.
2229 MPM.addPass(Pass: GlobalOptPass());
2230
2231 // Run the OpenMPOpt pass again after global optimizations.
2232 MPM.addPass(Pass: OpenMPOptPass(ThinOrFullLTOPhase::FullLTOPostLink));
2233
2234 // Garbage collect dead functions.
2235 MPM.addPass(Pass: GlobalDCEPass(/*InLTOPostLink=*/true));
2236
2237 // If we didn't decide to inline a function, check to see if we can
2238 // transform it to pass arguments by value instead of by reference.
2239 CGSCCPassManager CGPM;
2240 CGPM.addPass(Pass: ArgumentPromotionPass());
2241 CGPM.addPass(Pass: CoroSplitPass(Level != OptimizationLevel::O0));
2242 CGPM.addPass(Pass: CoroAnnotationElidePass());
2243 invokeCGSCCOptimizerLateEPCallbacks(CGPM, Level);
2244 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: std::move(CGPM)));
2245
2246 FunctionPassManager FPM;
2247 // The IPO Passes may leave cruft around. Clean up after them.
2248 FPM.addPass(Pass: InstCombinePass());
2249 invokePeepholeEPCallbacks(FPM, Level);
2250
2251 if (EnableConstraintElimination)
2252 FPM.addPass(Pass: ConstraintEliminationPass());
2253
2254 FPM.addPass(Pass: JumpThreadingPass());
2255
2256 // Do a post inline PGO instrumentation and use pass. This is a context
2257 // sensitive PGO pass.
2258 if (PGOOpt) {
2259 if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
2260 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/true,
2261 /*IsCS=*/true, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate,
2262 ProfileFile: PGOOpt->CSProfileGenFile, ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
2263 else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
2264 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/false,
2265 /*IsCS=*/true, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate,
2266 ProfileFile: PGOOpt->ProfileFile, ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
2267 }
2268
2269 // Break up allocas
2270 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
2271
2272 // LTO provides additional opportunities for tailcall elimination due to
2273 // link-time inlining, and visibility of nocapture attribute.
2274 FPM.addPass(
2275 Pass: TailCallElimPass(/*UpdateFunctionEntryCount=*/isInstrumentedPGOUse()));
2276
2277 // Run a few AA driver optimizations here and now to cleanup the code.
2278 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM),
2279 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
2280
2281 MPM.addPass(
2282 Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: PostOrderFunctionAttrsPass()));
2283
2284 // Require the GlobalsAA analysis for the module so we can query it within
2285 // MainFPM.
2286 if (EnableGlobalAnalyses) {
2287 MPM.addPass(Pass: RequireAnalysisPass<GlobalsAA, Module>());
2288 // Invalidate AAManager so it can be recreated and pick up the newly
2289 // available GlobalsAA.
2290 MPM.addPass(
2291 Pass: createModuleToFunctionPassAdaptor(Pass: InvalidateAnalysisPass<AAManager>()));
2292 }
2293
2294 FunctionPassManager MainFPM;
2295 MainFPM.addPass(Pass: createFunctionToLoopPassAdaptor(
2296 Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
2297 /*AllowSpeculation=*/true),
2298 /*USeMemorySSA=*/UseMemorySSA: true));
2299
2300 if (RunNewGVN)
2301 MainFPM.addPass(Pass: NewGVNPass());
2302 else
2303 MainFPM.addPass(Pass: GVNPass());
2304
2305 // Remove dead memcpy()'s.
2306 MainFPM.addPass(Pass: MemCpyOptPass());
2307
2308 // Nuke dead stores.
2309 MainFPM.addPass(Pass: DSEPass());
2310 MainFPM.addPass(Pass: MoveAutoInitPass());
2311 MainFPM.addPass(Pass: MergedLoadStoreMotionPass());
2312
2313 MainFPM.addPass(Pass: LowerConstantIntrinsicsPass());
2314
2315 invokeVectorizerStartEPCallbacks(FPM&: MainFPM, Level);
2316
2317 LoopPassManager LPM;
2318 if (EnableLoopFlatten && Level >= OptimizationLevel::O2)
2319 LPM.addPass(Pass: LoopFlattenPass());
2320 LPM.addPass(Pass: IndVarSimplifyPass());
2321 LPM.addPass(Pass: LoopDeletionPass());
2322 // FIXME: Add loop interchange.
2323
2324 // Unroll small loops and perform peeling.
2325 LPM.addPass(Pass: LoopFullUnrollPass(static_cast<int>(Level),
2326 /* OnlyWhenForced= */ !PTO.LoopUnrolling,
2327 PTO.ForgetAllSCEVInLoopUnroll));
2328 // The loop passes in LPM (LoopFullUnrollPass) do not preserve MemorySSA.
2329 // *All* loop passes must preserve it, in order to be able to use it.
2330 MainFPM.addPass(
2331 Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM), /*UseMemorySSA=*/false));
2332
2333 MainFPM.addPass(Pass: LoopDistributePass());
2334
2335 addVectorPasses(Level, FPM&: MainFPM, LTOPhase: ThinOrFullLTOPhase::FullLTOPostLink);
2336
2337 invokeVectorizerEndEPCallbacks(FPM&: MainFPM, Level);
2338
2339 // Run the OpenMPOpt CGSCC pass again late.
2340 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(
2341 Pass: OpenMPOptCGSCCPass(ThinOrFullLTOPhase::FullLTOPostLink)));
2342
2343 invokePeepholeEPCallbacks(FPM&: MainFPM, Level);
2344 MainFPM.addPass(Pass: JumpThreadingPass());
2345 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(MainFPM),
2346 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
2347
2348 // Lower type metadata and the type.test intrinsic. This pass supports
2349 // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
2350 // to be run at link time if CFI is enabled. This pass does nothing if
2351 // CFI is disabled.
2352 MPM.addPass(Pass: LowerTypeTestsPass(ExportSummary, nullptr));
2353 // Run a second time to clean up any type tests left behind by WPD for use
2354 // in ICP (which is performed earlier than this in the regular LTO pipeline).
2355 MPM.addPass(Pass: DropTypeTestsPass());
2356
2357 // Enable splitting late in the FullLTO post-link pipeline.
2358 if (EnableHotColdSplit)
2359 MPM.addPass(Pass: HotColdSplittingPass());
2360
2361 // Add late LTO optimization passes.
2362 FunctionPassManager LateFPM;
2363
2364 // LoopSink pass sinks instructions hoisted by LICM, which serves as a
2365 // canonicalization pass that enables other optimizations. As a result,
2366 // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
2367 // result too early.
2368 LateFPM.addPass(Pass: LoopSinkPass());
2369
2370 // This hoists/decomposes div/rem ops. It should run after other sink/hoist
2371 // passes to avoid re-sinking, but before SimplifyCFG because it can allow
2372 // flattening of blocks.
2373 LateFPM.addPass(Pass: DivRemPairsPass());
2374
2375 // Delete basic blocks, which optimization passes may have killed.
2376 LateFPM.addPass(Pass: SimplifyCFGPass(SimplifyCFGOptions()
2377 .convertSwitchRangeToICmp(B: true)
2378 .convertSwitchToArithmetic(B: true)
2379 .hoistCommonInsts(B: true)
2380 .speculateUnpredictables(B: true)));
2381 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(LateFPM)));
2382
2383 // Drop bodies of available eternally objects to improve GlobalDCE.
2384 MPM.addPass(Pass: EliminateAvailableExternallyPass());
2385
2386 // Now that we have optimized the program, discard unreachable functions.
2387 MPM.addPass(Pass: GlobalDCEPass(/*InLTOPostLink=*/true));
2388
2389 if (PTO.MergeFunctions)
2390 MPM.addPass(Pass: MergeFunctionsPass());
2391
2392 MPM.addPass(Pass: RelLookupTableConverterPass());
2393
2394 if (PTO.CallGraphProfile)
2395 MPM.addPass(Pass: CGProfilePass(/*InLTOPostLink=*/true));
2396
2397 MPM.addPass(Pass: CoroCleanupPass());
2398
2399 // AllocToken transforms heap allocation calls; this needs to run late after
2400 // other allocation call transformations (such as those in InstCombine).
2401 MPM.addPass(Pass: AllocTokenPass());
2402
2403 invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);
2404
2405 // Emit annotation remarks.
2406 addAnnotationRemarksPass(MPM);
2407
2408 instructionCountersPass(MPM, /* IsPreOptimization */ false);
2409
2410 return MPM;
2411}
2412
2413ModulePassManager
2414PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level,
2415 ThinOrFullLTOPhase Phase) {
2416 assert(Level == OptimizationLevel::O0 &&
2417 "buildO0DefaultPipeline should only be used with O0");
2418
2419 ModulePassManager MPM;
2420
2421 instructionCountersPass(MPM, /* IsPreOptimization */ true);
2422
2423 // Perform pseudo probe instrumentation in O0 mode. This is for the
2424 // consistency between different build modes. For example, a LTO build can be
2425 // mixed with an O0 prelink and an O2 postlink. Loading a sample profile in
2426 // the postlink will require pseudo probe instrumentation in the prelink.
2427 if (PGOOpt && PGOOpt->PseudoProbeForProfiling)
2428 MPM.addPass(Pass: SampleProfileProbePass(TM));
2429
2430 if (PGOOpt && (PGOOpt->Action == PGOOptions::IRInstr ||
2431 PGOOpt->Action == PGOOptions::IRUse))
2432 addPGOInstrPassesForO0(
2433 MPM,
2434 /*RunProfileGen=*/(PGOOpt->Action == PGOOptions::IRInstr),
2435 /*IsCS=*/false, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate, ProfileFile: PGOOpt->ProfileFile,
2436 ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
2437
2438 // Instrument function entry and exit before all inlining.
2439 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
2440 Pass: EntryExitInstrumenterPass(/*PostInlining=*/false)));
2441
2442 invokePipelineStartEPCallbacks(MPM, Level);
2443
2444 if (PGOOpt && PGOOpt->DebugInfoForProfiling)
2445 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: AddDiscriminatorsPass()));
2446
2447 if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
2448 // Explicitly disable sample loader inlining and use flattened profile in O0
2449 // pipeline.
2450 MPM.addPass(Pass: SampleProfileLoaderPass(PGOOpt->ProfileFile,
2451 PGOOpt->ProfileRemappingFile,
2452 ThinOrFullLTOPhase::None, FS,
2453 /*DisableSampleProfileInlining=*/true,
2454 /*UseFlattenedProfile=*/true));
2455 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
2456 // RequireAnalysisPass for PSI before subsequent non-module passes.
2457 MPM.addPass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
2458 }
2459
2460 invokePipelineEarlySimplificationEPCallbacks(MPM, Level, Phase);
2461
2462 // Build a minimal pipeline based on the semantics required by LLVM,
2463 // which is just that always inlining occurs. Further, disable generating
2464 // lifetime intrinsics to avoid enabling further optimizations during
2465 // code generation.
2466 MPM.addPass(Pass: AlwaysInlinerPass(
2467 /*InsertLifetimeIntrinsics=*/false));
2468
2469 if (PTO.MergeFunctions)
2470 MPM.addPass(Pass: MergeFunctionsPass());
2471
2472 if (EnableMatrix)
2473 MPM.addPass(
2474 Pass: createModuleToFunctionPassAdaptor(Pass: LowerMatrixIntrinsicsPass(true)));
2475
2476 if (!CGSCCOptimizerLateEPCallbacks.empty()) {
2477 CGSCCPassManager CGPM;
2478 invokeCGSCCOptimizerLateEPCallbacks(CGPM, Level);
2479 if (!CGPM.isEmpty())
2480 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: std::move(CGPM)));
2481 }
2482 if (!LateLoopOptimizationsEPCallbacks.empty()) {
2483 LoopPassManager LPM;
2484 invokeLateLoopOptimizationsEPCallbacks(LPM, Level);
2485 if (!LPM.isEmpty()) {
2486 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
2487 Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM))));
2488 }
2489 }
2490 if (!LoopOptimizerEndEPCallbacks.empty()) {
2491 LoopPassManager LPM;
2492 invokeLoopOptimizerEndEPCallbacks(LPM, Level);
2493 if (!LPM.isEmpty()) {
2494 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
2495 Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM))));
2496 }
2497 }
2498 if (!ScalarOptimizerLateEPCallbacks.empty()) {
2499 FunctionPassManager FPM;
2500 invokeScalarOptimizerLateEPCallbacks(FPM, Level);
2501 if (!FPM.isEmpty())
2502 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
2503 }
2504
2505 invokeOptimizerEarlyEPCallbacks(MPM, Level, Phase);
2506
2507 if (!VectorizerStartEPCallbacks.empty()) {
2508 FunctionPassManager FPM;
2509 invokeVectorizerStartEPCallbacks(FPM, Level);
2510 if (!FPM.isEmpty())
2511 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
2512 }
2513
2514 if (!VectorizerEndEPCallbacks.empty()) {
2515 FunctionPassManager FPM;
2516 invokeVectorizerEndEPCallbacks(FPM, Level);
2517 if (!FPM.isEmpty())
2518 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
2519 }
2520
2521 MPM.addPass(Pass: buildCoroWrapper(Phase));
2522
2523 // AllocToken transforms heap allocation calls; this needs to run late after
2524 // other allocation call transformations (such as those in InstCombine).
2525 if (!isLTOPreLink(Phase))
2526 MPM.addPass(Pass: AllocTokenPass());
2527
2528 invokeOptimizerLastEPCallbacks(MPM, Level, Phase);
2529
2530 if (EnableInstrumentor)
2531 MPM.addPass(Pass: InstrumentorPass(FS));
2532
2533 // Attach !implicit.ref metadata from all functions to copyright strings.
2534 MPM.addPass(Pass: LowerCommentStringPass());
2535
2536 if (isLTOPreLink(Phase))
2537 addRequiredLTOPreLinkPasses(MPM);
2538
2539 // Emit annotation remarks.
2540 addAnnotationRemarksPass(MPM);
2541
2542 instructionCountersPass(MPM, /* IsPreOptimization */ false);
2543
2544 return MPM;
2545}
2546
2547AAManager PassBuilder::buildDefaultAAPipeline() {
2548 AAManager AA;
2549
2550 // The order in which these are registered determines their priority when
2551 // being queried.
2552
2553 // Add any target-specific alias analyses that should be run early.
2554 if (TM)
2555 TM->registerEarlyDefaultAliasAnalyses(AA);
2556
2557 // First we register the basic alias analysis that provides the majority of
2558 // per-function local AA logic. This is a stateless, on-demand local set of
2559 // AA techniques.
2560 AA.registerFunctionAnalysis<BasicAA>();
2561
2562 // Next we query fast, specialized alias analyses that wrap IR-embedded
2563 // information about aliasing.
2564 AA.registerFunctionAnalysis<ScopedNoAliasAA>();
2565 AA.registerFunctionAnalysis<TypeBasedAA>();
2566
2567 // Add support for querying global aliasing information when available.
2568 // Because the `AAManager` is a function analysis and `GlobalsAA` is a module
2569 // analysis, all that the `AAManager` can do is query for any *cached*
2570 // results from `GlobalsAA` through a readonly proxy.
2571 if (EnableGlobalAnalyses)
2572 AA.registerModuleAnalysis<GlobalsAA>();
2573
2574 // Add target-specific alias analyses.
2575 if (TM)
2576 TM->registerDefaultAliasAnalyses(AA);
2577
2578 return AA;
2579}
2580
2581bool PassBuilder::isInstrumentedPGOUse() const {
2582 return (PGOOpt && PGOOpt->Action == PGOOptions::IRUse) ||
2583 !UseCtxProfile.empty();
2584}
2585