1//===- Construction of pass pipelines -------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9///
10/// This file provides the implementation of the PassBuilder based on our
11/// static pass registry as well as related functionality. It also provides
12/// helpers to aid in analyzing, debugging, and testing passes and pass
13/// pipelines.
14///
15//===----------------------------------------------------------------------===//
16
17#include "llvm/ADT/Statistic.h"
18#include "llvm/Analysis/AliasAnalysis.h"
19#include "llvm/Analysis/BasicAliasAnalysis.h"
20#include "llvm/Analysis/CGSCCPassManager.h"
21#include "llvm/Analysis/CtxProfAnalysis.h"
22#include "llvm/Analysis/FunctionPropertiesAnalysis.h"
23#include "llvm/Analysis/GlobalsModRef.h"
24#include "llvm/Analysis/InlineAdvisor.h"
25#include "llvm/Analysis/InstCount.h"
26#include "llvm/Analysis/ProfileSummaryInfo.h"
27#include "llvm/Analysis/ScopedNoAliasAA.h"
28#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
29#include "llvm/IR/PassManager.h"
30#include "llvm/IR/Verifier.h"
31#include "llvm/Pass.h"
32#include "llvm/Passes/OptimizationLevel.h"
33#include "llvm/Passes/PassBuilder.h"
34#include "llvm/Support/CommandLine.h"
35#include "llvm/Support/ErrorHandling.h"
36#include "llvm/Support/PGOOptions.h"
37#include "llvm/Support/VirtualFileSystem.h"
38#include "llvm/Target/TargetMachine.h"
39#include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
40#include "llvm/Transforms/Coroutines/CoroAnnotationElide.h"
41#include "llvm/Transforms/Coroutines/CoroCleanup.h"
42#include "llvm/Transforms/Coroutines/CoroConditionalWrapper.h"
43#include "llvm/Transforms/Coroutines/CoroEarly.h"
44#include "llvm/Transforms/Coroutines/CoroElide.h"
45#include "llvm/Transforms/Coroutines/CoroSplit.h"
46#include "llvm/Transforms/HipStdPar/HipStdPar.h"
47#include "llvm/Transforms/IPO/AlwaysInliner.h"
48#include "llvm/Transforms/IPO/Annotation2Metadata.h"
49#include "llvm/Transforms/IPO/ArgumentPromotion.h"
50#include "llvm/Transforms/IPO/Attributor.h"
51#include "llvm/Transforms/IPO/CalledValuePropagation.h"
52#include "llvm/Transforms/IPO/ConstantMerge.h"
53#include "llvm/Transforms/IPO/CrossDSOCFI.h"
54#include "llvm/Transforms/IPO/DeadArgumentElimination.h"
55#include "llvm/Transforms/IPO/ElimAvailExtern.h"
56#include "llvm/Transforms/IPO/EmbedBitcodePass.h"
57#include "llvm/Transforms/IPO/ExpandVariadics.h"
58#include "llvm/Transforms/IPO/FatLTOCleanup.h"
59#include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
60#include "llvm/Transforms/IPO/FunctionAttrs.h"
61#include "llvm/Transforms/IPO/GlobalDCE.h"
62#include "llvm/Transforms/IPO/GlobalOpt.h"
63#include "llvm/Transforms/IPO/GlobalSplit.h"
64#include "llvm/Transforms/IPO/HotColdSplitting.h"
65#include "llvm/Transforms/IPO/IROutliner.h"
66#include "llvm/Transforms/IPO/InferFunctionAttrs.h"
67#include "llvm/Transforms/IPO/Inliner.h"
68#include "llvm/Transforms/IPO/Instrumentor.h"
69#include "llvm/Transforms/IPO/LowerTypeTests.h"
70#include "llvm/Transforms/IPO/MemProfContextDisambiguation.h"
71#include "llvm/Transforms/IPO/MergeFunctions.h"
72#include "llvm/Transforms/IPO/ModuleInliner.h"
73#include "llvm/Transforms/IPO/OpenMPOpt.h"
74#include "llvm/Transforms/IPO/PartialInlining.h"
75#include "llvm/Transforms/IPO/SCCP.h"
76#include "llvm/Transforms/IPO/SampleProfile.h"
77#include "llvm/Transforms/IPO/SampleProfileProbe.h"
78#include "llvm/Transforms/IPO/WholeProgramDevirt.h"
79#include "llvm/Transforms/InstCombine/InstCombine.h"
80#include "llvm/Transforms/Instrumentation/AllocToken.h"
81#include "llvm/Transforms/Instrumentation/CGProfile.h"
82#include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
83#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
84#include "llvm/Transforms/Instrumentation/MemProfInstrumentation.h"
85#include "llvm/Transforms/Instrumentation/MemProfUse.h"
86#include "llvm/Transforms/Instrumentation/PGOCtxProfFlattening.h"
87#include "llvm/Transforms/Instrumentation/PGOCtxProfLowering.h"
88#include "llvm/Transforms/Instrumentation/PGOForceFunctionAttrs.h"
89#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
90#include "llvm/Transforms/Scalar/ADCE.h"
91#include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
92#include "llvm/Transforms/Scalar/AnnotationRemarks.h"
93#include "llvm/Transforms/Scalar/BDCE.h"
94#include "llvm/Transforms/Scalar/CallSiteSplitting.h"
95#include "llvm/Transforms/Scalar/ConstraintElimination.h"
96#include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
97#include "llvm/Transforms/Scalar/DFAJumpThreading.h"
98#include "llvm/Transforms/Scalar/DeadStoreElimination.h"
99#include "llvm/Transforms/Scalar/DivRemPairs.h"
100#include "llvm/Transforms/Scalar/DropUnnecessaryAssumes.h"
101#include "llvm/Transforms/Scalar/EarlyCSE.h"
102#include "llvm/Transforms/Scalar/ExpandMemCmp.h"
103#include "llvm/Transforms/Scalar/Float2Int.h"
104#include "llvm/Transforms/Scalar/GVN.h"
105#include "llvm/Transforms/Scalar/IndVarSimplify.h"
106#include "llvm/Transforms/Scalar/InferAlignment.h"
107#include "llvm/Transforms/Scalar/InstSimplifyPass.h"
108#include "llvm/Transforms/Scalar/JumpTableToSwitch.h"
109#include "llvm/Transforms/Scalar/JumpThreading.h"
110#include "llvm/Transforms/Scalar/LICM.h"
111#include "llvm/Transforms/Scalar/LoopDeletion.h"
112#include "llvm/Transforms/Scalar/LoopDistribute.h"
113#include "llvm/Transforms/Scalar/LoopFlatten.h"
114#include "llvm/Transforms/Scalar/LoopFuse.h"
115#include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
116#include "llvm/Transforms/Scalar/LoopInstSimplify.h"
117#include "llvm/Transforms/Scalar/LoopInterchange.h"
118#include "llvm/Transforms/Scalar/LoopLoadElimination.h"
119#include "llvm/Transforms/Scalar/LoopPassManager.h"
120#include "llvm/Transforms/Scalar/LoopRotation.h"
121#include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
122#include "llvm/Transforms/Scalar/LoopSink.h"
123#include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
124#include "llvm/Transforms/Scalar/LoopUnrollPass.h"
125#include "llvm/Transforms/Scalar/LoopVersioningLICM.h"
126#include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
127#include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
128#include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h"
129#include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
130#include "llvm/Transforms/Scalar/MergeICmps.h"
131#include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"
132#include "llvm/Transforms/Scalar/NewGVN.h"
133#include "llvm/Transforms/Scalar/Reassociate.h"
134#include "llvm/Transforms/Scalar/SCCP.h"
135#include "llvm/Transforms/Scalar/SROA.h"
136#include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
137#include "llvm/Transforms/Scalar/SimplifyCFG.h"
138#include "llvm/Transforms/Scalar/SpeculativeExecution.h"
139#include "llvm/Transforms/Scalar/TailRecursionElimination.h"
140#include "llvm/Transforms/Scalar/WarnMissedTransforms.h"
141#include "llvm/Transforms/Utils/AddDiscriminators.h"
142#include "llvm/Transforms/Utils/AssignGUID.h"
143#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
144#include "llvm/Transforms/Utils/CanonicalizeAliases.h"
145#include "llvm/Transforms/Utils/CountVisits.h"
146#include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
147#include "llvm/Transforms/Utils/ExtraPassManager.h"
148#include "llvm/Transforms/Utils/InjectTLIMappings.h"
149#include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
150#include "llvm/Transforms/Utils/LowerCommentStringPass.h"
151#include "llvm/Transforms/Utils/Mem2Reg.h"
152#include "llvm/Transforms/Utils/MoveAutoInit.h"
153#include "llvm/Transforms/Utils/NameAnonGlobals.h"
154#include "llvm/Transforms/Utils/RelLookupTableConverter.h"
155#include "llvm/Transforms/Utils/SimplifyCFGOptions.h"
156#include "llvm/Transforms/Utils/TriggerCrashPass.h"
157#include "llvm/Transforms/Vectorize/LoopVectorize.h"
158#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
159#include "llvm/Transforms/Vectorize/VectorCombine.h"
160
161using namespace llvm;
162
163namespace llvm {
164
165static cl::opt<InliningAdvisorMode> UseInlineAdvisor(
166 "enable-ml-inliner", cl::init(Val: InliningAdvisorMode::Default), cl::Hidden,
167 cl::desc("Enable ML policy for inliner. Currently trained for -Oz only"),
168 cl::values(clEnumValN(InliningAdvisorMode::Default, "default",
169 "Heuristics-based inliner version"),
170 clEnumValN(InliningAdvisorMode::Development, "development",
171 "Use development mode (runtime-loadable model)"),
172 clEnumValN(InliningAdvisorMode::Release, "release",
173 "Use release mode (AOT-compiled model)")));
174
175/// Flag to enable inline deferral during PGO.
176static cl::opt<bool>
177 EnablePGOInlineDeferral("enable-npm-pgo-inline-deferral", cl::init(Val: true),
178 cl::Hidden,
179 cl::desc("Enable inline deferral during PGO"));
180
181static cl::opt<bool> EnableModuleInliner("enable-module-inliner",
182 cl::init(Val: false), cl::Hidden,
183 cl::desc("Enable module inliner"));
184
185static cl::opt<bool> PerformMandatoryInliningsFirst(
186 "mandatory-inlining-first", cl::init(Val: false), cl::Hidden,
187 cl::desc("Perform mandatory inlinings module-wide, before performing "
188 "inlining"));
189
190static cl::opt<bool> EnableEagerlyInvalidateAnalyses(
191 "eagerly-invalidate-analyses", cl::init(Val: true), cl::Hidden,
192 cl::desc("Eagerly invalidate more analyses in default pipelines"));
193
194static cl::opt<bool> EnableMergeFunctions(
195 "enable-merge-functions", cl::init(Val: false), cl::Hidden,
196 cl::desc("Enable function merging as part of the optimization pipeline"));
197
198static cl::opt<bool> EnablePostPGOLoopRotation(
199 "enable-post-pgo-loop-rotation", cl::init(Val: true), cl::Hidden,
200 cl::desc("Run the loop rotation transformation after PGO instrumentation"));
201
202static cl::opt<bool>
203 TriggerCrash("opt-pipeline-trigger-crash", cl::init(Val: false), cl::Hidden,
204 cl::desc("Trigger crash in optimization pipeline"));
205
206static cl::opt<bool> EnableGlobalAnalyses(
207 "enable-global-analyses", cl::init(Val: true), cl::Hidden,
208 cl::desc("Enable inter-procedural analyses"));
209
210static cl::opt<bool> RunPartialInlining("enable-partial-inlining",
211 cl::init(Val: false), cl::Hidden,
212 cl::desc("Run Partial inlining pass"));
213
214static cl::opt<bool> ExtraVectorizerPasses(
215 "extra-vectorizer-passes", cl::init(Val: false), cl::Hidden,
216 cl::desc("Run cleanup optimization passes after vectorization"));
217
218static cl::opt<bool> RunNewGVN("enable-newgvn", cl::init(Val: false), cl::Hidden,
219 cl::desc("Run the NewGVN pass"));
220
221static cl::opt<bool>
222 EnableLoopInterchange("enable-loopinterchange", cl::init(Val: true), cl::Hidden,
223 cl::desc("Enable the LoopInterchange Pass"));
224
225static cl::opt<bool> EnableUnrollAndJam("enable-unroll-and-jam",
226 cl::init(Val: false), cl::Hidden,
227 cl::desc("Enable Unroll And Jam Pass"));
228
229static cl::opt<bool> EnableLoopFlatten("enable-loop-flatten", cl::init(Val: false),
230 cl::Hidden,
231 cl::desc("Enable the LoopFlatten Pass"));
232
233static cl::opt<bool>
234 EnableInstrumentor("enable-instrumentor", cl::init(Val: false), cl::Hidden,
235 cl::desc("Enable the Instrumentor Pass"));
236
237static cl::opt<bool>
238 EnableDFAJumpThreading("enable-dfa-jump-thread",
239 cl::desc("Enable DFA jump threading"),
240 cl::init(Val: true), cl::Hidden);
241
242static cl::opt<bool>
243 EnableHotColdSplit("hot-cold-split",
244 cl::desc("Enable hot-cold splitting pass"));
245
246static cl::opt<bool> EnableIROutliner("ir-outliner", cl::init(Val: false),
247 cl::Hidden,
248 cl::desc("Enable ir outliner pass"));
249
250static cl::opt<bool>
251 DisablePreInliner("disable-preinline", cl::init(Val: false), cl::Hidden,
252 cl::desc("Disable pre-instrumentation inliner"));
253
254static cl::opt<int> PreInlineThreshold(
255 "preinline-threshold", cl::Hidden, cl::init(Val: 75),
256 cl::desc("Control the amount of inlining in pre-instrumentation inliner "
257 "(default = 75)"));
258
259static cl::opt<bool>
260 EnableGVNHoist("enable-gvn-hoist",
261 cl::desc("Enable the GVN hoisting pass (default = off)"));
262
263static cl::opt<bool>
264 EnableGVNSink("enable-gvn-sink",
265 cl::desc("Enable the GVN sinking pass (default = off)"));
266
267static cl::opt<bool> EnableJumpTableToSwitch(
268 "enable-jump-table-to-switch", cl::init(Val: true),
269 cl::desc("Enable JumpTableToSwitch pass (default = true)"));
270
271// This option is used in simplifying testing SampleFDO optimizations for
272// profile loading.
273static cl::opt<bool>
274 EnableCHR("enable-chr", cl::init(Val: true), cl::Hidden,
275 cl::desc("Enable control height reduction optimization (CHR)"));
276
277static cl::opt<bool> FlattenedProfileUsed(
278 "flattened-profile-used", cl::init(Val: false), cl::Hidden,
279 cl::desc("Indicate the sample profile being used is flattened, i.e., "
280 "no inline hierarchy exists in the profile"));
281
282static cl::opt<bool>
283 EnableMatrix("enable-matrix", cl::init(Val: false), cl::Hidden,
284 cl::desc("Enable lowering of the matrix intrinsics"));
285
286static cl::opt<bool> EnableMergeICmps(
287 "enable-mergeicmps", cl::init(Val: true), cl::Hidden,
288 cl::desc("Enable MergeICmps pass in the optimization pipeline"));
289
290static cl::opt<bool> EnableConstraintElimination(
291 "enable-constraint-elimination", cl::init(Val: true), cl::Hidden,
292 cl::desc(
293 "Enable pass to eliminate conditions based on linear constraints"));
294
295static cl::opt<AttributorRunOption> AttributorRun(
296 "attributor-enable", cl::Hidden, cl::init(Val: AttributorRunOption::NONE),
297 cl::desc("Enable the attributor inter-procedural deduction pass"),
298 cl::values(clEnumValN(AttributorRunOption::FULL, "full",
299 "enable all full attributor runs"),
300 clEnumValN(AttributorRunOption::LIGHT, "light",
301 "enable all attributor-light runs"),
302 clEnumValN(AttributorRunOption::MODULE, "module",
303 "enable module-wide attributor runs"),
304 clEnumValN(AttributorRunOption::MODULE_LIGHT, "module-light",
305 "enable module-wide attributor-light runs"),
306 clEnumValN(AttributorRunOption::CGSCC, "cgscc",
307 "enable call graph SCC attributor runs"),
308 clEnumValN(AttributorRunOption::CGSCC_LIGHT, "cgscc-light",
309 "enable call graph SCC attributor-light runs"),
310 clEnumValN(AttributorRunOption::NONE, "none",
311 "disable attributor runs")));
312
313static cl::opt<bool> EnableSampledInstr(
314 "enable-sampled-instrumentation", cl::init(Val: false), cl::Hidden,
315 cl::desc("Enable profile instrumentation sampling (default = off)"));
316static cl::opt<bool> UseLoopVersioningLICM(
317 "enable-loop-versioning-licm", cl::init(Val: false), cl::Hidden,
318 cl::desc("Enable the experimental Loop Versioning LICM pass"));
319
320static cl::opt<std::string> InstrumentColdFuncOnlyPath(
321 "instrument-cold-function-only-path", cl::init(Val: ""),
322 cl::desc("File path for cold function only instrumentation(requires use "
323 "with --pgo-instrument-cold-function-only)"),
324 cl::Hidden);
325
326// TODO: There is a similar flag in WPD pass, we should consolidate them by
327// parsing the option only once in PassBuilder and share it across both places.
328static cl::opt<bool> EnableDevirtualizeSpeculatively(
329 "enable-devirtualize-speculatively",
330 cl::desc("Enable speculative devirtualization optimization"),
331 cl::init(Val: false));
332
333extern cl::opt<std::string> UseCtxProfile;
334extern cl::opt<bool> PGOInstrumentColdFunctionOnly;
335
336extern cl::opt<bool> EnableMemProfContextDisambiguation;
337} // namespace llvm
338
339PipelineTuningOptions::PipelineTuningOptions() {
340 LoopInterleaving = true;
341 LoopVectorization = true;
342 SLPVectorization = false;
343 LoopUnrolling = true;
344 LoopInterchange = EnableLoopInterchange;
345 LoopFusion = false;
346 ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll;
347 LicmMssaOptCap = SetLicmMssaOptCap;
348 LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;
349 CallGraphProfile = true;
350 UnifiedLTO = false;
351 MergeFunctions = EnableMergeFunctions;
352 InlinerThreshold = -1;
353 EagerlyInvalidateAnalyses = EnableEagerlyInvalidateAnalyses;
354 DevirtualizeSpeculatively = EnableDevirtualizeSpeculatively;
355}
356
357namespace llvm {
358extern cl::opt<unsigned> MaxDevirtIterations;
359} // namespace llvm
360
361void PassBuilder::invokePeepholeEPCallbacks(FunctionPassManager &FPM,
362 OptimizationLevel Level) {
363 for (auto &C : PeepholeEPCallbacks)
364 C(FPM, Level);
365}
366void PassBuilder::invokeLateLoopOptimizationsEPCallbacks(
367 LoopPassManager &LPM, OptimizationLevel Level) {
368 for (auto &C : LateLoopOptimizationsEPCallbacks)
369 C(LPM, Level);
370}
371void PassBuilder::invokeLoopOptimizerEndEPCallbacks(LoopPassManager &LPM,
372 OptimizationLevel Level) {
373 for (auto &C : LoopOptimizerEndEPCallbacks)
374 C(LPM, Level);
375}
376void PassBuilder::invokeScalarOptimizerLateEPCallbacks(
377 FunctionPassManager &FPM, OptimizationLevel Level) {
378 for (auto &C : ScalarOptimizerLateEPCallbacks)
379 C(FPM, Level);
380}
381void PassBuilder::invokeCGSCCOptimizerLateEPCallbacks(CGSCCPassManager &CGPM,
382 OptimizationLevel Level) {
383 for (auto &C : CGSCCOptimizerLateEPCallbacks)
384 C(CGPM, Level);
385}
386void PassBuilder::invokeVectorizerStartEPCallbacks(FunctionPassManager &FPM,
387 OptimizationLevel Level) {
388 for (auto &C : VectorizerStartEPCallbacks)
389 C(FPM, Level);
390}
391void PassBuilder::invokeVectorizerEndEPCallbacks(FunctionPassManager &FPM,
392 OptimizationLevel Level) {
393 for (auto &C : VectorizerEndEPCallbacks)
394 C(FPM, Level);
395}
396void PassBuilder::invokeOptimizerEarlyEPCallbacks(ModulePassManager &MPM,
397 OptimizationLevel Level,
398 ThinOrFullLTOPhase Phase) {
399 for (auto &C : OptimizerEarlyEPCallbacks)
400 C(MPM, Level, Phase);
401}
402void PassBuilder::invokeOptimizerLastEPCallbacks(ModulePassManager &MPM,
403 OptimizationLevel Level,
404 ThinOrFullLTOPhase Phase) {
405 for (auto &C : OptimizerLastEPCallbacks)
406 C(MPM, Level, Phase);
407}
408void PassBuilder::invokeFullLinkTimeOptimizationEarlyEPCallbacks(
409 ModulePassManager &MPM, OptimizationLevel Level) {
410 for (auto &C : FullLinkTimeOptimizationEarlyEPCallbacks)
411 C(MPM, Level);
412}
413void PassBuilder::invokeFullLinkTimeOptimizationLastEPCallbacks(
414 ModulePassManager &MPM, OptimizationLevel Level) {
415 for (auto &C : FullLinkTimeOptimizationLastEPCallbacks)
416 C(MPM, Level);
417}
418void PassBuilder::invokePipelineStartEPCallbacks(ModulePassManager &MPM,
419 OptimizationLevel Level) {
420 for (auto &C : PipelineStartEPCallbacks)
421 C(MPM, Level);
422}
423void PassBuilder::invokePipelineEarlySimplificationEPCallbacks(
424 ModulePassManager &MPM, OptimizationLevel Level, ThinOrFullLTOPhase Phase) {
425 for (auto &C : PipelineEarlySimplificationEPCallbacks)
426 C(MPM, Level, Phase);
427}
428
429// Get IR stats with InstCount before/after the optimization pipeline
430static void instructionCountersPass(ModulePassManager &MPM,
431 bool IsPreOptimization) {
432 if (AreStatisticsEnabled()) {
433 MPM.addPass(
434 Pass: createModuleToFunctionPassAdaptor(Pass: InstCountPass(IsPreOptimization)));
435 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
436 Pass: FunctionPropertiesStatisticsPass(IsPreOptimization)));
437 }
438}
439
440// Helper to add AnnotationRemarksPass.
441static void addAnnotationRemarksPass(ModulePassManager &MPM) {
442 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: AnnotationRemarksPass()));
443}
444
445// Helper to check if the current compilation phase is preparing for LTO
446static bool isLTOPreLink(ThinOrFullLTOPhase Phase) {
447 return Phase == ThinOrFullLTOPhase::ThinLTOPreLink ||
448 Phase == ThinOrFullLTOPhase::FullLTOPreLink;
449}
450
451// Helper to check if the current compilation phase is preparing for FullLTO
452[[maybe_unused]] static bool isFullLTOPreLink(ThinOrFullLTOPhase Phase) {
453 return Phase == ThinOrFullLTOPhase::FullLTOPreLink;
454}
455
456// Helper to check if the current compilation phase is preparing for ThinLTO
457static bool isThinLTOPreLink(ThinOrFullLTOPhase Phase) {
458 return Phase == ThinOrFullLTOPhase::ThinLTOPreLink;
459}
460
461// Helper to check if the current compilation phase is LTO backend
462static bool isLTOPostLink(ThinOrFullLTOPhase Phase) {
463 return Phase == ThinOrFullLTOPhase::ThinLTOPostLink ||
464 Phase == ThinOrFullLTOPhase::FullLTOPostLink;
465}
466
467// Helper to check if the current compilation phase is FullLTO backend
468static bool isFullLTOPostLink(ThinOrFullLTOPhase Phase) {
469 return Phase == ThinOrFullLTOPhase::FullLTOPostLink;
470}
471
472// Helper to check if the current compilation phase is ThinLTO backend
473static bool isThinLTOPostLink(ThinOrFullLTOPhase Phase) {
474 return Phase == ThinOrFullLTOPhase::ThinLTOPostLink;
475}
476
477// Helper to wrap conditionally Coro passes.
478static CoroConditionalWrapper buildCoroWrapper(ThinOrFullLTOPhase Phase) {
479 // TODO: Skip passes according to Phase.
480 ModulePassManager CoroPM;
481 CoroPM.addPass(Pass: CoroEarlyPass());
482 CGSCCPassManager CGPM;
483 CGPM.addPass(Pass: CoroSplitPass());
484 CoroPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: std::move(CGPM)));
485 CoroPM.addPass(Pass: CoroCleanupPass());
486 CoroPM.addPass(Pass: GlobalDCEPass());
487 return CoroConditionalWrapper(std::move(CoroPM));
488}
489
490// TODO: Investigate the cost/benefit of tail call elimination on debugging.
491FunctionPassManager
492PassBuilder::buildO1FunctionSimplificationPipeline(OptimizationLevel Level,
493 ThinOrFullLTOPhase Phase) {
494
495 FunctionPassManager FPM;
496
497 if (AreStatisticsEnabled())
498 FPM.addPass(Pass: CountVisitsPass());
499
500 // Form SSA out of local memory accesses after breaking apart aggregates into
501 // scalars.
502 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
503
504 // Catch trivial redundancies
505 FPM.addPass(Pass: EarlyCSEPass(true /* Enable mem-ssa. */));
506
507 // Hoisting of scalars and load expressions.
508 FPM.addPass(
509 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
510 FPM.addPass(Pass: InstCombinePass());
511
512 FPM.addPass(Pass: LibCallsShrinkWrapPass());
513
514 invokePeepholeEPCallbacks(FPM, Level);
515
516 FPM.addPass(
517 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
518
519 // Form canonically associated expression trees, and simplify the trees using
520 // basic mathematical properties. For example, this will form (nearly)
521 // minimal multiplication trees.
522 FPM.addPass(Pass: ReassociatePass());
523
524 // Add the primary loop simplification pipeline.
525 // FIXME: Currently this is split into two loop pass pipelines because we run
526 // some function passes in between them. These can and should be removed
527 // and/or replaced by scheduling the loop pass equivalents in the correct
528 // positions. But those equivalent passes aren't powerful enough yet.
529 // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
530 // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
531 // fully replace `SimplifyCFGPass`, and the closest to the other we have is
532 // `LoopInstSimplify`.
533 LoopPassManager LPM1, LPM2;
534
535 // Simplify the loop body. We do this initially to clean up after other loop
536 // passes run, either when iterating on a loop or on inner loops with
537 // implications on the outer loop.
538 LPM1.addPass(Pass: LoopInstSimplifyPass());
539 LPM1.addPass(Pass: LoopSimplifyCFGPass());
540
541 // Try to remove as much code from the loop header as possible,
542 // to reduce amount of IR that will have to be duplicated. However,
543 // do not perform speculative hoisting the first time as LICM
544 // will destroy metadata that may not need to be destroyed if run
545 // after loop rotation.
546 // TODO: Investigate promotion cap for O1.
547 LPM1.addPass(Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
548 /*AllowSpeculation=*/false));
549
550 LPM1.addPass(
551 Pass: LoopRotatePass(/*EnableHeaderDuplication=*/true, isLTOPreLink(Phase)));
552 // TODO: Investigate promotion cap for O1.
553 LPM1.addPass(Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
554 /*AllowSpeculation=*/true));
555 LPM1.addPass(Pass: SimpleLoopUnswitchPass());
556 if (EnableLoopFlatten)
557 LPM1.addPass(Pass: LoopFlattenPass());
558
559 LPM2.addPass(Pass: LoopIdiomRecognizePass());
560 LPM2.addPass(Pass: IndVarSimplifyPass());
561
562 invokeLateLoopOptimizationsEPCallbacks(LPM&: LPM2, Level);
563
564 LPM2.addPass(Pass: LoopDeletionPass());
565
566 // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
567 // because it changes IR to makes profile annotation in back compile
568 // inaccurate. The normal unroller doesn't pay attention to forced full unroll
569 // attributes so we need to make sure and allow the full unroll pass to pay
570 // attention to it.
571 if (!isThinLTOPreLink(Phase) || !PGOOpt ||
572 PGOOpt->Action != PGOOptions::SampleUse)
573 LPM2.addPass(Pass: LoopFullUnrollPass(static_cast<int>(Level),
574 /* OnlyWhenForced= */ !PTO.LoopUnrolling,
575 PTO.ForgetAllSCEVInLoopUnroll));
576
577 invokeLoopOptimizerEndEPCallbacks(LPM&: LPM2, Level);
578
579 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM1),
580 /*UseMemorySSA=*/true));
581 FPM.addPass(
582 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
583 FPM.addPass(Pass: InstCombinePass());
584 // The loop passes in LPM2 (LoopFullUnrollPass) do not preserve MemorySSA.
585 // *All* loop passes must preserve it, in order to be able to use it.
586 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM2),
587 /*UseMemorySSA=*/false));
588
589 // Delete small array after loop unroll.
590 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
591
592 // Specially optimize memory movement as it doesn't look like dataflow in SSA.
593 FPM.addPass(Pass: MemCpyOptPass());
594
595 // Sparse conditional constant propagation.
596 // FIXME: It isn't clear why we do this *after* loop passes rather than
597 // before...
598 FPM.addPass(Pass: SCCPPass());
599
600 // Delete dead bit computations (instcombine runs after to fold away the dead
601 // computations, and then ADCE will run later to exploit any new DCE
602 // opportunities that creates).
603 FPM.addPass(Pass: BDCEPass());
604
605 // Run instcombine after redundancy and dead bit elimination to exploit
606 // opportunities opened up by them.
607 FPM.addPass(Pass: InstCombinePass());
608 invokePeepholeEPCallbacks(FPM, Level);
609
610 FPM.addPass(Pass: CoroElidePass());
611
612 invokeScalarOptimizerLateEPCallbacks(FPM, Level);
613
614 // Finally, do an expensive DCE pass to catch all the dead code exposed by
615 // the simplifications and basic cleanup after all the simplifications.
616 // TODO: Investigate if this is too expensive.
617 FPM.addPass(Pass: ADCEPass());
618 FPM.addPass(
619 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
620 FPM.addPass(Pass: InstCombinePass());
621 invokePeepholeEPCallbacks(FPM, Level);
622
623 return FPM;
624}
625
626FunctionPassManager
627PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
628 ThinOrFullLTOPhase Phase) {
629 assert(Level != OptimizationLevel::O0 && "Must request optimizations!");
630
631 // The O1 pipeline has a separate pipeline creation function to simplify
632 // construction readability.
633 if (Level == OptimizationLevel::O1)
634 return buildO1FunctionSimplificationPipeline(Level, Phase);
635
636 FunctionPassManager FPM;
637
638 if (AreStatisticsEnabled())
639 FPM.addPass(Pass: CountVisitsPass());
640
641 // Form SSA out of local memory accesses after breaking apart aggregates into
642 // scalars.
643 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
644
645 // Catch trivial redundancies
646 FPM.addPass(Pass: EarlyCSEPass(true /* Enable mem-ssa. */));
647 if (EnableKnowledgeRetention)
648 FPM.addPass(Pass: AssumeSimplifyPass());
649
650 // Hoisting of scalars and load expressions.
651 if (EnableGVNHoist)
652 FPM.addPass(Pass: GVNHoistPass());
653
654 // Global value numbering based sinking.
655 if (EnableGVNSink) {
656 FPM.addPass(Pass: GVNSinkPass());
657 FPM.addPass(
658 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
659 }
660
661 // Speculative execution if the target has divergent branches; otherwise nop.
662 FPM.addPass(Pass: SpeculativeExecutionPass(/* OnlyIfDivergentTarget =*/true));
663
664 // Optimize based on known information about branches, and cleanup afterward.
665 FPM.addPass(Pass: JumpThreadingPass());
666 FPM.addPass(Pass: CorrelatedValuePropagationPass());
667
668 // Jump table to switch conversion.
669 if (EnableJumpTableToSwitch)
670 FPM.addPass(Pass: JumpTableToSwitchPass(/*InLTO=*/isLTOPostLink(Phase)));
671
672 FPM.addPass(
673 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
674 FPM.addPass(Pass: InstCombinePass());
675 FPM.addPass(Pass: AggressiveInstCombinePass());
676 FPM.addPass(Pass: LibCallsShrinkWrapPass());
677
678 invokePeepholeEPCallbacks(FPM, Level);
679
680 // For PGO use pipeline, try to optimize memory intrinsics such as memcpy
681 // using the size value profile. Don't perform this when optimizing for size.
682 if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse)
683 FPM.addPass(Pass: PGOMemOPSizeOpt());
684
685 FPM.addPass(Pass: TailCallElimPass(/*UpdateFunctionEntryCount=*/
686 isInstrumentedPGOUse()));
687 FPM.addPass(
688 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
689
690 // Form canonically associated expression trees, and simplify the trees using
691 // basic mathematical properties. For example, this will form (nearly)
692 // minimal multiplication trees.
693 FPM.addPass(Pass: ReassociatePass());
694
695 if (EnableConstraintElimination)
696 FPM.addPass(Pass: ConstraintEliminationPass());
697
698 // Add the primary loop simplification pipeline.
699 // FIXME: Currently this is split into two loop pass pipelines because we run
700 // some function passes in between them. These can and should be removed
701 // and/or replaced by scheduling the loop pass equivalents in the correct
702 // positions. But those equivalent passes aren't powerful enough yet.
703 // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
704 // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
705 // fully replace `SimplifyCFGPass`, and the closest to the other we have is
706 // `LoopInstSimplify`.
707 LoopPassManager LPM1, LPM2;
708
709 // Simplify the loop body. We do this initially to clean up after other loop
710 // passes run, either when iterating on a loop or on inner loops with
711 // implications on the outer loop.
712 LPM1.addPass(Pass: LoopInstSimplifyPass());
713 LPM1.addPass(Pass: LoopSimplifyCFGPass());
714
715 // Try to remove as much code from the loop header as possible,
716 // to reduce amount of IR that will have to be duplicated. However,
717 // do not perform speculative hoisting the first time as LICM
718 // will destroy metadata that may not need to be destroyed if run
719 // after loop rotation.
720 // TODO: Investigate promotion cap for O1.
721 LPM1.addPass(Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
722 /*AllowSpeculation=*/false));
723
724 LPM1.addPass(
725 Pass: LoopRotatePass(/*EnableHeaderDuplication=*/true, isLTOPreLink(Phase)));
726 // TODO: Investigate promotion cap for O1.
727 LPM1.addPass(Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
728 /*AllowSpeculation=*/true));
729 LPM1.addPass(
730 Pass: SimpleLoopUnswitchPass(/* NonTrivial */ Level == OptimizationLevel::O3));
731 if (EnableLoopFlatten)
732 LPM1.addPass(Pass: LoopFlattenPass());
733
734 LPM2.addPass(Pass: LoopIdiomRecognizePass());
735 LPM2.addPass(Pass: IndVarSimplifyPass());
736
737 {
738 ExtraLoopPassManager<ShouldRunExtraSimpleLoopUnswitch> ExtraPasses;
739 ExtraPasses.addPass(Pass: SimpleLoopUnswitchPass(/* NonTrivial */ Level ==
740 OptimizationLevel::O3));
741 LPM2.addPass(Pass: std::move(ExtraPasses));
742 }
743
744 invokeLateLoopOptimizationsEPCallbacks(LPM&: LPM2, Level);
745
746 LPM2.addPass(Pass: LoopDeletionPass());
747
748 // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
749 // because it changes IR to makes profile annotation in back compile
750 // inaccurate. The normal unroller doesn't pay attention to forced full unroll
751 // attributes so we need to make sure and allow the full unroll pass to pay
752 // attention to it.
753 if (!isThinLTOPreLink(Phase) || !PGOOpt ||
754 PGOOpt->Action != PGOOptions::SampleUse)
755 LPM2.addPass(Pass: LoopFullUnrollPass(static_cast<int>(Level),
756 /* OnlyWhenForced= */ !PTO.LoopUnrolling,
757 PTO.ForgetAllSCEVInLoopUnroll));
758
759 invokeLoopOptimizerEndEPCallbacks(LPM&: LPM2, Level);
760
761 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM1),
762 /*UseMemorySSA=*/true));
763 FPM.addPass(
764 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
765 FPM.addPass(Pass: InstCombinePass());
766 // The loop passes in LPM2 (LoopIdiomRecognizePass, IndVarSimplifyPass,
767 // LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA.
768 // *All* loop passes must preserve it, in order to be able to use it.
769 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM2),
770 /*UseMemorySSA=*/false));
771
772 // Delete small array after loop unroll.
773 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
774
775 // Try vectorization/scalarization transforms that are both improvements
776 // themselves and can allow further folds with GVN and InstCombine.
777 FPM.addPass(Pass: VectorCombinePass(/*TryEarlyFoldsOnly=*/true));
778
779 // Eliminate redundancies.
780 FPM.addPass(Pass: MergedLoadStoreMotionPass());
781 if (RunNewGVN)
782 FPM.addPass(Pass: NewGVNPass());
783 else
784 FPM.addPass(Pass: GVNPass());
785
786 // Sparse conditional constant propagation.
787 // FIXME: It isn't clear why we do this *after* loop passes rather than
788 // before...
789 FPM.addPass(Pass: SCCPPass());
790
791 // Delete dead bit computations (instcombine runs after to fold away the dead
792 // computations, and then ADCE will run later to exploit any new DCE
793 // opportunities that creates).
794 FPM.addPass(Pass: BDCEPass());
795
796 // Run instcombine after redundancy and dead bit elimination to exploit
797 // opportunities opened up by them.
798 FPM.addPass(Pass: InstCombinePass());
799 invokePeepholeEPCallbacks(FPM, Level);
800
801 // Re-consider control flow based optimizations after redundancy elimination,
802 // redo DCE, etc.
803 if (EnableDFAJumpThreading)
804 FPM.addPass(Pass: DFAJumpThreadingPass());
805
806 FPM.addPass(Pass: JumpThreadingPass());
807 FPM.addPass(Pass: CorrelatedValuePropagationPass());
808
809 // Finally, do an expensive DCE pass to catch all the dead code exposed by
810 // the simplifications and basic cleanup after all the simplifications.
811 // TODO: Investigate if this is too expensive.
812 FPM.addPass(Pass: ADCEPass());
813
814 // Specially optimize memory movement as it doesn't look like dataflow in SSA.
815 FPM.addPass(Pass: MemCpyOptPass());
816
817 FPM.addPass(Pass: DSEPass());
818 FPM.addPass(Pass: MoveAutoInitPass());
819
820 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(
821 Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
822 /*AllowSpeculation=*/true),
823 /*UseMemorySSA=*/true));
824
825 FPM.addPass(Pass: CoroElidePass());
826
827 invokeScalarOptimizerLateEPCallbacks(FPM, Level);
828
829 FPM.addPass(Pass: SimplifyCFGPass(SimplifyCFGOptions()
830 .convertSwitchRangeToICmp(B: true)
831 .convertSwitchToArithmetic(B: true)
832 .hoistCommonInsts(B: true)
833 .sinkCommonInsts(B: true)));
834 FPM.addPass(Pass: InstCombinePass());
835 invokePeepholeEPCallbacks(FPM, Level);
836
837 return FPM;
838}
839
840void PassBuilder::addRequiredLTOPreLinkPasses(ModulePassManager &MPM) {
841 MPM.addPass(Pass: CanonicalizeAliasesPass());
842 MPM.addPass(Pass: NameAnonGlobalPass());
843 MPM.addPass(Pass: AssignGUIDPass());
844}
845
846void PassBuilder::addPreInlinerPasses(ModulePassManager &MPM,
847 OptimizationLevel Level,
848 ThinOrFullLTOPhase LTOPhase) {
849 assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");
850 if (DisablePreInliner)
851 return;
852 InlineParams IP;
853
854 IP.DefaultThreshold = PreInlineThreshold;
855
856 // FIXME: The hint threshold has the same value used by the regular inliner
857 // when not optimzing for size. This should probably be lowered after
858 // performance testing.
859 // FIXME: this comment is cargo culted from the old pass manager, revisit).
860 IP.HintThreshold = 325;
861 IP.OptSizeHintThreshold = PreInlineThreshold;
862 ModuleInlinerWrapperPass MIWP(
863 IP, /* MandatoryFirst */ true,
864 InlineContext{.LTOPhase: LTOPhase, .Pass: InlinePass::EarlyInliner});
865 CGSCCPassManager &CGPipeline = MIWP.getPM();
866
867 FunctionPassManager FPM;
868 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
869 FPM.addPass(Pass: EarlyCSEPass()); // Catch trivial redundancies.
870 FPM.addPass(Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(
871 B: true))); // Merge & remove basic blocks.
872 FPM.addPass(Pass: InstCombinePass()); // Combine silly sequences.
873 invokePeepholeEPCallbacks(FPM, Level);
874
875 CGPipeline.addPass(Pass: createCGSCCToFunctionPassAdaptor(
876 Pass: std::move(FPM), EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
877
878 MPM.addPass(Pass: std::move(MIWP));
879
880 // Delete anything that is now dead to make sure that we don't instrument
881 // dead code. Instrumentation can end up keeping dead code around and
882 // dramatically increase code size.
883 MPM.addPass(Pass: GlobalDCEPass());
884}
885
886void PassBuilder::addPostPGOLoopRotation(ModulePassManager &MPM,
887 OptimizationLevel Level) {
888 if (EnablePostPGOLoopRotation) {
889 // Disable header duplication in loop rotation at -Oz.
890 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
891 Pass: createFunctionToLoopPassAdaptor(Pass: LoopRotatePass(),
892 /*UseMemorySSA=*/false),
893 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
894 }
895}
896
897void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM,
898 OptimizationLevel Level, bool RunProfileGen,
899 bool IsCS, bool AtomicCounterUpdate,
900 std::string ProfileFile,
901 std::string ProfileRemappingFile) {
902 assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");
903
904 if (!RunProfileGen) {
905 assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
906 MPM.addPass(
907 Pass: PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS, FS));
908 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
909 // RequireAnalysisPass for PSI before subsequent non-module passes.
910 MPM.addPass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
911 return;
912 }
913
914 // Perform PGO instrumentation.
915 MPM.addPass(Pass: PGOInstrumentationGen(IsCS ? PGOInstrumentationType::CSFDO
916 : PGOInstrumentationType::FDO));
917
918 addPostPGOLoopRotation(MPM, Level);
919 // Add the profile lowering pass.
920 InstrProfOptions Options;
921 if (!ProfileFile.empty())
922 Options.InstrProfileOutput = ProfileFile;
923 // Do counter promotion at Level greater than O0.
924 Options.DoCounterPromotion = true;
925 Options.UseBFIInPromotion = IsCS;
926 if (EnableSampledInstr) {
927 Options.Sampling = true;
928 // With sampling, there is little beneifit to enable counter promotion.
929 // But note that sampling does work with counter promotion.
930 Options.DoCounterPromotion = false;
931 }
932 Options.Atomic = AtomicCounterUpdate;
933 MPM.addPass(Pass: InstrProfilingLoweringPass(Options, IsCS));
934}
935
936void PassBuilder::addPGOInstrPassesForO0(ModulePassManager &MPM,
937 bool RunProfileGen, bool IsCS,
938 bool AtomicCounterUpdate,
939 std::string ProfileFile,
940 std::string ProfileRemappingFile) {
941 if (!RunProfileGen) {
942 assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
943 MPM.addPass(
944 Pass: PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS, FS));
945 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
946 // RequireAnalysisPass for PSI before subsequent non-module passes.
947 MPM.addPass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
948 return;
949 }
950
951 // Perform PGO instrumentation.
952 MPM.addPass(Pass: PGOInstrumentationGen(IsCS ? PGOInstrumentationType::CSFDO
953 : PGOInstrumentationType::FDO));
954 // Add the profile lowering pass.
955 InstrProfOptions Options;
956 if (!ProfileFile.empty())
957 Options.InstrProfileOutput = ProfileFile;
958 // Do not do counter promotion at O0.
959 Options.DoCounterPromotion = false;
960 Options.UseBFIInPromotion = IsCS;
961 Options.Atomic = AtomicCounterUpdate;
962 MPM.addPass(Pass: InstrProfilingLoweringPass(Options, IsCS));
963}
964
965static InlineParams getInlineParamsFromOptLevel(OptimizationLevel Level) {
966 return getInlineParamsFromOptLevel(OptLevel: static_cast<unsigned>(Level));
967}
968
969ModuleInlinerWrapperPass
970PassBuilder::buildInlinerPipeline(OptimizationLevel Level,
971 ThinOrFullLTOPhase Phase) {
972 InlineParams IP;
973 if (PTO.InlinerThreshold == -1)
974 IP = ::getInlineParamsFromOptLevel(Level);
975 else
976 IP = getInlineParams(Threshold: PTO.InlinerThreshold);
977 // For PreLinkThinLTO + SamplePGO or PreLinkFullLTO + SamplePGO,
978 // set hot-caller threshold to 0 to disable hot
979 // callsite inline (as much as possible [1]) because it makes
980 // profile annotation in the backend inaccurate.
981 //
982 // [1] Note the cost of a function could be below zero due to erased
983 // prologue / epilogue.
984 if (isLTOPreLink(Phase) && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse)
985 IP.HotCallSiteThreshold = 0;
986
987 if (PGOOpt)
988 IP.EnableDeferral = EnablePGOInlineDeferral;
989
990 ModuleInlinerWrapperPass MIWP(IP, PerformMandatoryInliningsFirst,
991 InlineContext{.LTOPhase: Phase, .Pass: InlinePass::CGSCCInliner},
992 UseInlineAdvisor, MaxDevirtIterations);
993
994 // Require the GlobalsAA analysis for the module so we can query it within
995 // the CGSCC pipeline.
996 if (EnableGlobalAnalyses) {
997 MIWP.addModulePass(Pass: RequireAnalysisPass<GlobalsAA, Module>());
998 // Invalidate AAManager so it can be recreated and pick up the newly
999 // available GlobalsAA.
1000 MIWP.addModulePass(
1001 Pass: createModuleToFunctionPassAdaptor(Pass: InvalidateAnalysisPass<AAManager>()));
1002 }
1003
1004 // Require the ProfileSummaryAnalysis for the module so we can query it within
1005 // the inliner pass.
1006 MIWP.addModulePass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
1007
1008 // Now begin the main postorder CGSCC pipeline.
1009 // FIXME: The current CGSCC pipeline has its origins in the legacy pass
1010 // manager and trying to emulate its precise behavior. Much of this doesn't
1011 // make a lot of sense and we should revisit the core CGSCC structure.
1012 CGSCCPassManager &MainCGPipeline = MIWP.getPM();
1013
1014 // Note: historically, the PruneEH pass was run first to deduce nounwind and
1015 // generally clean up exception handling overhead. It isn't clear this is
1016 // valuable as the inliner doesn't currently care whether it is inlining an
1017 // invoke or a call.
1018
1019 if (AttributorRun & AttributorRunOption::CGSCC)
1020 MainCGPipeline.addPass(Pass: AttributorCGSCCPass());
1021 else if (AttributorRun & AttributorRunOption::CGSCC_LIGHT)
1022 MainCGPipeline.addPass(Pass: AttributorLightCGSCCPass());
1023
1024 // Deduce function attributes. We do another run of this after the function
1025 // simplification pipeline, so this only needs to run when it could affect the
1026 // function simplification pipeline, which is only the case with recursive
1027 // functions.
1028 MainCGPipeline.addPass(Pass: PostOrderFunctionAttrsPass(/*SkipNonRecursive*/ true));
1029
1030 // When at O3 add argument promotion to the pass pipeline.
1031 // FIXME: It isn't at all clear why this should be limited to O3.
1032 if (Level == OptimizationLevel::O3)
1033 MainCGPipeline.addPass(Pass: ArgumentPromotionPass());
1034
1035 // Try to perform OpenMP specific optimizations. This is a (quick!) no-op if
1036 // there are no OpenMP runtime calls present in the module.
1037 if (Level == OptimizationLevel::O2 || Level == OptimizationLevel::O3)
1038 MainCGPipeline.addPass(Pass: OpenMPOptCGSCCPass(Phase));
1039
1040 invokeCGSCCOptimizerLateEPCallbacks(CGPM&: MainCGPipeline, Level);
1041
1042 // Add the core function simplification pipeline nested inside the
1043 // CGSCC walk.
1044 MainCGPipeline.addPass(Pass: createCGSCCToFunctionPassAdaptor(
1045 Pass: buildFunctionSimplificationPipeline(Level, Phase),
1046 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses, /*NoRerun=*/true));
1047
1048 // Finally, deduce any function attributes based on the fully simplified
1049 // function.
1050 MainCGPipeline.addPass(Pass: PostOrderFunctionAttrsPass());
1051
1052 // Mark that the function is fully simplified and that it shouldn't be
1053 // simplified again if we somehow revisit it due to CGSCC mutations unless
1054 // it's been modified since.
1055 MainCGPipeline.addPass(Pass: createCGSCCToFunctionPassAdaptor(
1056 Pass: RequireAnalysisPass<ShouldNotRunFunctionPassesAnalysis, Function>()));
1057
1058 if (!isThinLTOPreLink(Phase)) {
1059 MainCGPipeline.addPass(Pass: CoroSplitPass(Level != OptimizationLevel::O0));
1060 MainCGPipeline.addPass(Pass: CoroAnnotationElidePass());
1061 }
1062
1063 // Make sure we don't affect potential future NoRerun CGSCC adaptors.
1064 MIWP.addLateModulePass(Pass: createModuleToFunctionPassAdaptor(
1065 Pass: InvalidateAnalysisPass<ShouldNotRunFunctionPassesAnalysis>()));
1066
1067 return MIWP;
1068}
1069
1070ModulePassManager
1071PassBuilder::buildModuleInlinerPipeline(OptimizationLevel Level,
1072 ThinOrFullLTOPhase Phase) {
1073 ModulePassManager MPM;
1074
1075 InlineParams IP = ::getInlineParamsFromOptLevel(Level);
1076 // For PreLinkThinLTO + SamplePGO or PreLinkFullLTO + SamplePGO,
1077 // set hot-caller threshold to 0 to disable hot
1078 // callsite inline (as much as possible [1]) because it makes
1079 // profile annotation in the backend inaccurate.
1080 //
1081 // [1] Note the cost of a function could be below zero due to erased
1082 // prologue / epilogue.
1083 if (isLTOPreLink(Phase) && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse)
1084 IP.HotCallSiteThreshold = 0;
1085
1086 if (PGOOpt)
1087 IP.EnableDeferral = EnablePGOInlineDeferral;
1088
1089 // The inline deferral logic is used to avoid losing some
1090 // inlining chance in future. It is helpful in SCC inliner, in which
1091 // inlining is processed in bottom-up order.
1092 // While in module inliner, the inlining order is a priority-based order
1093 // by default. The inline deferral is unnecessary there. So we disable the
1094 // inline deferral logic in module inliner.
1095 IP.EnableDeferral = false;
1096
1097 MPM.addPass(Pass: ModuleInlinerPass(IP, UseInlineAdvisor, Phase));
1098 if (!UseCtxProfile.empty() && Phase == ThinOrFullLTOPhase::ThinLTOPostLink) {
1099 MPM.addPass(Pass: GlobalOptPass());
1100 MPM.addPass(Pass: GlobalDCEPass());
1101 MPM.addPass(Pass: AssignGUIDPass());
1102 MPM.addPass(Pass: PGOCtxProfFlatteningPass(/*IsPreThinlink=*/false));
1103 }
1104
1105 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
1106 Pass: buildFunctionSimplificationPipeline(Level, Phase),
1107 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
1108
1109 if (!isThinLTOPreLink(Phase)) {
1110 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(
1111 Pass: CoroSplitPass(Level != OptimizationLevel::O0)));
1112 MPM.addPass(
1113 Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: CoroAnnotationElidePass()));
1114 }
1115
1116 return MPM;
1117}
1118
1119ModulePassManager
1120PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
1121 ThinOrFullLTOPhase Phase) {
1122 assert(Level != OptimizationLevel::O0 &&
1123 "Should not be used for O0 pipeline");
1124
1125 assert(!isFullLTOPostLink(Phase) &&
1126 "FullLTOPostLink shouldn't call buildModuleSimplificationPipeline!");
1127
1128 ModulePassManager MPM;
1129
1130 // Place pseudo probe instrumentation as the first pass of the pipeline to
1131 // minimize the impact of optimization changes.
1132 if (PGOOpt && PGOOpt->PseudoProbeForProfiling && !isThinLTOPostLink(Phase))
1133 MPM.addPass(Pass: SampleProfileProbePass(TM));
1134
1135 bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse);
1136
1137 // In ThinLTO mode, when flattened profile is used, all the available
1138 // profile information will be annotated in PreLink phase so there is
1139 // no need to load the profile again in PostLink.
1140 bool LoadSampleProfile =
1141 HasSampleProfile && !(FlattenedProfileUsed && isThinLTOPostLink(Phase));
1142
1143 // During the ThinLTO backend phase we perform early indirect call promotion
1144 // here, before globalopt. Otherwise imported available_externally functions
1145 // look unreferenced and are removed. If we are going to load the sample
1146 // profile then defer until later.
1147 // TODO: See if we can move later and consolidate with the location where
1148 // we perform ICP when we are loading a sample profile.
1149 // TODO: We pass HasSampleProfile (whether there was a sample profile file
1150 // passed to the compile) to the SamplePGO flag of ICP. This is used to
1151 // determine whether the new direct calls are annotated with prof metadata.
1152 // Ideally this should be determined from whether the IR is annotated with
1153 // sample profile, and not whether the a sample profile was provided on the
1154 // command line. E.g. for flattened profiles where we will not be reloading
1155 // the sample profile in the ThinLTO backend, we ideally shouldn't have to
1156 // provide the sample profile file.
1157 if (isThinLTOPostLink(Phase) && !LoadSampleProfile)
1158 MPM.addPass(Pass: PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile));
1159
1160 // Create an early function pass manager to cleanup the output of the
1161 // frontend. Not necessary with LTO post link pipelines since the pre link
1162 // pipeline already cleaned up the frontend output.
1163 if (!isThinLTOPostLink(Phase)) {
1164 // Do basic inference of function attributes from known properties of system
1165 // libraries and other oracles.
1166 MPM.addPass(Pass: InferFunctionAttrsPass());
1167 MPM.addPass(Pass: CoroEarlyPass());
1168
1169 FunctionPassManager EarlyFPM;
1170 EarlyFPM.addPass(Pass: EntryExitInstrumenterPass(/*PostInlining=*/false));
1171 // Lower llvm.expect to metadata before attempting transforms.
1172 // Compare/branch metadata may alter the behavior of passes like
1173 // SimplifyCFG.
1174 EarlyFPM.addPass(Pass: LowerExpectIntrinsicPass());
1175 EarlyFPM.addPass(Pass: SimplifyCFGPass());
1176 EarlyFPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
1177 EarlyFPM.addPass(Pass: EarlyCSEPass());
1178 if (Level == OptimizationLevel::O3)
1179 EarlyFPM.addPass(Pass: CallSiteSplittingPass());
1180 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
1181 Pass: std::move(EarlyFPM), EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
1182 }
1183
1184 if (LoadSampleProfile) {
1185 // Annotate sample profile right after early FPM to ensure freshness of
1186 // the debug info.
1187 MPM.addPass(Pass: SampleProfileLoaderPass(
1188 PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile, Phase, FS));
1189 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1190 // RequireAnalysisPass for PSI before subsequent non-module passes.
1191 MPM.addPass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
1192 // Do not invoke ICP in the LTOPrelink phase as it makes it hard
1193 // for the profile annotation to be accurate in the LTO backend.
1194 if (!isLTOPreLink(Phase))
1195 // We perform early indirect call promotion here, before globalopt.
1196 // This is important for the ThinLTO backend phase because otherwise
1197 // imported available_externally functions look unreferenced and are
1198 // removed.
1199 MPM.addPass(
1200 Pass: PGOIndirectCallPromotion(true /* IsInLTO */, true /* SamplePGO */));
1201 }
1202
1203 // Try to perform OpenMP specific optimizations on the module. This is a
1204 // (quick!) no-op if there are no OpenMP runtime calls present in the module.
1205 MPM.addPass(Pass: OpenMPOptPass(Phase));
1206
1207 if (AttributorRun & AttributorRunOption::MODULE)
1208 MPM.addPass(Pass: AttributorPass());
1209 else if (AttributorRun & AttributorRunOption::MODULE_LIGHT)
1210 MPM.addPass(Pass: AttributorLightPass());
1211
1212 // Lower type metadata and the type.test intrinsic in the ThinLTO
1213 // post link pipeline after ICP. This is to enable usage of the type
1214 // tests in ICP sequences.
1215 if (isThinLTOPostLink(Phase))
1216 MPM.addPass(Pass: DropTypeTestsPass());
1217
1218 invokePipelineEarlySimplificationEPCallbacks(MPM, Level, Phase);
1219
1220 // Interprocedural constant propagation now that basic cleanup has occurred
1221 // and prior to optimizing globals.
1222 // FIXME: This position in the pipeline hasn't been carefully considered in
1223 // years, it should be re-analyzed.
1224 MPM.addPass(
1225 Pass: IPSCCPPass(IPSCCPOptions(/*AllowFuncSpec=*/!isLTOPreLink(Phase))));
1226
1227 // Attach metadata to indirect call sites indicating the set of functions
1228 // they may target at run-time. This should follow IPSCCP.
1229 MPM.addPass(Pass: CalledValuePropagationPass());
1230
1231 // Optimize globals to try and fold them into constants.
1232 MPM.addPass(Pass: GlobalOptPass());
1233
1234 // Create a small function pass pipeline to cleanup after all the global
1235 // optimizations.
1236 FunctionPassManager GlobalCleanupPM;
1237 // FIXME: Should this instead by a run of SROA?
1238 GlobalCleanupPM.addPass(Pass: PromotePass());
1239 GlobalCleanupPM.addPass(Pass: InstCombinePass());
1240 invokePeepholeEPCallbacks(FPM&: GlobalCleanupPM, Level);
1241 GlobalCleanupPM.addPass(
1242 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
1243 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(GlobalCleanupPM),
1244 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
1245
1246 // We already asserted this happens in non-FullLTOPostLink earlier.
1247 const bool IsPreLink = !isThinLTOPostLink(Phase);
1248 // Enable contextual profiling instrumentation.
1249 const bool IsCtxProfGen =
1250 IsPreLink && PGOCtxProfLoweringPass::isCtxIRPGOInstrEnabled();
1251 const bool IsPGOPreLink = !IsCtxProfGen && PGOOpt && IsPreLink;
1252 const bool IsPGOInstrGen =
1253 IsPGOPreLink && PGOOpt->Action == PGOOptions::IRInstr;
1254 const bool IsPGOInstrUse =
1255 IsPGOPreLink && PGOOpt->Action == PGOOptions::IRUse;
1256 const bool IsMemprofUse = IsPGOPreLink && !PGOOpt->MemoryProfile.empty();
1257 // We don't want to mix pgo ctx gen and pgo gen; we also don't currently
1258 // enable ctx profiling from the frontend.
1259 assert(!(IsPGOInstrGen && PGOCtxProfLoweringPass::isCtxIRPGOInstrEnabled()) &&
1260 "Enabling both instrumented PGO and contextual instrumentation is not "
1261 "supported.");
1262 const bool IsCtxProfUse = !UseCtxProfile.empty() && isThinLTOPreLink(Phase);
1263
1264 assert(
1265 (InstrumentColdFuncOnlyPath.empty() || PGOInstrumentColdFunctionOnly) &&
1266 "--instrument-cold-function-only-path is provided but "
1267 "--pgo-instrument-cold-function-only is not enabled");
1268 const bool IsColdFuncOnlyInstrGen = PGOInstrumentColdFunctionOnly &&
1269 IsPGOPreLink &&
1270 !InstrumentColdFuncOnlyPath.empty();
1271
1272 if (IsPGOInstrGen || IsPGOInstrUse || IsMemprofUse || IsCtxProfGen ||
1273 IsCtxProfUse || IsColdFuncOnlyInstrGen)
1274 addPreInlinerPasses(MPM, Level, LTOPhase: Phase);
1275
1276 // Add all the requested passes for instrumentation PGO, if requested.
1277 if (IsPGOInstrGen || IsPGOInstrUse) {
1278 addPGOInstrPasses(MPM, Level,
1279 /*RunProfileGen=*/IsPGOInstrGen,
1280 /*IsCS=*/false, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate,
1281 ProfileFile: PGOOpt->ProfileFile, ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
1282 } else if (IsCtxProfGen || IsCtxProfUse) {
1283 MPM.addPass(Pass: PGOInstrumentationGen(PGOInstrumentationType::CTXPROF));
1284 // In pre-link, we just want the instrumented IR. We use the contextual
1285 // profile in the post-thinlink phase.
1286 // The instrumentation will be removed in post-thinlink after IPO.
1287 if (IsCtxProfUse) {
1288 MPM.addPass(Pass: AssignGUIDPass());
1289 MPM.addPass(Pass: PGOCtxProfFlatteningPass(/*IsPreThinlink=*/true));
1290 return MPM;
1291 }
1292 // Block further inlining in the instrumented ctxprof case. This avoids
1293 // confusingly collecting profiles for the same GUID corresponding to
1294 // different variants of the function. We could do like PGO and identify
1295 // functions by a (GUID, Hash) tuple, but since the ctxprof "use" waits for
1296 // thinlto to happen before performing any further optimizations, it's
1297 // unnecessary to collect profiles for non-prevailing copies.
1298 MPM.addPass(Pass: NoinlineNonPrevailing());
1299 addPostPGOLoopRotation(MPM, Level);
1300 MPM.addPass(Pass: AssignGUIDPass());
1301 MPM.addPass(Pass: PGOCtxProfLoweringPass());
1302 } else if (IsColdFuncOnlyInstrGen) {
1303 addPGOInstrPasses(MPM, Level, /* RunProfileGen */ true, /* IsCS */ false,
1304 /* AtomicCounterUpdate */ false,
1305 ProfileFile: InstrumentColdFuncOnlyPath,
1306 /* ProfileRemappingFile */ "");
1307 }
1308
1309 if (IsPGOInstrGen || IsPGOInstrUse || IsCtxProfGen)
1310 MPM.addPass(Pass: PGOIndirectCallPromotion(false, false));
1311
1312 if (IsPGOPreLink && PGOOpt->CSAction == PGOOptions::CSIRInstr)
1313 MPM.addPass(Pass: PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile,
1314 EnableSampledInstr));
1315
1316 if (IsMemprofUse)
1317 MPM.addPass(Pass: MemProfUsePass(PGOOpt->MemoryProfile, FS));
1318
1319 if (PGOOpt && (PGOOpt->Action == PGOOptions::IRUse ||
1320 PGOOpt->Action == PGOOptions::SampleUse))
1321 MPM.addPass(Pass: PGOForceFunctionAttrsPass(PGOOpt->ColdOptType));
1322
1323 MPM.addPass(Pass: AlwaysInlinerPass(/*InsertLifetimeIntrinsics=*/true));
1324
1325 if (EnableModuleInliner)
1326 MPM.addPass(Pass: buildModuleInlinerPipeline(Level, Phase));
1327 else
1328 MPM.addPass(Pass: buildInlinerPipeline(Level, Phase));
1329
1330 // Remove any dead arguments exposed by cleanups, constant folding globals,
1331 // and argument promotion.
1332 MPM.addPass(Pass: DeadArgumentEliminationPass());
1333
1334 if (isThinLTOPostLink(Phase))
1335 MPM.addPass(Pass: SimplifyTypeTestsPass());
1336
1337 if (!isThinLTOPreLink(Phase))
1338 MPM.addPass(Pass: CoroCleanupPass());
1339
1340 // Optimize globals now that functions are fully simplified.
1341 MPM.addPass(Pass: GlobalOptPass());
1342 MPM.addPass(Pass: GlobalDCEPass());
1343
1344 return MPM;
1345}
1346
1347/// TODO: Should LTO cause any differences to this set of passes?
1348void PassBuilder::addVectorPasses(OptimizationLevel Level,
1349 FunctionPassManager &FPM,
1350 ThinOrFullLTOPhase LTOPhase) {
1351 FPM.addPass(Pass: LoopVectorizePass(
1352 LoopVectorizeOptions(!PTO.LoopInterleaving, !PTO.LoopVectorization)));
1353
1354 // Drop dereferenceable assumes after vectorization, as they are no longer
1355 // needed and can inhibit further optimization.
1356 if (!isLTOPreLink(Phase: LTOPhase))
1357 FPM.addPass(Pass: DropUnnecessaryAssumesPass(/*DropDereferenceable=*/true));
1358
1359 FPM.addPass(Pass: InferAlignmentPass());
1360 if (isFullLTOPostLink(Phase: LTOPhase)) {
1361 // The vectorizer may have significantly shortened a loop body; unroll
1362 // again. Unroll small loops to hide loop backedge latency and saturate any
1363 // parallel execution resources of an out-of-order processor. We also then
1364 // need to clean up redundancies and loop invariant code.
1365 // FIXME: It would be really good to use a loop-integrated instruction
1366 // combiner for cleanup here so that the unrolling and LICM can be pipelined
1367 // across the loop nests.
1368 // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1369 if (EnableUnrollAndJam && PTO.LoopUnrolling)
1370 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(
1371 Pass: LoopUnrollAndJamPass(static_cast<int>(Level))));
1372 FPM.addPass(Pass: LoopUnrollPass(LoopUnrollOptions(
1373 static_cast<int>(Level), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1374 PTO.ForgetAllSCEVInLoopUnroll)));
1375 FPM.addPass(Pass: WarnMissedTransformationsPass());
1376 // Now that we are done with loop unrolling, be it either by LoopVectorizer,
1377 // or LoopUnroll passes, some variable-offset GEP's into alloca's could have
1378 // become constant-offset, thus enabling SROA and alloca promotion. Do so.
1379 // NOTE: we are very late in the pipeline, and we don't have any LICM
1380 // or SimplifyCFG passes scheduled after us, that would cleanup
1381 // the CFG mess this may created if allowed to modify CFG, so forbid that.
1382
1383 // We also turn on struct to vector canonicalization here, which allows
1384 // converting allocas of homogeneous structs into vector allocas when the
1385 // allocas' users are all memory intrinsics. This allows promotion in some
1386 // cases because structs cannot promote to SSA values, but vectors can. We
1387 // only turn this on after memcpyopt runs because this might hinder
1388 // memcpyopt's optimizations if done before. Look at the documentation for
1389 // `tryCanonicalizeStructToVector` in SROA.cpp to see why.
1390 FPM.addPass(Pass: SROAPass(SROAOptions(SROAOptions::PreserveCFG,
1391 /*AggregateToVector=*/true)));
1392 }
1393
1394 if (!isFullLTOPostLink(Phase: LTOPhase)) {
1395 // Eliminate loads by forwarding stores from the previous iteration to loads
1396 // of the current iteration.
1397 FPM.addPass(Pass: LoopLoadEliminationPass());
1398 }
1399 // Cleanup after the loop optimization passes.
1400 FPM.addPass(Pass: InstCombinePass());
1401
1402 if (Level > OptimizationLevel::O1 && ExtraVectorizerPasses) {
1403 ExtraFunctionPassManager<ShouldRunExtraVectorPasses> ExtraPasses;
1404 // At higher optimization levels, try to clean up any runtime overlap and
1405 // alignment checks inserted by the vectorizer. We want to track correlated
1406 // runtime checks for two inner loops in the same outer loop, fold any
1407 // common computations, hoist loop-invariant aspects out of any outer loop,
1408 // and unswitch the runtime checks if possible. Once hoisted, we may have
1409 // dead (or speculatable) control flows or more combining opportunities.
1410 ExtraPasses.addPass(Pass: EarlyCSEPass());
1411 ExtraPasses.addPass(Pass: CorrelatedValuePropagationPass());
1412 ExtraPasses.addPass(Pass: InstCombinePass());
1413 LoopPassManager LPM;
1414 LPM.addPass(Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
1415 /*AllowSpeculation=*/true));
1416 LPM.addPass(Pass: SimpleLoopUnswitchPass(/* NonTrivial */ Level ==
1417 OptimizationLevel::O3));
1418 ExtraPasses.addPass(
1419 Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM), /*UseMemorySSA=*/true));
1420 ExtraPasses.addPass(
1421 Pass: SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(B: true)));
1422 ExtraPasses.addPass(Pass: InstCombinePass());
1423 FPM.addPass(Pass: std::move(ExtraPasses));
1424 }
1425
1426 // Now that we've formed fast to execute loop structures, we do further
1427 // optimizations. These are run afterward as they might block doing complex
1428 // analyses and transforms such as what are needed for loop vectorization.
1429
1430 // Cleanup after loop vectorization, etc. Simplification passes like CVP and
1431 // GVN, loop transforms, and others have already run, so it's now better to
1432 // convert to more optimized IR using more aggressive simplify CFG options.
1433 // The extra sinking transform can create larger basic blocks, so do this
1434 // before SLP vectorization.
1435 FPM.addPass(Pass: SimplifyCFGPass(SimplifyCFGOptions()
1436 .forwardSwitchCondToPhi(B: true)
1437 .convertSwitchRangeToICmp(B: true)
1438 .convertSwitchToArithmetic(B: true)
1439 .convertSwitchToLookupTable(B: true)
1440 .needCanonicalLoops(B: false)
1441 .hoistCommonInsts(B: true)
1442 .sinkCommonInsts(B: true)));
1443
1444 if (isFullLTOPostLink(Phase: LTOPhase)) {
1445 FPM.addPass(Pass: SCCPPass());
1446 FPM.addPass(Pass: InstCombinePass());
1447 FPM.addPass(Pass: BDCEPass());
1448 }
1449
1450 // Optimize parallel scalar instruction chains into SIMD instructions.
1451 if (PTO.SLPVectorization) {
1452 FPM.addPass(Pass: SLPVectorizerPass());
1453 if (Level >= OptimizationLevel::O2 && ExtraVectorizerPasses) {
1454 FPM.addPass(Pass: EarlyCSEPass());
1455 }
1456 }
1457 // Enhance/cleanup vector code.
1458 FPM.addPass(Pass: VectorCombinePass());
1459
1460 if (!isFullLTOPostLink(Phase: LTOPhase)) {
1461 FPM.addPass(Pass: InstCombinePass());
1462 // Unroll small loops to hide loop backedge latency and saturate any
1463 // parallel execution resources of an out-of-order processor. We also then
1464 // need to clean up redundancies and loop invariant code.
1465 // FIXME: It would be really good to use a loop-integrated instruction
1466 // combiner for cleanup here so that the unrolling and LICM can be pipelined
1467 // across the loop nests.
1468 // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1469 if (EnableUnrollAndJam && PTO.LoopUnrolling) {
1470 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(
1471 Pass: LoopUnrollAndJamPass(static_cast<int>(Level))));
1472 }
1473 FPM.addPass(Pass: LoopUnrollPass(LoopUnrollOptions(
1474 static_cast<int>(Level), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1475 PTO.ForgetAllSCEVInLoopUnroll)));
1476 FPM.addPass(Pass: WarnMissedTransformationsPass());
1477 // Now that we are done with loop unrolling, be it either by LoopVectorizer,
1478 // or LoopUnroll passes, some variable-offset GEP's into alloca's could have
1479 // become constant-offset, thus enabling SROA and alloca promotion. Do so.
1480 // NOTE: we are very late in the pipeline, and we don't have any LICM
1481 // or SimplifyCFG passes scheduled after us, that would cleanup
1482 // the CFG mess this may created if allowed to modify CFG, so forbid that.
1483
1484 // We also turn on struct to vector canonicalization here, which allows
1485 // converting allocas of homogeneous structs into vector allocas when the
1486 // allocas' users are all memory intrinsics. This allows promotion in some
1487 // cases because structs cannot promote to SSA values, but vectors can. We
1488 // only turn this on after memcpyopt runs because this might hinder
1489 // memcpyopt's optimizations if done before. Look at the documentation for
1490 // `tryCanonicalizeStructToVector` in SROA.cpp to see why.
1491 FPM.addPass(Pass: SROAPass(SROAOptions(SROAOptions::PreserveCFG,
1492 /*AggregateToVector=*/true)));
1493 }
1494
1495 FPM.addPass(Pass: InferAlignmentPass());
1496 FPM.addPass(Pass: InstCombinePass());
1497
1498 // This is needed for two reasons:
1499 // 1. It works around problems that instcombine introduces, such as sinking
1500 // expensive FP divides into loops containing multiplications using the
1501 // divide result.
1502 // 2. It helps to clean up some loop-invariant code created by the loop
1503 // unroll pass when IsFullLTO=false.
1504 FPM.addPass(Pass: createFunctionToLoopPassAdaptor(
1505 Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
1506 /*AllowSpeculation=*/true),
1507 /*UseMemorySSA=*/true));
1508
1509 // Now that we've vectorized and unrolled loops, we may have more refined
1510 // alignment information, try to re-derive it here.
1511 FPM.addPass(Pass: AlignmentFromAssumptionsPass());
1512}
1513
1514ModulePassManager
1515PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
1516 ThinOrFullLTOPhase LTOPhase) {
1517 ModulePassManager MPM;
1518
1519 // Run partial inlining pass to partially inline functions that have
1520 // large bodies.
1521 if (RunPartialInlining)
1522 MPM.addPass(Pass: PartialInlinerPass());
1523
1524 // Remove avail extern fns and globals definitions since we aren't compiling
1525 // an object file for later LTO. For LTO we want to preserve these so they
1526 // are eligible for inlining at link-time. Note if they are unreferenced they
1527 // will be removed by GlobalDCE later, so this only impacts referenced
1528 // available externally globals. Eventually they will be suppressed during
1529 // codegen, but eliminating here enables more opportunity for GlobalDCE as it
1530 // may make globals referenced by available external functions dead and saves
1531 // running remaining passes on the eliminated functions. These should be
1532 // preserved during prelinking for link-time inlining decisions.
1533 if (!isLTOPreLink(Phase: LTOPhase))
1534 MPM.addPass(Pass: EliminateAvailableExternallyPass());
1535
1536 // Do RPO function attribute inference across the module to forward-propagate
1537 // attributes where applicable.
1538 // FIXME: Is this really an optimization rather than a canonicalization?
1539 MPM.addPass(Pass: ReversePostOrderFunctionAttrsPass());
1540
1541 // Do a post inline PGO instrumentation and use pass. This is a context
1542 // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as
1543 // cross-module inline has not been done yet. The context sensitive
1544 // instrumentation is after all the inlines are done.
1545 if (!isLTOPreLink(Phase: LTOPhase) && PGOOpt) {
1546 if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1547 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/true,
1548 /*IsCS=*/true, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate,
1549 ProfileFile: PGOOpt->CSProfileGenFile, ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
1550 else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1551 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/false,
1552 /*IsCS=*/true, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate,
1553 ProfileFile: PGOOpt->ProfileFile, ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
1554 }
1555
1556 // Re-compute GlobalsAA here prior to function passes. This is particularly
1557 // useful as the above will have inlined, DCE'ed, and function-attr
1558 // propagated everything. We should at this point have a reasonably minimal
1559 // and richly annotated call graph. By computing aliasing and mod/ref
1560 // information for all local globals here, the late loop passes and notably
1561 // the vectorizer will be able to use them to help recognize vectorizable
1562 // memory operations.
1563 if (EnableGlobalAnalyses)
1564 MPM.addPass(Pass: RecomputeGlobalsAAPass());
1565
1566 invokeOptimizerEarlyEPCallbacks(MPM, Level, Phase: LTOPhase);
1567
1568 FunctionPassManager OptimizePM;
1569
1570 // Only drop unnecessary assumes post-inline and post-link, as otherwise
1571 // additional uses of the affected value may be introduced through inlining
1572 // and CSE.
1573 if (!isLTOPreLink(Phase: LTOPhase))
1574 OptimizePM.addPass(Pass: DropUnnecessaryAssumesPass());
1575
1576 // Scheduling LoopVersioningLICM when inlining is over, because after that
1577 // we may see more accurate aliasing. Reason to run this late is that too
1578 // early versioning may prevent further inlining due to increase of code
1579 // size. Other optimizations which runs later might get benefit of no-alias
1580 // assumption in clone loop.
1581 if (UseLoopVersioningLICM) {
1582 OptimizePM.addPass(
1583 Pass: createFunctionToLoopPassAdaptor(Pass: LoopVersioningLICMPass()));
1584 // LoopVersioningLICM pass might increase new LICM opportunities.
1585 OptimizePM.addPass(Pass: createFunctionToLoopPassAdaptor(
1586 Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
1587 /*AllowSpeculation=*/true),
1588 /*USeMemorySSA=*/UseMemorySSA: true));
1589 }
1590
1591 OptimizePM.addPass(Pass: Float2IntPass());
1592 OptimizePM.addPass(Pass: LowerConstantIntrinsicsPass());
1593
1594 if (EnableMatrix) {
1595 OptimizePM.addPass(Pass: LowerMatrixIntrinsicsPass());
1596 OptimizePM.addPass(Pass: EarlyCSEPass());
1597 }
1598
1599 // CHR pass should only be applied with the profile information.
1600 // The check is to check the profile summary information in CHR.
1601 if (EnableCHR && Level == OptimizationLevel::O3)
1602 OptimizePM.addPass(Pass: ControlHeightReductionPass());
1603
1604 // FIXME: We need to run some loop optimizations to re-rotate loops after
1605 // simplifycfg and others undo their rotation.
1606
1607 // Optimize the loop execution. These passes operate on entire loop nests
1608 // rather than on each loop in an inside-out manner, and so they are actually
1609 // function passes.
1610
1611 invokeVectorizerStartEPCallbacks(FPM&: OptimizePM, Level);
1612
1613 LoopPassManager LPM;
1614 // First rotate loops that may have been un-rotated by prior passes.
1615 // Disable header duplication at -Oz.
1616 LPM.addPass(Pass: LoopRotatePass(/*EnableLoopHeaderDuplication=*/true,
1617 isLTOPreLink(Phase: LTOPhase),
1618 /*CheckExitCount=*/true));
1619 // Some loops may have become dead by now. Try to delete them.
1620 // FIXME: see discussion in https://reviews.llvm.org/D112851,
1621 // this may need to be revisited once we run GVN before loop deletion
1622 // in the simplification pipeline.
1623 LPM.addPass(Pass: LoopDeletionPass());
1624
1625 if (PTO.LoopInterchange)
1626 LPM.addPass(Pass: LoopInterchangePass());
1627
1628 OptimizePM.addPass(
1629 Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM), /*UseMemorySSA=*/false));
1630
1631 // FIXME: This may not be the right place in the pipeline.
1632 // We need to have the data to support the right place.
1633 if (PTO.LoopFusion)
1634 OptimizePM.addPass(Pass: LoopFusePass());
1635
1636 // Distribute loops to allow partial vectorization. I.e. isolate dependences
1637 // into separate loop that would otherwise inhibit vectorization. This is
1638 // currently only performed for loops marked with the metadata
1639 // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
1640 OptimizePM.addPass(Pass: LoopDistributePass());
1641
1642 // Populates the VFABI attribute with the scalar-to-vector mappings
1643 // from the TargetLibraryInfo.
1644 OptimizePM.addPass(Pass: InjectTLIMappings());
1645
1646 addVectorPasses(Level, FPM&: OptimizePM, LTOPhase);
1647
1648 invokeVectorizerEndEPCallbacks(FPM&: OptimizePM, Level);
1649
1650 // LoopSink pass sinks instructions hoisted by LICM, which serves as a
1651 // canonicalization pass that enables other optimizations. As a result,
1652 // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
1653 // result too early.
1654 OptimizePM.addPass(Pass: LoopSinkPass());
1655
1656 // And finally clean up LCSSA form before generating code.
1657 OptimizePM.addPass(Pass: InstSimplifyPass());
1658
1659 // This hoists/decomposes div/rem ops. It should run after other sink/hoist
1660 // passes to avoid re-sinking, but before SimplifyCFG because it can allow
1661 // flattening of blocks.
1662 OptimizePM.addPass(Pass: DivRemPairsPass());
1663
1664 // Merge adjacent icmps into memcmp, then expand memcmp to loads/compares.
1665 // TODO: move this furter up so that it can be optimized by GVN, etc.
1666 if (EnableMergeICmps)
1667 OptimizePM.addPass(Pass: MergeICmpsPass());
1668 OptimizePM.addPass(Pass: ExpandMemCmpPass());
1669
1670 // Try to annotate calls that were created during optimization.
1671 OptimizePM.addPass(
1672 Pass: TailCallElimPass(/*UpdateFunctionEntryCount=*/isInstrumentedPGOUse()));
1673
1674 // LoopSink (and other loop passes since the last simplifyCFG) might have
1675 // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
1676 OptimizePM.addPass(
1677 Pass: SimplifyCFGPass(SimplifyCFGOptions()
1678 .convertSwitchRangeToICmp(B: true)
1679 .convertSwitchToArithmetic(B: true)
1680 .speculateUnpredictables(B: true)
1681 .hoistLoadsStoresWithCondFaulting(B: true)));
1682
1683 // Add the core optimizing pipeline.
1684 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(OptimizePM),
1685 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
1686
1687 // AllocToken transforms heap allocation calls; this needs to run late after
1688 // other allocation call transformations (such as those in InstCombine).
1689 if (!isLTOPreLink(Phase: LTOPhase))
1690 MPM.addPass(Pass: AllocTokenPass());
1691
1692 invokeOptimizerLastEPCallbacks(MPM, Level, Phase: LTOPhase);
1693
1694 // Run the Instrumentor pass late.
1695 if (EnableInstrumentor)
1696 MPM.addPass(Pass: InstrumentorPass(FS));
1697
1698 // Split out cold code. Splitting is done late to avoid hiding context from
1699 // other optimizations and inadvertently regressing performance. The tradeoff
1700 // is that this has a higher code size cost than splitting early.
1701 if (EnableHotColdSplit && !isLTOPreLink(Phase: LTOPhase))
1702 MPM.addPass(Pass: HotColdSplittingPass());
1703
1704 // Search the code for similar regions of code. If enough similar regions can
1705 // be found where extracting the regions into their own function will decrease
1706 // the size of the program, we extract the regions, a deduplicate the
1707 // structurally similar regions.
1708 if (EnableIROutliner)
1709 MPM.addPass(Pass: IROutlinerPass());
1710
1711 // Now we need to do some global optimization transforms.
1712 // FIXME: It would seem like these should come first in the optimization
1713 // pipeline and maybe be the bottom of the canonicalization pipeline? Weird
1714 // ordering here.
1715 MPM.addPass(Pass: GlobalDCEPass());
1716 MPM.addPass(Pass: ConstantMergePass());
1717
1718 // Merge functions if requested. It has a better chance to merge functions
1719 // after ConstantMerge folded jump tables.
1720 if (PTO.MergeFunctions)
1721 MPM.addPass(Pass: MergeFunctionsPass());
1722
1723 if (PTO.CallGraphProfile && !isLTOPreLink(Phase: LTOPhase))
1724 MPM.addPass(Pass: CGProfilePass(isLTOPostLink(Phase: LTOPhase)));
1725
1726 // RelLookupTableConverterPass runs later in LTO post-link pipeline.
1727 if (!isLTOPreLink(Phase: LTOPhase))
1728 MPM.addPass(Pass: RelLookupTableConverterPass());
1729
1730 // Add devirtualization pass only when LTO is not enabled, as otherwise
1731 // the pass is already enabled in the LTO pipeline.
1732 if (PTO.DevirtualizeSpeculatively && LTOPhase == ThinOrFullLTOPhase::None) {
1733 // TODO: explore a better pipeline configuration that can improve
1734 // compilation time overhead.
1735 // FIXME: move this earlier (lots of pass ordering tests will need fixing)
1736 MPM.addPass(Pass: AssignGUIDPass());
1737 MPM.addPass(Pass: WholeProgramDevirtPass(
1738 /*ExportSummary*/ nullptr,
1739 /*ImportSummary*/ nullptr,
1740 /*DevirtSpeculatively*/ PTO.DevirtualizeSpeculatively));
1741 MPM.addPass(Pass: DropTypeTestsPass());
1742 // Given that the devirtualization creates more opportunities for inlining,
1743 // we run the Inliner again here to maximize the optimization gain we
1744 // get from devirtualization.
1745 // Also, we can't run devirtualization before inlining because the
1746 // devirtualization depends on the passes optimizing/eliminating vtable GVs
1747 // and those passes are only effective after inlining.
1748 if (EnableModuleInliner) {
1749 MPM.addPass(Pass: ModuleInlinerPass(::getInlineParamsFromOptLevel(Level),
1750 UseInlineAdvisor,
1751 ThinOrFullLTOPhase::None));
1752 } else {
1753 MPM.addPass(Pass: ModuleInlinerWrapperPass(
1754 ::getInlineParamsFromOptLevel(Level),
1755 /* MandatoryFirst */ true,
1756 InlineContext{.LTOPhase: ThinOrFullLTOPhase::None, .Pass: InlinePass::CGSCCInliner}));
1757 }
1758 }
1759
1760 // Attach !implicit.ref metadata from all functions to copyright strings.
1761 MPM.addPass(Pass: LowerCommentStringPass());
1762
1763 return MPM;
1764}
1765
1766ModulePassManager
1767PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level,
1768 ThinOrFullLTOPhase Phase) {
1769 if (Level == OptimizationLevel::O0)
1770 return buildO0DefaultPipeline(Level, Phase);
1771
1772 ModulePassManager MPM;
1773 instructionCountersPass(MPM, /* IsPreOptimization */ true);
1774 // Currently this pipeline is only invoked in an LTO pre link pass or when we
1775 // are not running LTO. If that changes the below checks may need updating.
1776 assert(isLTOPreLink(Phase) || Phase == ThinOrFullLTOPhase::None);
1777
1778 // If we are invoking this in non-LTO mode, remove any MemProf related
1779 // attributes and metadata, as we don't know whether we are linking with
1780 // a library containing the necessary interfaces.
1781 if (Phase == ThinOrFullLTOPhase::None)
1782 MPM.addPass(Pass: MemProfRemoveInfo());
1783
1784 // Convert @llvm.global.annotations to !annotation metadata.
1785 MPM.addPass(Pass: Annotation2MetadataPass());
1786
1787 // Force any function attributes we want the rest of the pipeline to observe.
1788 MPM.addPass(Pass: ForceFunctionAttrsPass());
1789
1790 if (TriggerCrash)
1791 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: TriggerCrashFunctionPass()));
1792
1793 if (PGOOpt && PGOOpt->DebugInfoForProfiling)
1794 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: AddDiscriminatorsPass()));
1795
1796 // Apply module pipeline start EP callback.
1797 invokePipelineStartEPCallbacks(MPM, Level);
1798
1799 // Add the core simplification pipeline.
1800 MPM.addPass(Pass: buildModuleSimplificationPipeline(Level, Phase));
1801
1802 // Now add the optimization pipeline.
1803 MPM.addPass(Pass: buildModuleOptimizationPipeline(Level, LTOPhase: Phase));
1804
1805 if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&
1806 PGOOpt->Action == PGOOptions::SampleUse)
1807 MPM.addPass(Pass: PseudoProbeUpdatePass());
1808
1809 // Emit annotation remarks.
1810 addAnnotationRemarksPass(MPM);
1811
1812 if (isLTOPreLink(Phase))
1813 addRequiredLTOPreLinkPasses(MPM);
1814
1815 instructionCountersPass(MPM, /* IsPreOptimization */ false);
1816 return MPM;
1817}
1818
1819ModulePassManager
1820PassBuilder::buildFatLTODefaultPipeline(OptimizationLevel Level, bool ThinLTO,
1821 bool EmitSummary, bool Verify) {
1822 ModulePassManager MPM;
1823
1824 instructionCountersPass(MPM, /* IsPreOptimization */ true);
1825
1826 if (ThinLTO)
1827 MPM.addPass(Pass: buildThinLTOPreLinkDefaultPipeline(Level));
1828 else
1829 MPM.addPass(Pass: buildLTOPreLinkDefaultPipeline(Level));
1830 // AssignGUIDPass attaches !guid metadata (MD_unique_id) to global objects,
1831 // triggering the bitcode writer to emit a METADATA_KIND_BLOCK. Standard LTO
1832 // bitcode emission runs VerifierPass by default, which registers metadata
1833 // kind IDs in LLVMContext. Running VerifierPass here before EmbedBitcodePass
1834 // to get the same behavior.
1835 if (Verify)
1836 MPM.addPass(Pass: VerifierPass());
1837 MPM.addPass(Pass: EmbedBitcodePass(ThinLTO, EmitSummary));
1838
1839 // Perform any cleanups to the IR that aren't suitable for per TU compilation,
1840 // like removing CFI/WPD related instructions. Note, we reuse
1841 // DropTypeTestsPass to clean up type tests rather than duplicate that logic
1842 // in FatLtoCleanup.
1843 MPM.addPass(Pass: FatLtoCleanup());
1844
1845 // If we're doing FatLTO w/ CFI enabled, we don't want the type tests in the
1846 // object code, only in the bitcode section, so drop it before we run
1847 // module optimization and generate machine code. If llvm.type.test() isn't in
1848 // the IR, this won't do anything.
1849 MPM.addPass(Pass: DropTypeTestsPass(lowertypetests::DropTestKind::All));
1850
1851 // Use the ThinLTO post-link pipeline with sample profiling
1852 if (ThinLTO && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse)
1853 MPM.addPass(Pass: buildThinLTODefaultPipeline(Level, /*ImportSummary=*/nullptr));
1854 else {
1855 // ModuleSimplification does not run the coroutine passes for
1856 // ThinLTOPreLink, so we need the coroutine passes to run for ThinLTO
1857 // builds, otherwise they will miscompile.
1858 if (ThinLTO) {
1859 // TODO: replace w/ buildCoroWrapper() when it takes phase and level into
1860 // consideration.
1861 CGSCCPassManager CGPM;
1862 CGPM.addPass(Pass: CoroSplitPass(Level != OptimizationLevel::O0));
1863 CGPM.addPass(Pass: CoroAnnotationElidePass());
1864 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: std::move(CGPM)));
1865 MPM.addPass(Pass: CoroCleanupPass());
1866 }
1867
1868 // otherwise, just use module optimization
1869 MPM.addPass(
1870 Pass: buildModuleOptimizationPipeline(Level, LTOPhase: ThinOrFullLTOPhase::None));
1871 // Emit annotation remarks.
1872 addAnnotationRemarksPass(MPM);
1873 }
1874
1875 instructionCountersPass(MPM, /* IsPreOptimization */ false);
1876
1877 return MPM;
1878}
1879
1880ModulePassManager
1881PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level) {
1882 if (Level == OptimizationLevel::O0)
1883 return buildO0DefaultPipeline(Level, Phase: ThinOrFullLTOPhase::ThinLTOPreLink);
1884
1885 ModulePassManager MPM;
1886
1887 instructionCountersPass(MPM, /* IsPreOptimization */ true);
1888
1889 // Convert @llvm.global.annotations to !annotation metadata.
1890 MPM.addPass(Pass: Annotation2MetadataPass());
1891
1892 // Force any function attributes we want the rest of the pipeline to observe.
1893 MPM.addPass(Pass: ForceFunctionAttrsPass());
1894
1895 if (PGOOpt && PGOOpt->DebugInfoForProfiling)
1896 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: AddDiscriminatorsPass()));
1897
1898 // Apply module pipeline start EP callback.
1899 invokePipelineStartEPCallbacks(MPM, Level);
1900
1901 // If we are planning to perform ThinLTO later, we don't bloat the code with
1902 // unrolling/vectorization/... now. Just simplify the module as much as we
1903 // can.
1904 MPM.addPass(Pass: buildModuleSimplificationPipeline(
1905 Level, Phase: ThinOrFullLTOPhase::ThinLTOPreLink));
1906 // In pre-link, for ctx prof use, we stop here with an instrumented IR. We let
1907 // thinlto use the contextual info to perform imports; then use the contextual
1908 // profile in the post-thinlink phase.
1909 if (!UseCtxProfile.empty()) {
1910 addRequiredLTOPreLinkPasses(MPM);
1911 return MPM;
1912 }
1913
1914 // Run partial inlining pass to partially inline functions that have
1915 // large bodies.
1916 // FIXME: It isn't clear whether this is really the right place to run this
1917 // in ThinLTO. Because there is another canonicalization and simplification
1918 // phase that will run after the thin link, running this here ends up with
1919 // less information than will be available later and it may grow functions in
1920 // ways that aren't beneficial.
1921 if (RunPartialInlining)
1922 MPM.addPass(Pass: PartialInlinerPass());
1923
1924 if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&
1925 PGOOpt->Action == PGOOptions::SampleUse)
1926 MPM.addPass(Pass: PseudoProbeUpdatePass());
1927
1928 // Handle Optimizer{Early,Last}EPCallbacks added by clang on PreLink. Actual
1929 // optimization is going to be done in PostLink stage, but clang can't add
1930 // callbacks there in case of in-process ThinLTO called by linker.
1931 invokeOptimizerEarlyEPCallbacks(MPM, Level,
1932 /*Phase=*/ThinOrFullLTOPhase::ThinLTOPreLink);
1933 invokeOptimizerLastEPCallbacks(MPM, Level,
1934 /*Phase=*/ThinOrFullLTOPhase::ThinLTOPreLink);
1935
1936 // Emit annotation remarks.
1937 addAnnotationRemarksPass(MPM);
1938
1939 // Attach !implicit.ref metadata from all functions to copyright strings.
1940 MPM.addPass(Pass: LowerCommentStringPass());
1941
1942 addRequiredLTOPreLinkPasses(MPM);
1943
1944 instructionCountersPass(MPM, /* IsPreOptimization */ false);
1945
1946 return MPM;
1947}
1948
1949ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
1950 OptimizationLevel Level, const ModuleSummaryIndex *ImportSummary) {
1951 ModulePassManager MPM;
1952
1953 instructionCountersPass(MPM, /* IsPreOptimization */ true);
1954
1955 // If we are invoking this without a summary index noting that we are linking
1956 // with a library containing the necessary APIs, remove any MemProf related
1957 // attributes and metadata.
1958 if (!ImportSummary || !ImportSummary->withSupportsHotColdNew())
1959 MPM.addPass(Pass: MemProfRemoveInfo());
1960
1961 if (ImportSummary) {
1962 // For ThinLTO we must apply the context disambiguation decisions early, to
1963 // ensure we can correctly match the callsites to summary data.
1964 if (EnableMemProfContextDisambiguation)
1965 MPM.addPass(Pass: MemProfContextDisambiguation(
1966 ImportSummary, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
1967
1968 // These passes import type identifier resolutions for whole-program
1969 // devirtualization and CFI. They must run early because other passes may
1970 // disturb the specific instruction patterns that these passes look for,
1971 // creating dependencies on resolutions that may not appear in the summary.
1972 //
1973 // For example, GVN may transform the pattern assume(type.test) appearing in
1974 // two basic blocks into assume(phi(type.test, type.test)), which would
1975 // transform a dependency on a WPD resolution into a dependency on a type
1976 // identifier resolution for CFI.
1977 //
1978 // Also, WPD has access to more precise information than ICP and can
1979 // devirtualize more effectively, so it should operate on the IR first.
1980 //
1981 // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1982 // metadata and intrinsics.
1983 MPM.addPass(Pass: WholeProgramDevirtPass(nullptr, ImportSummary));
1984 MPM.addPass(Pass: LowerTypeTestsPass(nullptr, ImportSummary));
1985 }
1986
1987 if (Level == OptimizationLevel::O0) {
1988 // Run a second time to clean up any type tests left behind by WPD for use
1989 // in ICP.
1990 MPM.addPass(Pass: DropTypeTestsPass());
1991 MPM.addPass(Pass: buildCoroWrapper(Phase: ThinOrFullLTOPhase::ThinLTOPostLink));
1992
1993 // AllocToken transforms heap allocation calls; this needs to run late after
1994 // other allocation call transformations (such as those in InstCombine).
1995 MPM.addPass(Pass: AllocTokenPass());
1996
1997 // Drop available_externally and unreferenced globals. This is necessary
1998 // with ThinLTO in order to avoid leaving undefined references to dead
1999 // globals in the object file.
2000 MPM.addPass(Pass: EliminateAvailableExternallyPass());
2001 MPM.addPass(Pass: GlobalDCEPass());
2002 return MPM;
2003 }
2004 if (!UseCtxProfile.empty()) {
2005 MPM.addPass(
2006 Pass: buildModuleInlinerPipeline(Level, Phase: ThinOrFullLTOPhase::ThinLTOPostLink));
2007 } else {
2008 // Add the core simplification pipeline.
2009 MPM.addPass(Pass: buildModuleSimplificationPipeline(
2010 Level, Phase: ThinOrFullLTOPhase::ThinLTOPostLink));
2011 }
2012 // Now add the optimization pipeline.
2013 MPM.addPass(Pass: buildModuleOptimizationPipeline(
2014 Level, LTOPhase: ThinOrFullLTOPhase::ThinLTOPostLink));
2015
2016 // Emit annotation remarks.
2017 addAnnotationRemarksPass(MPM);
2018
2019 instructionCountersPass(MPM, /* IsPreOptimization */ false);
2020
2021 return MPM;
2022}
2023
2024ModulePassManager
2025PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level) {
2026 // FIXME: We should use a customized pre-link pipeline!
2027 return buildPerModuleDefaultPipeline(Level,
2028 Phase: ThinOrFullLTOPhase::FullLTOPreLink);
2029}
2030
2031ModulePassManager
2032PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
2033 ModuleSummaryIndex *ExportSummary) {
2034 ModulePassManager MPM;
2035
2036 instructionCountersPass(MPM, /* IsPreOptimization */ true);
2037
2038 invokeFullLinkTimeOptimizationEarlyEPCallbacks(MPM, Level);
2039
2040 // If we are invoking this without a summary index noting that we are linking
2041 // with a library containing the necessary APIs, remove any MemProf related
2042 // attributes and metadata.
2043 if (!ExportSummary || !ExportSummary->withSupportsHotColdNew())
2044 MPM.addPass(Pass: MemProfRemoveInfo());
2045
2046 // Create a function that performs CFI checks for cross-DSO calls with targets
2047 // in the current module.
2048 MPM.addPass(Pass: CrossDSOCFIPass());
2049
2050 if (Level == OptimizationLevel::O0) {
2051 // The WPD and LowerTypeTest passes need to run at -O0 to lower type
2052 // metadata and intrinsics.
2053 MPM.addPass(Pass: WholeProgramDevirtPass(ExportSummary, nullptr));
2054 MPM.addPass(Pass: LowerTypeTestsPass(ExportSummary, nullptr));
2055 // Run a second time to clean up any type tests left behind by WPD for use
2056 // in ICP.
2057 MPM.addPass(Pass: DropTypeTestsPass());
2058
2059 MPM.addPass(Pass: buildCoroWrapper(Phase: ThinOrFullLTOPhase::FullLTOPostLink));
2060
2061 // AllocToken transforms heap allocation calls; this needs to run late after
2062 // other allocation call transformations (such as those in InstCombine).
2063 MPM.addPass(Pass: AllocTokenPass());
2064
2065 invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);
2066
2067 // Emit annotation remarks.
2068 addAnnotationRemarksPass(MPM);
2069
2070 return MPM;
2071 }
2072
2073 if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
2074 // Load sample profile before running the LTO optimization pipeline.
2075 MPM.addPass(Pass: SampleProfileLoaderPass(PGOOpt->ProfileFile,
2076 PGOOpt->ProfileRemappingFile,
2077 ThinOrFullLTOPhase::FullLTOPostLink));
2078 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
2079 // RequireAnalysisPass for PSI before subsequent non-module passes.
2080 MPM.addPass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
2081 }
2082
2083 // Try to run OpenMP optimizations, quick no-op if no OpenMP metadata present.
2084 MPM.addPass(Pass: OpenMPOptPass(ThinOrFullLTOPhase::FullLTOPostLink));
2085
2086 // Remove unused virtual tables to improve the quality of code generated by
2087 // whole-program devirtualization and bitset lowering.
2088 MPM.addPass(Pass: GlobalDCEPass(/*InLTOPostLink=*/true));
2089
2090 // Do basic inference of function attributes from known properties of system
2091 // libraries and other oracles.
2092 MPM.addPass(Pass: InferFunctionAttrsPass());
2093
2094 if (Level >= OptimizationLevel::O2) {
2095 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
2096 Pass: CallSiteSplittingPass(), EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
2097
2098 // Indirect call promotion. This should promote all the targets that are
2099 // left by the earlier promotion pass that promotes intra-module targets.
2100 // This two-step promotion is to save the compile time. For LTO, it should
2101 // produce the same result as if we only do promotion here.
2102 MPM.addPass(Pass: PGOIndirectCallPromotion(
2103 true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
2104
2105 // Promoting by-reference arguments to by-value exposes more constants to
2106 // IPSCCP.
2107 CGSCCPassManager CGPM;
2108 CGPM.addPass(Pass: PostOrderFunctionAttrsPass());
2109 CGPM.addPass(Pass: ArgumentPromotionPass());
2110 CGPM.addPass(
2111 Pass: createCGSCCToFunctionPassAdaptor(Pass: SROAPass(SROAOptions::ModifyCFG)));
2112 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: std::move(CGPM)));
2113
2114 // Propagate constants at call sites into the functions they call. This
2115 // opens opportunities for globalopt (and inlining) by substituting function
2116 // pointers passed as arguments to direct uses of functions.
2117 MPM.addPass(Pass: IPSCCPPass(IPSCCPOptions(/*AllowFuncSpec=*/true)));
2118
2119 // Attach metadata to indirect call sites indicating the set of functions
2120 // they may target at run-time. This should follow IPSCCP.
2121 MPM.addPass(Pass: CalledValuePropagationPass());
2122 }
2123
2124 // Do RPO function attribute inference across the module to forward-propagate
2125 // attributes where applicable.
2126 // FIXME: Is this really an optimization rather than a canonicalization?
2127 MPM.addPass(Pass: ReversePostOrderFunctionAttrsPass());
2128
2129 // Use in-range annotations on GEP indices to split globals where beneficial.
2130 MPM.addPass(Pass: GlobalSplitPass());
2131
2132 // Run whole program optimization of virtual call when the list of callees
2133 // is fixed.
2134 MPM.addPass(Pass: WholeProgramDevirtPass(ExportSummary, nullptr));
2135
2136 MPM.addPass(Pass: NoRecurseLTOInferencePass());
2137 // Stop here at -O1.
2138 if (Level == OptimizationLevel::O1) {
2139 // The LowerTypeTestsPass needs to run to lower type metadata and the
2140 // type.test intrinsics. The pass does nothing if CFI is disabled.
2141 MPM.addPass(Pass: LowerTypeTestsPass(ExportSummary, nullptr));
2142 // Run a second time to clean up any type tests left behind by WPD for use
2143 // in ICP (which is performed earlier than this in the regular LTO
2144 // pipeline).
2145 MPM.addPass(Pass: DropTypeTestsPass());
2146
2147 MPM.addPass(Pass: buildCoroWrapper(Phase: ThinOrFullLTOPhase::FullLTOPostLink));
2148
2149 // AllocToken transforms heap allocation calls; this needs to run late after
2150 // other allocation call transformations (such as those in InstCombine).
2151 MPM.addPass(Pass: AllocTokenPass());
2152
2153 invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);
2154
2155 // Emit annotation remarks.
2156 addAnnotationRemarksPass(MPM);
2157
2158 instructionCountersPass(MPM, /* IsPreOptimization */ false);
2159
2160 return MPM;
2161 }
2162
2163 // TODO: Skip to match buildCoroWrapper.
2164 MPM.addPass(Pass: CoroEarlyPass());
2165
2166 // Optimize globals to try and fold them into constants.
2167 MPM.addPass(Pass: GlobalOptPass());
2168
2169 // Promote any localized globals to SSA registers.
2170 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: PromotePass()));
2171
2172 // Linking modules together can lead to duplicate global constant, only
2173 // keep one copy of each constant.
2174 MPM.addPass(Pass: ConstantMergePass());
2175
2176 // Remove unused arguments from functions.
2177 MPM.addPass(Pass: DeadArgumentEliminationPass());
2178
2179 // Reduce the code after globalopt and ipsccp. Both can open up significant
2180 // simplification opportunities, and both can propagate functions through
2181 // function pointers. When this happens, we often have to resolve varargs
2182 // calls, etc, so let instcombine do this.
2183 FunctionPassManager PeepholeFPM;
2184 PeepholeFPM.addPass(Pass: InstCombinePass());
2185 if (Level >= OptimizationLevel::O2)
2186 PeepholeFPM.addPass(Pass: AggressiveInstCombinePass());
2187 invokePeepholeEPCallbacks(FPM&: PeepholeFPM, Level);
2188
2189 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(PeepholeFPM),
2190 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
2191
2192 // Lower variadic functions for supported targets prior to inlining.
2193 MPM.addPass(Pass: ExpandVariadicsPass(ExpandVariadicsMode::Optimize));
2194
2195 // Note: historically, the PruneEH pass was run first to deduce nounwind and
2196 // generally clean up exception handling overhead. It isn't clear this is
2197 // valuable as the inliner doesn't currently care whether it is inlining an
2198 // invoke or a call.
2199 // Run the inliner now.
2200 if (EnableModuleInliner) {
2201 MPM.addPass(Pass: ModuleInlinerPass(::getInlineParamsFromOptLevel(Level),
2202 UseInlineAdvisor,
2203 ThinOrFullLTOPhase::FullLTOPostLink));
2204 } else {
2205 MPM.addPass(Pass: ModuleInlinerWrapperPass(
2206 ::getInlineParamsFromOptLevel(Level),
2207 /* MandatoryFirst */ true,
2208 InlineContext{.LTOPhase: ThinOrFullLTOPhase::FullLTOPostLink,
2209 .Pass: InlinePass::CGSCCInliner}));
2210 }
2211
2212 // Perform context disambiguation after inlining, since that would reduce the
2213 // amount of additional cloning required to distinguish the allocation
2214 // contexts.
2215 if (EnableMemProfContextDisambiguation)
2216 MPM.addPass(Pass: MemProfContextDisambiguation(
2217 /*Summary=*/nullptr,
2218 PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
2219
2220 // Optimize globals again after we ran the inliner.
2221 MPM.addPass(Pass: GlobalOptPass());
2222
2223 // Run the OpenMPOpt pass again after global optimizations.
2224 MPM.addPass(Pass: OpenMPOptPass(ThinOrFullLTOPhase::FullLTOPostLink));
2225
2226 // Garbage collect dead functions.
2227 MPM.addPass(Pass: GlobalDCEPass(/*InLTOPostLink=*/true));
2228
2229 // If we didn't decide to inline a function, check to see if we can
2230 // transform it to pass arguments by value instead of by reference.
2231 CGSCCPassManager CGPM;
2232 CGPM.addPass(Pass: ArgumentPromotionPass());
2233 CGPM.addPass(Pass: CoroSplitPass(Level != OptimizationLevel::O0));
2234 CGPM.addPass(Pass: CoroAnnotationElidePass());
2235 invokeCGSCCOptimizerLateEPCallbacks(CGPM, Level);
2236 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: std::move(CGPM)));
2237
2238 FunctionPassManager FPM;
2239 // The IPO Passes may leave cruft around. Clean up after them.
2240 FPM.addPass(Pass: InstCombinePass());
2241 invokePeepholeEPCallbacks(FPM, Level);
2242
2243 if (EnableConstraintElimination)
2244 FPM.addPass(Pass: ConstraintEliminationPass());
2245
2246 FPM.addPass(Pass: JumpThreadingPass());
2247
2248 // Do a post inline PGO instrumentation and use pass. This is a context
2249 // sensitive PGO pass.
2250 if (PGOOpt) {
2251 if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
2252 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/true,
2253 /*IsCS=*/true, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate,
2254 ProfileFile: PGOOpt->CSProfileGenFile, ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
2255 else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
2256 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/false,
2257 /*IsCS=*/true, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate,
2258 ProfileFile: PGOOpt->ProfileFile, ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
2259 }
2260
2261 // Break up allocas
2262 FPM.addPass(Pass: SROAPass(SROAOptions::ModifyCFG));
2263
2264 // LTO provides additional opportunities for tailcall elimination due to
2265 // link-time inlining, and visibility of nocapture attribute.
2266 FPM.addPass(
2267 Pass: TailCallElimPass(/*UpdateFunctionEntryCount=*/isInstrumentedPGOUse()));
2268
2269 // Run a few AA driver optimizations here and now to cleanup the code.
2270 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM),
2271 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
2272
2273 MPM.addPass(
2274 Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: PostOrderFunctionAttrsPass()));
2275
2276 // Require the GlobalsAA analysis for the module so we can query it within
2277 // MainFPM.
2278 if (EnableGlobalAnalyses) {
2279 MPM.addPass(Pass: RequireAnalysisPass<GlobalsAA, Module>());
2280 // Invalidate AAManager so it can be recreated and pick up the newly
2281 // available GlobalsAA.
2282 MPM.addPass(
2283 Pass: createModuleToFunctionPassAdaptor(Pass: InvalidateAnalysisPass<AAManager>()));
2284 }
2285
2286 FunctionPassManager MainFPM;
2287 MainFPM.addPass(Pass: createFunctionToLoopPassAdaptor(
2288 Pass: LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
2289 /*AllowSpeculation=*/true),
2290 /*USeMemorySSA=*/UseMemorySSA: true));
2291
2292 if (RunNewGVN)
2293 MainFPM.addPass(Pass: NewGVNPass());
2294 else
2295 MainFPM.addPass(Pass: GVNPass());
2296
2297 // Remove dead memcpy()'s.
2298 MainFPM.addPass(Pass: MemCpyOptPass());
2299
2300 // Nuke dead stores.
2301 MainFPM.addPass(Pass: DSEPass());
2302 MainFPM.addPass(Pass: MoveAutoInitPass());
2303 MainFPM.addPass(Pass: MergedLoadStoreMotionPass());
2304
2305 invokeVectorizerStartEPCallbacks(FPM&: MainFPM, Level);
2306
2307 LoopPassManager LPM;
2308 if (EnableLoopFlatten && Level >= OptimizationLevel::O2)
2309 LPM.addPass(Pass: LoopFlattenPass());
2310 LPM.addPass(Pass: IndVarSimplifyPass());
2311 LPM.addPass(Pass: LoopDeletionPass());
2312 // FIXME: Add loop interchange.
2313
2314 // Unroll small loops and perform peeling.
2315 LPM.addPass(Pass: LoopFullUnrollPass(static_cast<int>(Level),
2316 /* OnlyWhenForced= */ !PTO.LoopUnrolling,
2317 PTO.ForgetAllSCEVInLoopUnroll));
2318 // The loop passes in LPM (LoopFullUnrollPass) do not preserve MemorySSA.
2319 // *All* loop passes must preserve it, in order to be able to use it.
2320 MainFPM.addPass(
2321 Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM), /*UseMemorySSA=*/false));
2322
2323 MainFPM.addPass(Pass: LoopDistributePass());
2324
2325 addVectorPasses(Level, FPM&: MainFPM, LTOPhase: ThinOrFullLTOPhase::FullLTOPostLink);
2326
2327 invokeVectorizerEndEPCallbacks(FPM&: MainFPM, Level);
2328
2329 // Run the OpenMPOpt CGSCC pass again late.
2330 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(
2331 Pass: OpenMPOptCGSCCPass(ThinOrFullLTOPhase::FullLTOPostLink)));
2332
2333 invokePeepholeEPCallbacks(FPM&: MainFPM, Level);
2334 MainFPM.addPass(Pass: JumpThreadingPass());
2335 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(MainFPM),
2336 EagerlyInvalidate: PTO.EagerlyInvalidateAnalyses));
2337
2338 // Lower type metadata and the type.test intrinsic. This pass supports
2339 // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
2340 // to be run at link time if CFI is enabled. This pass does nothing if
2341 // CFI is disabled.
2342 MPM.addPass(Pass: LowerTypeTestsPass(ExportSummary, nullptr));
2343 // Run a second time to clean up any type tests left behind by WPD for use
2344 // in ICP (which is performed earlier than this in the regular LTO pipeline).
2345 MPM.addPass(Pass: DropTypeTestsPass());
2346
2347 // Enable splitting late in the FullLTO post-link pipeline.
2348 if (EnableHotColdSplit)
2349 MPM.addPass(Pass: HotColdSplittingPass());
2350
2351 // Add late LTO optimization passes.
2352 FunctionPassManager LateFPM;
2353
2354 // LoopSink pass sinks instructions hoisted by LICM, which serves as a
2355 // canonicalization pass that enables other optimizations. As a result,
2356 // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
2357 // result too early.
2358 LateFPM.addPass(Pass: LoopSinkPass());
2359
2360 // This hoists/decomposes div/rem ops. It should run after other sink/hoist
2361 // passes to avoid re-sinking, but before SimplifyCFG because it can allow
2362 // flattening of blocks.
2363 LateFPM.addPass(Pass: DivRemPairsPass());
2364
2365 // Delete basic blocks, which optimization passes may have killed.
2366 LateFPM.addPass(Pass: SimplifyCFGPass(SimplifyCFGOptions()
2367 .convertSwitchRangeToICmp(B: true)
2368 .convertSwitchToArithmetic(B: true)
2369 .hoistCommonInsts(B: true)
2370 .speculateUnpredictables(B: true)));
2371 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(LateFPM)));
2372
2373 // Drop bodies of available eternally objects to improve GlobalDCE.
2374 MPM.addPass(Pass: EliminateAvailableExternallyPass());
2375
2376 // Now that we have optimized the program, discard unreachable functions.
2377 MPM.addPass(Pass: GlobalDCEPass(/*InLTOPostLink=*/true));
2378
2379 if (PTO.MergeFunctions)
2380 MPM.addPass(Pass: MergeFunctionsPass());
2381
2382 MPM.addPass(Pass: RelLookupTableConverterPass());
2383
2384 if (PTO.CallGraphProfile)
2385 MPM.addPass(Pass: CGProfilePass(/*InLTOPostLink=*/true));
2386
2387 MPM.addPass(Pass: CoroCleanupPass());
2388
2389 // AllocToken transforms heap allocation calls; this needs to run late after
2390 // other allocation call transformations (such as those in InstCombine).
2391 MPM.addPass(Pass: AllocTokenPass());
2392
2393 invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);
2394
2395 // Emit annotation remarks.
2396 addAnnotationRemarksPass(MPM);
2397
2398 instructionCountersPass(MPM, /* IsPreOptimization */ false);
2399
2400 return MPM;
2401}
2402
2403ModulePassManager
2404PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level,
2405 ThinOrFullLTOPhase Phase) {
2406 assert(Level == OptimizationLevel::O0 &&
2407 "buildO0DefaultPipeline should only be used with O0");
2408
2409 ModulePassManager MPM;
2410
2411 instructionCountersPass(MPM, /* IsPreOptimization */ true);
2412
2413 // Perform pseudo probe instrumentation in O0 mode. This is for the
2414 // consistency between different build modes. For example, a LTO build can be
2415 // mixed with an O0 prelink and an O2 postlink. Loading a sample profile in
2416 // the postlink will require pseudo probe instrumentation in the prelink.
2417 if (PGOOpt && PGOOpt->PseudoProbeForProfiling)
2418 MPM.addPass(Pass: SampleProfileProbePass(TM));
2419
2420 if (PGOOpt && (PGOOpt->Action == PGOOptions::IRInstr ||
2421 PGOOpt->Action == PGOOptions::IRUse))
2422 addPGOInstrPassesForO0(
2423 MPM,
2424 /*RunProfileGen=*/(PGOOpt->Action == PGOOptions::IRInstr),
2425 /*IsCS=*/false, AtomicCounterUpdate: PGOOpt->AtomicCounterUpdate, ProfileFile: PGOOpt->ProfileFile,
2426 ProfileRemappingFile: PGOOpt->ProfileRemappingFile);
2427
2428 // Instrument function entry and exit before all inlining.
2429 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
2430 Pass: EntryExitInstrumenterPass(/*PostInlining=*/false)));
2431
2432 invokePipelineStartEPCallbacks(MPM, Level);
2433
2434 if (PGOOpt && PGOOpt->DebugInfoForProfiling)
2435 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: AddDiscriminatorsPass()));
2436
2437 if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
2438 // Explicitly disable sample loader inlining and use flattened profile in O0
2439 // pipeline.
2440 MPM.addPass(Pass: SampleProfileLoaderPass(PGOOpt->ProfileFile,
2441 PGOOpt->ProfileRemappingFile,
2442 ThinOrFullLTOPhase::None, FS,
2443 /*DisableSampleProfileInlining=*/true,
2444 /*UseFlattenedProfile=*/true));
2445 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
2446 // RequireAnalysisPass for PSI before subsequent non-module passes.
2447 MPM.addPass(Pass: RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
2448 }
2449
2450 invokePipelineEarlySimplificationEPCallbacks(MPM, Level, Phase);
2451
2452 // Build a minimal pipeline based on the semantics required by LLVM,
2453 // which is just that always inlining occurs. Further, disable generating
2454 // lifetime intrinsics to avoid enabling further optimizations during
2455 // code generation.
2456 MPM.addPass(Pass: AlwaysInlinerPass(
2457 /*InsertLifetimeIntrinsics=*/false));
2458
2459 if (PTO.MergeFunctions)
2460 MPM.addPass(Pass: MergeFunctionsPass());
2461
2462 if (EnableMatrix)
2463 MPM.addPass(
2464 Pass: createModuleToFunctionPassAdaptor(Pass: LowerMatrixIntrinsicsPass(true)));
2465
2466 if (!CGSCCOptimizerLateEPCallbacks.empty()) {
2467 CGSCCPassManager CGPM;
2468 invokeCGSCCOptimizerLateEPCallbacks(CGPM, Level);
2469 if (!CGPM.isEmpty())
2470 MPM.addPass(Pass: createModuleToPostOrderCGSCCPassAdaptor(Pass: std::move(CGPM)));
2471 }
2472 if (!LateLoopOptimizationsEPCallbacks.empty()) {
2473 LoopPassManager LPM;
2474 invokeLateLoopOptimizationsEPCallbacks(LPM, Level);
2475 if (!LPM.isEmpty()) {
2476 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
2477 Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM))));
2478 }
2479 }
2480 if (!LoopOptimizerEndEPCallbacks.empty()) {
2481 LoopPassManager LPM;
2482 invokeLoopOptimizerEndEPCallbacks(LPM, Level);
2483 if (!LPM.isEmpty()) {
2484 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(
2485 Pass: createFunctionToLoopPassAdaptor(Pass: std::move(LPM))));
2486 }
2487 }
2488 if (!ScalarOptimizerLateEPCallbacks.empty()) {
2489 FunctionPassManager FPM;
2490 invokeScalarOptimizerLateEPCallbacks(FPM, Level);
2491 if (!FPM.isEmpty())
2492 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
2493 }
2494
2495 invokeOptimizerEarlyEPCallbacks(MPM, Level, Phase);
2496
2497 if (!VectorizerStartEPCallbacks.empty()) {
2498 FunctionPassManager FPM;
2499 invokeVectorizerStartEPCallbacks(FPM, Level);
2500 if (!FPM.isEmpty())
2501 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
2502 }
2503
2504 if (!VectorizerEndEPCallbacks.empty()) {
2505 FunctionPassManager FPM;
2506 invokeVectorizerEndEPCallbacks(FPM, Level);
2507 if (!FPM.isEmpty())
2508 MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
2509 }
2510
2511 MPM.addPass(Pass: buildCoroWrapper(Phase));
2512
2513 // AllocToken transforms heap allocation calls; this needs to run late after
2514 // other allocation call transformations (such as those in InstCombine).
2515 if (!isLTOPreLink(Phase))
2516 MPM.addPass(Pass: AllocTokenPass());
2517
2518 invokeOptimizerLastEPCallbacks(MPM, Level, Phase);
2519
2520 if (EnableInstrumentor)
2521 MPM.addPass(Pass: InstrumentorPass(FS));
2522
2523 // Attach !implicit.ref metadata from all functions to copyright strings.
2524 MPM.addPass(Pass: LowerCommentStringPass());
2525
2526 if (isLTOPreLink(Phase))
2527 addRequiredLTOPreLinkPasses(MPM);
2528
2529 // Emit annotation remarks.
2530 addAnnotationRemarksPass(MPM);
2531
2532 instructionCountersPass(MPM, /* IsPreOptimization */ false);
2533
2534 return MPM;
2535}
2536
2537AAManager PassBuilder::buildDefaultAAPipeline() {
2538 AAManager AA;
2539
2540 // The order in which these are registered determines their priority when
2541 // being queried.
2542
2543 // Add any target-specific alias analyses that should be run early.
2544 if (TM)
2545 TM->registerEarlyDefaultAliasAnalyses(AA);
2546
2547 // First we register the basic alias analysis that provides the majority of
2548 // per-function local AA logic. This is a stateless, on-demand local set of
2549 // AA techniques.
2550 AA.registerFunctionAnalysis<BasicAA>();
2551
2552 // Next we query fast, specialized alias analyses that wrap IR-embedded
2553 // information about aliasing.
2554 AA.registerFunctionAnalysis<ScopedNoAliasAA>();
2555 AA.registerFunctionAnalysis<TypeBasedAA>();
2556
2557 // Add support for querying global aliasing information when available.
2558 // Because the `AAManager` is a function analysis and `GlobalsAA` is a module
2559 // analysis, all that the `AAManager` can do is query for any *cached*
2560 // results from `GlobalsAA` through a readonly proxy.
2561 if (EnableGlobalAnalyses)
2562 AA.registerModuleAnalysis<GlobalsAA>();
2563
2564 // Add target-specific alias analyses.
2565 if (TM)
2566 TM->registerDefaultAliasAnalyses(AA);
2567
2568 return AA;
2569}
2570
2571bool PassBuilder::isInstrumentedPGOUse() const {
2572 return (PGOOpt && PGOOpt->Action == PGOOptions::IRUse) ||
2573 !UseCtxProfile.empty();
2574}
2575