1//===- TargetPassConfig.cpp - Target independent code generation passes ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines interfaces to access the target independent code
10// generation passes provided by the LLVM backend.
11//
12//===---------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/TargetPassConfig.h"
15#include "llvm/ADT/DenseMap.h"
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/Analysis/BasicAliasAnalysis.h"
19#include "llvm/Analysis/CallGraphSCCPass.h"
20#include "llvm/Analysis/ScopedNoAliasAA.h"
21#include "llvm/Analysis/TargetTransformInfo.h"
22#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
23#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
24#include "llvm/CodeGen/CSEConfigBase.h"
25#include "llvm/CodeGen/CodeGenTargetMachineImpl.h"
26#include "llvm/CodeGen/MachineFunctionPass.h"
27#include "llvm/CodeGen/MachinePassRegistry.h"
28#include "llvm/CodeGen/Passes.h"
29#include "llvm/CodeGen/RegAllocRegistry.h"
30#include "llvm/IR/IRPrintingPasses.h"
31#include "llvm/IR/LegacyPassManager.h"
32#include "llvm/IR/PassInstrumentation.h"
33#include "llvm/IR/Verifier.h"
34#include "llvm/InitializePasses.h"
35#include "llvm/MC/MCAsmInfo.h"
36#include "llvm/MC/MCTargetOptions.h"
37#include "llvm/Pass.h"
38#include "llvm/Support/CodeGen.h"
39#include "llvm/Support/CommandLine.h"
40#include "llvm/Support/Compiler.h"
41#include "llvm/Support/Debug.h"
42#include "llvm/Support/Discriminator.h"
43#include "llvm/Support/ErrorHandling.h"
44#include "llvm/Support/SaveAndRestore.h"
45#include "llvm/Support/Threading.h"
46#include "llvm/Support/VirtualFileSystem.h"
47#include "llvm/Support/WithColor.h"
48#include "llvm/Target/CGPassBuilderOption.h"
49#include "llvm/Target/TargetMachine.h"
50#include "llvm/Transforms/ObjCARC.h"
51#include "llvm/Transforms/Scalar.h"
52#include "llvm/Transforms/Utils.h"
53#include "llvm/Transforms/Utils/TriggerCrashPass.h"
54#include <cassert>
55#include <optional>
56#include <string>
57
58using namespace llvm;
59
60static cl::opt<bool>
61 EnableIPRA("enable-ipra", cl::init(Val: false), cl::Hidden,
62 cl::desc("Enable interprocedural register allocation "
63 "to reduce load/store at procedure calls."));
64static cl::opt<bool> DisablePostRASched("disable-post-ra", cl::Hidden,
65 cl::desc("Disable Post Regalloc Scheduler"));
66static cl::opt<bool> DisableBranchFold("disable-branch-fold", cl::Hidden,
67 cl::desc("Disable branch folding"));
68static cl::opt<bool> DisableTailDuplicate("disable-tail-duplicate", cl::Hidden,
69 cl::desc("Disable tail duplication"));
70static cl::opt<bool> DisableEarlyTailDup("disable-early-taildup", cl::Hidden,
71 cl::desc("Disable pre-register allocation tail duplication"));
72static cl::opt<bool> DisableBlockPlacement("disable-block-placement",
73 cl::Hidden, cl::desc("Disable probability-driven block placement"));
74static cl::opt<bool> EnableBlockPlacementStats("enable-block-placement-stats",
75 cl::Hidden, cl::desc("Collect probability-driven block placement stats"));
76static cl::opt<bool> DisableSSC("disable-ssc", cl::Hidden,
77 cl::desc("Disable Stack Slot Coloring"));
78static cl::opt<bool> DisableMachineDCE("disable-machine-dce", cl::Hidden,
79 cl::desc("Disable Machine Dead Code Elimination"));
80static cl::opt<bool> DisableEarlyIfConversion("disable-early-ifcvt", cl::Hidden,
81 cl::desc("Disable Early If-conversion"));
82static cl::opt<bool> DisableMachineLICM("disable-machine-licm", cl::Hidden,
83 cl::desc("Disable Machine LICM"));
84static cl::opt<bool> DisableMachineCSE("disable-machine-cse", cl::Hidden,
85 cl::desc("Disable Machine Common Subexpression Elimination"));
86static cl::opt<cl::boolOrDefault> OptimizeRegAlloc(
87 "optimize-regalloc", cl::Hidden,
88 cl::desc("Enable optimized register allocation compilation path."));
89static cl::opt<bool> DisablePostRAMachineLICM("disable-postra-machine-licm",
90 cl::Hidden,
91 cl::desc("Disable Machine LICM"));
92static cl::opt<bool> DisableMachineSink("disable-machine-sink", cl::Hidden,
93 cl::desc("Disable Machine Sinking"));
94static cl::opt<bool> DisablePostRAMachineSink("disable-postra-machine-sink",
95 cl::Hidden,
96 cl::desc("Disable PostRA Machine Sinking"));
97static cl::opt<bool> DisableLSR("disable-lsr", cl::Hidden,
98 cl::desc("Disable Loop Strength Reduction Pass"));
99static cl::opt<bool> DisableConstantHoisting("disable-constant-hoisting",
100 cl::Hidden, cl::desc("Disable ConstantHoisting"));
101static cl::opt<bool> DisableCGP("disable-cgp", cl::Hidden,
102 cl::desc("Disable Codegen Prepare"));
103
104static cl::opt<bool>
105 TriggerCrash("codegen-pipeline-trigger-crash", cl::init(Val: false), cl::Hidden,
106 cl::desc("Trigger crash in codegen pipeline"));
107
108static cl::opt<bool> DisableCopyProp("disable-copyprop", cl::Hidden,
109 cl::desc("Disable Copy Propagation pass"));
110static cl::opt<bool> DisablePartialLibcallInlining("disable-partial-libcall-inlining",
111 cl::Hidden, cl::desc("Disable Partial Libcall Inlining"));
112static cl::opt<bool> DisableAtExitBasedGlobalDtorLowering(
113 "disable-atexit-based-global-dtor-lowering", cl::Hidden,
114 cl::desc("For MachO, disable atexit()-based global destructor lowering"));
115static cl::opt<bool> EnableImplicitNullChecks(
116 "enable-implicit-null-checks",
117 cl::desc("Fold null checks into faulting memory operations"),
118 cl::init(Val: false), cl::Hidden);
119static cl::opt<bool>
120 PrintISelInput("print-isel-input", cl::Hidden,
121 cl::desc("Print LLVM IR input to isel pass"));
122cl::opt<bool>
123 PrintRegUsage("print-regusage", cl::Hidden,
124 cl::desc("Print register usage details collected for IPRA"));
125static cl::opt<cl::boolOrDefault>
126 VerifyMachineCode("verify-machineinstrs", cl::Hidden,
127 cl::desc("Verify generated machine code"));
128static cl::opt<cl::boolOrDefault>
129 DebugifyAndStripAll("debugify-and-strip-all-safe", cl::Hidden,
130 cl::desc("Debugify MIR before and Strip debug after "
131 "each pass except those known to be unsafe "
132 "when debug info is present"));
133static cl::opt<cl::boolOrDefault> DebugifyCheckAndStripAll(
134 "debugify-check-and-strip-all-safe", cl::Hidden,
135 cl::desc(
136 "Debugify MIR before, by checking and stripping the debug info after, "
137 "each pass except those known to be unsafe when debug info is "
138 "present"));
139// Enable or disable the MachineOutliner.
140static cl::opt<RunOutliner> EnableMachineOutliner(
141 "enable-machine-outliner", cl::desc("Enable the machine outliner"),
142 cl::Hidden, cl::ValueOptional, cl::init(Val: RunOutliner::TargetDefault),
143 cl::values(
144 clEnumValN(RunOutliner::AlwaysOutline, "always",
145 "Run on all functions guaranteed to be beneficial"),
146 clEnumValN(RunOutliner::OptimisticPGO, "optimistic-pgo",
147 "Outline cold code only. If a code block does not have "
148 "profile data, optimistically assume it is cold."),
149 clEnumValN(RunOutliner::ConservativePGO, "conservative-pgo",
150 "Outline cold code only. If a code block does not have "
151 "profile, data, conservatively assume it is hot."),
152 clEnumValN(RunOutliner::NeverOutline, "never", "Disable all outlining"),
153 // Sentinel value for unspecified option.
154 clEnumValN(RunOutliner::AlwaysOutline, "", "")));
155static cl::opt<bool> EnableGlobalMergeFunc(
156 "enable-global-merge-func", cl::Hidden,
157 cl::desc("Enable global merge functions that are based on hash function"));
158// Disable the pass to fix unwind information. Whether the pass is included in
159// the pipeline is controlled via the target options, this option serves as
160// manual override.
161static cl::opt<bool> DisableCFIFixup("disable-cfi-fixup", cl::Hidden,
162 cl::desc("Disable the CFI fixup pass"));
163// Enable or disable FastISel. Both options are needed, because
164// FastISel is enabled by default with -fast, and we wish to be
165// able to enable or disable fast-isel independently from -O0.
166static cl::opt<cl::boolOrDefault>
167EnableFastISelOption("fast-isel", cl::Hidden,
168 cl::desc("Enable the \"fast\" instruction selector"));
169
170static cl::opt<cl::boolOrDefault> EnableGlobalISelOption(
171 "global-isel", cl::Hidden,
172 cl::desc("Enable the \"global\" instruction selector"));
173
174// FIXME: remove this after switching to NPM or GlobalISel, whichever gets there
175// first...
176static cl::opt<bool>
177 PrintAfterISel("print-after-isel", cl::init(Val: false), cl::Hidden,
178 cl::desc("Print machine instrs after ISel"));
179
180static cl::opt<GlobalISelAbortMode> EnableGlobalISelAbort(
181 "global-isel-abort", cl::Hidden,
182 cl::desc("Enable abort calls when \"global\" instruction selection "
183 "fails to lower/select an instruction"),
184 cl::values(
185 clEnumValN(GlobalISelAbortMode::Disable, "0", "Disable the abort"),
186 clEnumValN(GlobalISelAbortMode::Enable, "1", "Enable the abort"),
187 clEnumValN(GlobalISelAbortMode::DisableWithDiag, "2",
188 "Disable the abort but emit a diagnostic on failure")));
189
190// Disable MIRProfileLoader before RegAlloc. This is for for debugging and
191// tuning purpose.
192static cl::opt<bool> DisableRAFSProfileLoader(
193 "disable-ra-fsprofile-loader", cl::init(Val: false), cl::Hidden,
194 cl::desc("Disable MIRProfileLoader before RegAlloc"));
195// Disable MIRProfileLoader before BloackPlacement. This is for for debugging
196// and tuning purpose.
197static cl::opt<bool> DisableLayoutFSProfileLoader(
198 "disable-layout-fsprofile-loader", cl::init(Val: false), cl::Hidden,
199 cl::desc("Disable MIRProfileLoader before BlockPlacement"));
200// Specify FSProfile file name.
201static cl::opt<std::string>
202 FSProfileFile("fs-profile-file", cl::init(Val: ""), cl::value_desc("filename"),
203 cl::desc("Flow Sensitive profile file name."), cl::Hidden);
204// Specify Remapping file for FSProfile.
205static cl::opt<std::string> FSRemappingFile(
206 "fs-remapping-file", cl::init(Val: ""), cl::value_desc("filename"),
207 cl::desc("Flow Sensitive profile remapping file name."), cl::Hidden);
208
209// Temporary option to allow experimenting with MachineScheduler as a post-RA
210// scheduler. Targets can "properly" enable this with
211// substitutePass(&PostRASchedulerID, &PostMachineSchedulerID).
212// Targets can return true in targetSchedulesPostRAScheduling() and
213// insert a PostRA scheduling pass wherever it wants.
214static cl::opt<bool> MISchedPostRA(
215 "misched-postra", cl::Hidden,
216 cl::desc(
217 "Run MachineScheduler post regalloc (independent of preRA sched)"));
218
219// Experimental option to run live interval analysis early.
220static cl::opt<bool> EarlyLiveIntervals("early-live-intervals", cl::Hidden,
221 cl::desc("Run live interval analysis earlier in the pipeline"));
222
223static cl::opt<bool> DisableReplaceWithVecLib(
224 "disable-replace-with-vec-lib", cl::Hidden,
225 cl::desc("Disable replace with vector math call pass"));
226
227/// Option names for limiting the codegen pipeline.
228/// Those are used in error reporting and we didn't want
229/// to duplicate their names all over the place.
230static const char StartAfterOptName[] = "start-after";
231static const char StartBeforeOptName[] = "start-before";
232static const char StopAfterOptName[] = "stop-after";
233static const char StopBeforeOptName[] = "stop-before";
234
235static cl::opt<std::string>
236 StartAfterOpt(StringRef(StartAfterOptName),
237 cl::desc("Resume compilation after a specific pass"),
238 cl::value_desc("pass-name"), cl::init(Val: ""), cl::Hidden);
239
240static cl::opt<std::string>
241 StartBeforeOpt(StringRef(StartBeforeOptName),
242 cl::desc("Resume compilation before a specific pass"),
243 cl::value_desc("pass-name"), cl::init(Val: ""), cl::Hidden);
244
245static cl::opt<std::string>
246 StopAfterOpt(StringRef(StopAfterOptName),
247 cl::desc("Stop compilation after a specific pass"),
248 cl::value_desc("pass-name"), cl::init(Val: ""), cl::Hidden);
249
250static cl::opt<std::string>
251 StopBeforeOpt(StringRef(StopBeforeOptName),
252 cl::desc("Stop compilation before a specific pass"),
253 cl::value_desc("pass-name"), cl::init(Val: ""), cl::Hidden);
254
255/// Enable the machine function splitter pass.
256static cl::opt<bool> EnableMachineFunctionSplitter(
257 "enable-split-machine-functions", cl::Hidden,
258 cl::desc("Split out cold blocks from machine functions based on profile "
259 "information."));
260
261/// Disable the expand reductions pass for testing.
262static cl::opt<bool> DisableExpandReductions(
263 "disable-expand-reductions", cl::init(Val: false), cl::Hidden,
264 cl::desc("Disable the expand reduction intrinsics pass from running"));
265
266/// Disable the select optimization pass.
267static cl::opt<bool> DisableSelectOptimize(
268 "disable-select-optimize", cl::init(Val: true), cl::Hidden,
269 cl::desc("Disable the select-optimization pass from running"));
270
271/// Enable garbage-collecting empty basic blocks.
272static cl::opt<bool> EnableGCEmptyBlocks(
273 "enable-gc-empty-basic-blocks", cl::init(Val: false), cl::Hidden,
274 cl::desc("Enable garbage-collecting empty basic blocks"));
275
276static cl::opt<bool>
277 SplitStaticData("split-static-data", cl::Hidden, cl::init(Val: false),
278 cl::desc("Split static data sections into hot and cold "
279 "sections using profile information"));
280
281/// Enable matching and inference when using propeller.
282static cl::opt<bool> BasicBlockSectionMatchInfer(
283 "basic-block-section-match-infer",
284 cl::desc(
285 "Enable matching and inference when generating basic block sections"),
286 cl::init(Val: false), cl::Optional);
287
288cl::opt<bool> EmitBBHash(
289 "emit-bb-hash",
290 cl::desc(
291 "Emit the hash of basic block in the SHT_LLVM_BB_ADDR_MAP section."),
292 cl::init(Val: false), cl::Optional);
293
294/// Allow standard passes to be disabled by command line options. This supports
295/// simple binary flags that either suppress the pass or do nothing.
296/// i.e. -disable-mypass=false has no effect.
297/// These should be converted to boolOrDefault in order to use applyOverride.
298static IdentifyingPassPtr applyDisable(IdentifyingPassPtr PassID,
299 bool Override) {
300 if (Override)
301 return IdentifyingPassPtr();
302 return PassID;
303}
304
305/// Allow standard passes to be disabled by the command line, regardless of who
306/// is adding the pass.
307///
308/// StandardID is the pass identified in the standard pass pipeline and provided
309/// to addPass(). It may be a target-specific ID in the case that the target
310/// directly adds its own pass, but in that case we harmlessly fall through.
311///
312/// TargetID is the pass that the target has configured to override StandardID.
313///
314/// StandardID may be a pseudo ID. In that case TargetID is the name of the real
315/// pass to run. This allows multiple options to control a single pass depending
316/// on where in the pipeline that pass is added.
317static IdentifyingPassPtr overridePass(AnalysisID StandardID,
318 IdentifyingPassPtr TargetID) {
319 if (StandardID == &PostRASchedulerID)
320 return applyDisable(PassID: TargetID, Override: DisablePostRASched);
321
322 if (StandardID == &BranchFolderPassID)
323 return applyDisable(PassID: TargetID, Override: DisableBranchFold);
324
325 if (StandardID == &TailDuplicateLegacyID)
326 return applyDisable(PassID: TargetID, Override: DisableTailDuplicate);
327
328 if (StandardID == &EarlyTailDuplicateLegacyID)
329 return applyDisable(PassID: TargetID, Override: DisableEarlyTailDup);
330
331 if (StandardID == &MachineBlockPlacementID)
332 return applyDisable(PassID: TargetID, Override: DisableBlockPlacement);
333
334 if (StandardID == &StackSlotColoringID)
335 return applyDisable(PassID: TargetID, Override: DisableSSC);
336
337 if (StandardID == &DeadMachineInstructionElimID)
338 return applyDisable(PassID: TargetID, Override: DisableMachineDCE);
339
340 if (StandardID == &EarlyIfConverterLegacyID)
341 return applyDisable(PassID: TargetID, Override: DisableEarlyIfConversion);
342
343 if (StandardID == &EarlyMachineLICMID)
344 return applyDisable(PassID: TargetID, Override: DisableMachineLICM);
345
346 if (StandardID == &MachineCSELegacyID)
347 return applyDisable(PassID: TargetID, Override: DisableMachineCSE);
348
349 if (StandardID == &MachineLICMID)
350 return applyDisable(PassID: TargetID, Override: DisablePostRAMachineLICM);
351
352 if (StandardID == &MachineSinkingLegacyID)
353 return applyDisable(PassID: TargetID, Override: DisableMachineSink);
354
355 if (StandardID == &PostRAMachineSinkingID)
356 return applyDisable(PassID: TargetID, Override: DisablePostRAMachineSink);
357
358 if (StandardID == &MachineCopyPropagationID)
359 return applyDisable(PassID: TargetID, Override: DisableCopyProp);
360
361 return TargetID;
362}
363
364// Find the FSProfile file name. The internal option takes the precedence
365// before getting from TargetMachine.
366static std::string getFSProfileFile(const TargetMachine *TM) {
367 if (!FSProfileFile.empty())
368 return FSProfileFile.getValue();
369 const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
370 if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
371 return std::string();
372 return PGOOpt->ProfileFile;
373}
374
375// Find the Profile remapping file name. The internal option takes the
376// precedence before getting from TargetMachine.
377static std::string getFSRemappingFile(const TargetMachine *TM) {
378 if (!FSRemappingFile.empty())
379 return FSRemappingFile.getValue();
380 const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
381 if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
382 return std::string();
383 return PGOOpt->ProfileRemappingFile;
384}
385
386//===---------------------------------------------------------------------===//
387/// TargetPassConfig
388//===---------------------------------------------------------------------===//
389
390INITIALIZE_PASS(TargetPassConfig, "targetpassconfig",
391 "Target Pass Configuration", false, false)
392char TargetPassConfig::ID = 0;
393
394namespace {
395
396struct InsertedPass {
397 AnalysisID TargetPassID;
398 IdentifyingPassPtr InsertedPassID;
399
400 InsertedPass(AnalysisID TargetPassID, IdentifyingPassPtr InsertedPassID)
401 : TargetPassID(TargetPassID), InsertedPassID(InsertedPassID) {}
402
403 Pass *getInsertedPass() const {
404 assert(InsertedPassID.isValid() && "Illegal Pass ID!");
405 if (InsertedPassID.isInstance())
406 return InsertedPassID.getInstance();
407 Pass *NP = Pass::createPass(ID: InsertedPassID.getID());
408 assert(NP && "Pass ID not registered");
409 return NP;
410 }
411};
412
413} // end anonymous namespace
414
415namespace llvm {
416
417class PassConfigImpl {
418public:
419 // List of passes explicitly substituted by this target. Normally this is
420 // empty, but it is a convenient way to suppress or replace specific passes
421 // that are part of a standard pass pipeline without overridding the entire
422 // pipeline. This mechanism allows target options to inherit a standard pass's
423 // user interface. For example, a target may disable a standard pass by
424 // default by substituting a pass ID of zero, and the user may still enable
425 // that standard pass with an explicit command line option.
426 DenseMap<AnalysisID,IdentifyingPassPtr> TargetPasses;
427
428 /// Store the pairs of <AnalysisID, AnalysisID> of which the second pass
429 /// is inserted after each instance of the first one.
430 SmallVector<InsertedPass, 4> InsertedPasses;
431};
432
433} // end namespace llvm
434
435// Out of line virtual method.
436TargetPassConfig::~TargetPassConfig() {
437 delete Impl;
438}
439
440static const PassInfo *getPassInfo(StringRef PassName) {
441 if (PassName.empty())
442 return nullptr;
443
444 const PassRegistry &PR = *PassRegistry::getPassRegistry();
445 const PassInfo *PI = PR.getPassInfo(Arg: PassName);
446 if (!PI)
447 reportFatalUsageError(reason: Twine('\"') + Twine(PassName) +
448 Twine("\" pass is not registered."));
449 return PI;
450}
451
452static AnalysisID getPassIDFromName(StringRef PassName) {
453 const PassInfo *PI = getPassInfo(PassName);
454 return PI ? PI->getTypeInfo() : nullptr;
455}
456
457static std::pair<StringRef, unsigned>
458getPassNameAndInstanceNum(StringRef PassName) {
459 StringRef Name, InstanceNumStr;
460 std::tie(args&: Name, args&: InstanceNumStr) = PassName.split(Separator: ',');
461
462 unsigned InstanceNum = 0;
463 if (!InstanceNumStr.empty() && InstanceNumStr.getAsInteger(Radix: 10, Result&: InstanceNum))
464 reportFatalUsageError(reason: "invalid pass instance specifier " + PassName);
465
466 return std::make_pair(x&: Name, y&: InstanceNum);
467}
468
469void TargetPassConfig::setStartStopPasses() {
470 StringRef StartBeforeName;
471 std::tie(args&: StartBeforeName, args&: StartBeforeInstanceNum) =
472 getPassNameAndInstanceNum(PassName: StartBeforeOpt);
473
474 StringRef StartAfterName;
475 std::tie(args&: StartAfterName, args&: StartAfterInstanceNum) =
476 getPassNameAndInstanceNum(PassName: StartAfterOpt);
477
478 StringRef StopBeforeName;
479 std::tie(args&: StopBeforeName, args&: StopBeforeInstanceNum)
480 = getPassNameAndInstanceNum(PassName: StopBeforeOpt);
481
482 StringRef StopAfterName;
483 std::tie(args&: StopAfterName, args&: StopAfterInstanceNum)
484 = getPassNameAndInstanceNum(PassName: StopAfterOpt);
485
486 StartBefore = getPassIDFromName(PassName: StartBeforeName);
487 StartAfter = getPassIDFromName(PassName: StartAfterName);
488 StopBefore = getPassIDFromName(PassName: StopBeforeName);
489 StopAfter = getPassIDFromName(PassName: StopAfterName);
490 if (StartBefore && StartAfter)
491 reportFatalUsageError(reason: Twine(StartBeforeOptName) + Twine(" and ") +
492 Twine(StartAfterOptName) + Twine(" specified!"));
493 if (StopBefore && StopAfter)
494 reportFatalUsageError(reason: Twine(StopBeforeOptName) + Twine(" and ") +
495 Twine(StopAfterOptName) + Twine(" specified!"));
496 Started = (StartAfter == nullptr) && (StartBefore == nullptr);
497}
498
499CGPassBuilderOption llvm::getCGPassBuilderOption() {
500 CGPassBuilderOption Opt;
501
502#define SET_OPTION_IF_PRESENT(Option) \
503 if (Option.getNumOccurrences()) \
504 Opt.Option = Option;
505
506 SET_OPTION_IF_PRESENT(EnableGlobalISelAbort)
507 SET_OPTION_IF_PRESENT(EnableIPRA)
508
509#define SET_OPTION(Option) Opt.Option = Option;
510
511 SET_OPTION(OptimizeRegAlloc)
512 SET_OPTION(EnableFastISelOption)
513 SET_OPTION(EnableGlobalISelOption)
514 SET_OPTION(VerifyMachineCode)
515 SET_OPTION(DisableAtExitBasedGlobalDtorLowering)
516 SET_OPTION(DisableExpandReductions)
517 SET_OPTION(PrintAfterISel)
518 SET_OPTION(FSProfileFile)
519 SET_OPTION(EnableGCEmptyBlocks)
520 SET_OPTION(EarlyLiveIntervals)
521 SET_OPTION(EnableBlockPlacementStats)
522 SET_OPTION(EnableGlobalMergeFunc)
523 SET_OPTION(EnableImplicitNullChecks)
524 SET_OPTION(EnableMachineOutliner)
525 SET_OPTION(MISchedPostRA)
526 SET_OPTION(DisableLSR)
527 SET_OPTION(DisableConstantHoisting)
528 SET_OPTION(DisableCGP)
529 SET_OPTION(DisablePartialLibcallInlining)
530 SET_OPTION(DisableSelectOptimize)
531 SET_OPTION(PrintISelInput)
532 SET_OPTION(PrintRegUsage)
533 SET_OPTION(DebugifyAndStripAll)
534 SET_OPTION(DebugifyCheckAndStripAll)
535 SET_OPTION(DisableRAFSProfileLoader)
536 SET_OPTION(DisableCFIFixup)
537 SET_OPTION(EnableMachineFunctionSplitter)
538
539 return Opt;
540}
541
542void llvm::registerCodeGenCallback(PassInstrumentationCallbacks &PIC,
543 TargetMachine &TM) {
544
545 // Register a callback for disabling passes.
546 PIC.registerShouldRunOptionalPassCallback(C: [](StringRef P, Any) {
547
548#define DISABLE_PASS(Option, Name) \
549 if (Option && P.contains(#Name)) \
550 return false;
551 DISABLE_PASS(DisableBlockPlacement, MachineBlockPlacementPass)
552 DISABLE_PASS(DisableBranchFold, BranchFolderPass)
553 DISABLE_PASS(DisableCopyProp, MachineCopyPropagationPass)
554 DISABLE_PASS(DisableEarlyIfConversion, EarlyIfConverterLegacyPass)
555 DISABLE_PASS(DisableEarlyTailDup, EarlyTailDuplicatePass)
556 DISABLE_PASS(DisableMachineCSE, MachineCSELegacyPass)
557 DISABLE_PASS(DisableMachineDCE, DeadMachineInstructionElimPass)
558 DISABLE_PASS(DisableMachineLICM, EarlyMachineLICMPass)
559 DISABLE_PASS(DisableMachineSink, MachineSinkingPass)
560 DISABLE_PASS(DisablePostRAMachineLICM, MachineLICMPass)
561 DISABLE_PASS(DisablePostRAMachineSink, PostRAMachineSinkingPass)
562 DISABLE_PASS(DisablePostRASched, PostRASchedulerPass)
563 DISABLE_PASS(DisableSSC, StackSlotColoringPass)
564 DISABLE_PASS(DisableTailDuplicate, TailDuplicatePass)
565
566 return true;
567 });
568}
569
570Expected<TargetPassConfig::StartStopInfo>
571TargetPassConfig::getStartStopInfo(PassInstrumentationCallbacks &PIC) {
572 auto [StartBefore, StartBeforeInstanceNum] =
573 getPassNameAndInstanceNum(PassName: StartBeforeOpt);
574 auto [StartAfter, StartAfterInstanceNum] =
575 getPassNameAndInstanceNum(PassName: StartAfterOpt);
576 auto [StopBefore, StopBeforeInstanceNum] =
577 getPassNameAndInstanceNum(PassName: StopBeforeOpt);
578 auto [StopAfter, StopAfterInstanceNum] =
579 getPassNameAndInstanceNum(PassName: StopAfterOpt);
580
581 if (!StartBefore.empty() && !StartAfter.empty())
582 return make_error<StringError>(
583 Args: Twine(StartBeforeOptName) + " and " + StartAfterOptName + " specified!",
584 Args: std::make_error_code(e: std::errc::invalid_argument));
585 if (!StopBefore.empty() && !StopAfter.empty())
586 return make_error<StringError>(
587 Args: Twine(StopBeforeOptName) + " and " + StopAfterOptName + " specified!",
588 Args: std::make_error_code(e: std::errc::invalid_argument));
589
590 StartStopInfo Result;
591 Result.StartPass = StartBefore.empty() ? StartAfter : StartBefore;
592 Result.StopPass = StopBefore.empty() ? StopAfter : StopBefore;
593 Result.StartInstanceNum =
594 StartBefore.empty() ? StartAfterInstanceNum : StartBeforeInstanceNum;
595 Result.StopInstanceNum =
596 StopBefore.empty() ? StopAfterInstanceNum : StopBeforeInstanceNum;
597 Result.StartAfter = !StartAfter.empty();
598 Result.StopAfter = !StopAfter.empty();
599 Result.StartInstanceNum += Result.StartInstanceNum == 0;
600 Result.StopInstanceNum += Result.StopInstanceNum == 0;
601 return Result;
602}
603
604// Out of line constructor provides default values for pass options and
605// registers all common codegen passes.
606TargetPassConfig::TargetPassConfig(TargetMachine &TM, PassManagerBase &PM)
607 : ImmutablePass(ID), PM(&PM), TM(&TM) {
608 Impl = new PassConfigImpl();
609
610 PassRegistry &PR = *PassRegistry::getPassRegistry();
611 // Register all target independent codegen passes to activate their PassIDs,
612 // including this pass itself.
613 initializeCodeGen(PR);
614
615 initializeLibcallLoweringInfoWrapperPass(PR);
616
617 // Also register alias analysis passes required by codegen passes.
618 initializeBasicAAWrapperPassPass(PR);
619 initializeAAResultsWrapperPassPass(PR);
620
621 if (EnableIPRA.getNumOccurrences()) {
622 TM.Options.EnableIPRA = EnableIPRA;
623 } else {
624 // If not explicitly specified, use target default.
625 TM.Options.EnableIPRA |= TM.useIPRA();
626 }
627
628 if (TM.Options.EnableIPRA)
629 setRequiresCodeGenSCCOrder();
630
631 if (EnableGlobalISelAbort.getNumOccurrences())
632 TM.Options.GlobalISelAbort = EnableGlobalISelAbort;
633
634 setStartStopPasses();
635}
636
637CodeGenOptLevel TargetPassConfig::getOptLevel() const {
638 return TM->getOptLevel();
639}
640
641/// Insert InsertedPassID pass after TargetPassID.
642void TargetPassConfig::insertPass(AnalysisID TargetPassID,
643 IdentifyingPassPtr InsertedPassID) {
644 assert(((!InsertedPassID.isInstance() &&
645 TargetPassID != InsertedPassID.getID()) ||
646 (InsertedPassID.isInstance() &&
647 TargetPassID != InsertedPassID.getInstance()->getPassID())) &&
648 "Insert a pass after itself!");
649 Impl->InsertedPasses.emplace_back(Args&: TargetPassID, Args&: InsertedPassID);
650}
651
652/// createPassConfig - Create a pass configuration object to be used by
653/// addPassToEmitX methods for generating a pipeline of CodeGen passes.
654///
655/// Targets may override this to extend TargetPassConfig.
656TargetPassConfig *
657CodeGenTargetMachineImpl::createPassConfig(PassManagerBase &PM) {
658 return new TargetPassConfig(*this, PM);
659}
660
661TargetPassConfig::TargetPassConfig()
662 : ImmutablePass(ID) {
663 reportFatalUsageError(reason: "trying to construct TargetPassConfig without a target "
664 "machine. Scheduling a CodeGen pass without a target "
665 "triple set?");
666}
667
668bool TargetPassConfig::willCompleteCodeGenPipeline() {
669 return StopBeforeOpt.empty() && StopAfterOpt.empty();
670}
671
672bool TargetPassConfig::hasLimitedCodeGenPipeline() {
673 return !StartBeforeOpt.empty() || !StartAfterOpt.empty() ||
674 !willCompleteCodeGenPipeline();
675}
676
677std::string TargetPassConfig::getLimitedCodeGenPipelineReason() {
678 if (!hasLimitedCodeGenPipeline())
679 return std::string();
680 std::string Res;
681 static cl::opt<std::string> *PassNames[] = {&StartAfterOpt, &StartBeforeOpt,
682 &StopAfterOpt, &StopBeforeOpt};
683 static const char *OptNames[] = {StartAfterOptName, StartBeforeOptName,
684 StopAfterOptName, StopBeforeOptName};
685 bool IsFirst = true;
686 for (int Idx = 0; Idx < 4; ++Idx)
687 if (!PassNames[Idx]->empty()) {
688 if (!IsFirst)
689 Res += " and ";
690 IsFirst = false;
691 Res += OptNames[Idx];
692 }
693 return Res;
694}
695
696// Helper to verify the analysis is really immutable.
697void TargetPassConfig::setOpt(bool &Opt, bool Val) {
698 assert(!Initialized && "PassConfig is immutable");
699 Opt = Val;
700}
701
702void TargetPassConfig::substitutePass(AnalysisID StandardID,
703 IdentifyingPassPtr TargetID) {
704 Impl->TargetPasses[StandardID] = TargetID;
705}
706
707IdentifyingPassPtr TargetPassConfig::getPassSubstitution(AnalysisID ID) const {
708 DenseMap<AnalysisID, IdentifyingPassPtr>::const_iterator
709 I = Impl->TargetPasses.find(Val: ID);
710 if (I == Impl->TargetPasses.end())
711 return ID;
712 return I->second;
713}
714
715bool TargetPassConfig::isPassSubstitutedOrOverridden(AnalysisID ID) const {
716 IdentifyingPassPtr TargetID = getPassSubstitution(ID);
717 IdentifyingPassPtr FinalPtr = overridePass(StandardID: ID, TargetID);
718 return !FinalPtr.isValid() || FinalPtr.isInstance() ||
719 FinalPtr.getID() != ID;
720}
721
722/// Add a pass to the PassManager if that pass is supposed to be run. If the
723/// Started/Stopped flags indicate either that the compilation should start at
724/// a later pass or that it should stop after an earlier pass, then do not add
725/// the pass. Finally, compare the current pass against the StartAfter
726/// and StopAfter options and change the Started/Stopped flags accordingly.
727void TargetPassConfig::addPass(Pass *P) {
728 assert(!Initialized && "PassConfig is immutable");
729
730 // Cache the Pass ID here in case the pass manager finds this pass is
731 // redundant with ones already scheduled / available, and deletes it.
732 // Fundamentally, once we add the pass to the manager, we no longer own it
733 // and shouldn't reference it.
734 AnalysisID PassID = P->getPassID();
735
736 if (StartBefore == PassID && StartBeforeCount++ == StartBeforeInstanceNum)
737 Started = true;
738 if (StopBefore == PassID && StopBeforeCount++ == StopBeforeInstanceNum)
739 Stopped = true;
740 if (Started && !Stopped) {
741 if (AddingMachinePasses) {
742 // Construct banner message before PM->add() as that may delete the pass.
743 std::string Banner =
744 std::string("After ") + std::string(P->getPassName());
745 addMachinePrePasses();
746 PM->add(P);
747 addMachinePostPasses(Banner);
748 } else {
749 PM->add(P);
750 }
751
752 // Add the passes after the pass P if there is any.
753 for (const auto &IP : Impl->InsertedPasses)
754 if (IP.TargetPassID == PassID)
755 addPass(P: IP.getInsertedPass());
756 } else {
757 delete P;
758 }
759
760 if (StopAfter == PassID && StopAfterCount++ == StopAfterInstanceNum)
761 Stopped = true;
762
763 if (StartAfter == PassID && StartAfterCount++ == StartAfterInstanceNum)
764 Started = true;
765 if (Stopped && !Started)
766 reportFatalUsageError(reason: "Cannot stop compilation after pass that is not run");
767}
768
769/// Add a CodeGen pass at this point in the pipeline after checking for target
770/// and command line overrides.
771///
772/// addPass cannot return a pointer to the pass instance because is internal the
773/// PassManager and the instance we create here may already be freed.
774AnalysisID TargetPassConfig::addPass(AnalysisID PassID) {
775 IdentifyingPassPtr TargetID = getPassSubstitution(ID: PassID);
776 IdentifyingPassPtr FinalPtr = overridePass(StandardID: PassID, TargetID);
777 if (!FinalPtr.isValid())
778 return nullptr;
779
780 Pass *P;
781 if (FinalPtr.isInstance())
782 P = FinalPtr.getInstance();
783 else {
784 P = Pass::createPass(ID: FinalPtr.getID());
785 if (!P)
786 llvm_unreachable("Pass ID not registered");
787 }
788 AnalysisID FinalID = P->getPassID();
789 addPass(P); // Ends the lifetime of P.
790
791 return FinalID;
792}
793
794void TargetPassConfig::printAndVerify(const std::string &Banner) {
795 addPrintPass(Banner);
796 addVerifyPass(Banner);
797}
798
799void TargetPassConfig::addPrintPass(const std::string &Banner) {
800 if (PrintAfterISel)
801 PM->add(P: createMachineFunctionPrinterPass(OS&: dbgs(), Banner));
802}
803
804void TargetPassConfig::addVerifyPass(const std::string &Banner) {
805 bool Verify = VerifyMachineCode == cl::boolOrDefault::BOU_TRUE;
806#ifdef EXPENSIVE_CHECKS
807 if (VerifyMachineCode == cl::boolOrDefault::BOU_UNSET)
808 Verify = TM->isMachineVerifierClean();
809#endif
810 if (Verify)
811 PM->add(P: createMachineVerifierPass(Banner));
812}
813
814void TargetPassConfig::addDebugifyPass() {
815 PM->add(P: createDebugifyMachineModulePass());
816}
817
818void TargetPassConfig::addStripDebugPass() {
819 PM->add(P: createStripDebugMachineModuleLegacyPass(/*OnlyDebugified=*/true));
820}
821
822void TargetPassConfig::addCheckDebugPass() {
823 PM->add(P: createCheckDebugMachineModuleLegacyPass());
824}
825
826void TargetPassConfig::addMachinePrePasses(bool AllowDebugify) {
827 if (AllowDebugify && DebugifyIsSafe &&
828 (DebugifyAndStripAll == cl::boolOrDefault::BOU_TRUE ||
829 DebugifyCheckAndStripAll == cl::boolOrDefault::BOU_TRUE))
830 addDebugifyPass();
831}
832
833void TargetPassConfig::addMachinePostPasses(const std::string &Banner) {
834 if (DebugifyIsSafe) {
835 if (DebugifyCheckAndStripAll == cl::boolOrDefault::BOU_TRUE) {
836 addCheckDebugPass();
837 addStripDebugPass();
838 } else if (DebugifyAndStripAll == cl::boolOrDefault::BOU_TRUE)
839 addStripDebugPass();
840 }
841 addVerifyPass(Banner);
842}
843
844/// Add common target configurable passes that perform LLVM IR to IR transforms
845/// following machine independent optimization.
846void TargetPassConfig::addIRPasses() {
847 // Before running any passes, run the verifier to determine if the input
848 // coming from the front-end and/or optimizer is valid.
849 if (!DisableVerify)
850 addPass(P: createVerifierPass());
851
852 if (getOptLevel() != CodeGenOptLevel::None) {
853 // Basic AliasAnalysis support.
854 // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that
855 // BasicAliasAnalysis wins if they disagree. This is intended to help
856 // support "obvious" type-punning idioms.
857 addPass(P: createTypeBasedAAWrapperPass());
858 addPass(P: createScopedNoAliasAAWrapperPass());
859 addPass(P: createBasicAAWrapperPass());
860
861 // Run loop strength reduction before anything else.
862 if (!DisableLSR) {
863 addPass(P: createCanonicalizeFreezeInLoopsPass());
864 addPass(P: createLoopStrengthReducePass());
865 if (EnableLoopTermFold)
866 addPass(P: createLoopTermFoldPass());
867 }
868 }
869
870 // Run GC lowering passes for builtin collectors
871 // TODO: add a pass insertion point here
872 addPass(PassID: &GCLoweringID);
873 addPass(PassID: &ShadowStackGCLoweringID);
874
875 // For MachO, lower @llvm.global_dtors into @llvm.global_ctors with
876 // __cxa_atexit() calls to avoid emitting the deprecated __mod_term_func.
877 if (TM->getTargetTriple().isOSBinFormatMachO() &&
878 !DisableAtExitBasedGlobalDtorLowering)
879 addPass(P: createLowerGlobalDtorsLegacyPass());
880
881 // Make sure that no unreachable blocks are instruction selected.
882 addPass(P: createUnreachableBlockEliminationPass());
883
884 // Prepare expensive constants for SelectionDAG.
885 if (getOptLevel() != CodeGenOptLevel::None && !DisableConstantHoisting)
886 addPass(P: createConstantHoistingPass());
887
888 if (getOptLevel() != CodeGenOptLevel::None && !DisableReplaceWithVecLib)
889 addPass(P: createReplaceWithVeclibLegacyPass());
890
891 if (getOptLevel() != CodeGenOptLevel::None && !DisablePartialLibcallInlining)
892 addPass(P: createPartiallyInlineLibCallsPass());
893
894 // Instrument function entry after all inlining.
895 addPass(P: createPostInlineEntryExitInstrumenterPass());
896
897 // Add scalarization of target's unsupported masked memory intrinsics pass.
898 // the unsupported intrinsic will be replaced with a chain of basic blocks,
899 // that stores/loads element one-by-one if the appropriate mask bit is set.
900 addPass(P: createScalarizeMaskedMemIntrinLegacyPass());
901
902 // Expand reduction intrinsics into shuffle sequences if the target wants to.
903 // Allow disabling it for testing purposes.
904 if (!DisableExpandReductions)
905 addPass(P: createExpandReductionsPass());
906
907 // Convert conditional moves to conditional jumps when profitable.
908 if (getOptLevel() != CodeGenOptLevel::None && !DisableSelectOptimize)
909 addPass(P: createSelectOptimizePass());
910
911 if (EnableGlobalMergeFunc)
912 addPass(P: createGlobalMergeFuncPass());
913
914 if (TM->getTargetTriple().isOSWindows())
915 addPass(P: createWindowsSecureHotPatchingPass());
916}
917
918/// Turn exception handling constructs into something the code generators can
919/// handle.
920void TargetPassConfig::addPassesToHandleExceptions() {
921 const MCAsmInfo &MCAI = TM->getMCAsmInfo();
922 switch (MCAI.getExceptionHandlingType()) {
923 case ExceptionHandling::SjLj:
924 // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both
925 // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise,
926 // catch info can get misplaced when a selector ends up more than one block
927 // removed from the parent invoke(s). This could happen when a landing
928 // pad is shared by multiple invokes and is also a target of a normal
929 // edge from elsewhere.
930 addPass(P: createSjLjEHPreparePass(TM));
931 [[fallthrough]];
932 case ExceptionHandling::DwarfCFI:
933 case ExceptionHandling::ARM:
934 case ExceptionHandling::AIX:
935 case ExceptionHandling::ZOS:
936 addPass(P: createDwarfEHPass(OptLevel: getOptLevel()));
937 break;
938 case ExceptionHandling::WinEH:
939 // We support using both GCC-style and MSVC-style exceptions on Windows, so
940 // add both preparation passes. Each pass will only actually run if it
941 // recognizes the personality function.
942 addPass(P: createWinEHPass());
943 addPass(P: createDwarfEHPass(OptLevel: getOptLevel()));
944 break;
945 case ExceptionHandling::Wasm:
946 // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs
947 // on catchpads and cleanuppads because it does not outline them into
948 // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we
949 // should remove PHIs there.
950 addPass(P: createWinEHPass(/*DemoteCatchSwitchPHIOnly=*/true));
951 addPass(P: createWasmEHPass());
952 break;
953 case ExceptionHandling::None:
954 addPass(P: createLowerInvokePass());
955
956 // The lower invoke pass may create unreachable code. Remove it.
957 addPass(P: createUnreachableBlockEliminationPass());
958 break;
959 }
960}
961
962/// Add pass to prepare the LLVM IR for code generation. This should be done
963/// before exception handling preparation passes.
964void TargetPassConfig::addCodeGenPrepare() {
965 if (getOptLevel() != CodeGenOptLevel::None && !DisableCGP)
966 addPass(P: createCodeGenPrepareLegacyPass());
967}
968
969/// Add common passes that perform LLVM IR to IR transforms in preparation for
970/// instruction selection.
971void TargetPassConfig::addISelPrepare() {
972 addPreISel();
973
974 // Force codegen to run according to the callgraph.
975 if (requiresCodeGenSCCOrder())
976 addPass(P: new DummyCGSCCPass);
977
978 addPass(P: createInlineAsmPreparePass());
979
980 // Add both the safe stack and the stack protection passes: each of them will
981 // only protect functions that have corresponding attributes.
982 addPass(P: createSafeStackPass());
983 addPass(P: createStackProtectorPass());
984
985 if (PrintISelInput)
986 addPass(P: createPrintFunctionPass(
987 OS&: dbgs(), Banner: "\n\n*** Final LLVM Code input to ISel ***\n"));
988
989 // All passes which modify the LLVM IR are now complete; run the verifier
990 // to ensure that the IR is valid.
991 if (!DisableVerify)
992 addPass(P: createVerifierPass());
993}
994
995bool TargetPassConfig::addCoreISelPasses() {
996 // Enable FastISel with -fast-isel, but allow that to be overridden.
997 TM->setO0WantsFastISel(EnableFastISelOption != cl::boolOrDefault::BOU_FALSE);
998
999 // Determine an instruction selector.
1000 enum class SelectorType { SelectionDAG, FastISel, GlobalISel };
1001 SelectorType Selector;
1002
1003 if (EnableFastISelOption == cl::boolOrDefault::BOU_TRUE)
1004 Selector = SelectorType::FastISel;
1005 else if (EnableGlobalISelOption == cl::boolOrDefault::BOU_TRUE ||
1006 (TM->Options.EnableGlobalISel &&
1007 EnableGlobalISelOption != cl::boolOrDefault::BOU_FALSE))
1008 Selector = SelectorType::GlobalISel;
1009 else if (TM->getOptLevel() == CodeGenOptLevel::None &&
1010 TM->getO0WantsFastISel())
1011 Selector = SelectorType::FastISel;
1012 else
1013 Selector = SelectorType::SelectionDAG;
1014
1015 // Set consistently TM->Options.EnableFastISel and EnableGlobalISel.
1016 if (Selector == SelectorType::FastISel) {
1017 TM->setFastISel(true);
1018 TM->setGlobalISel(false);
1019 } else if (Selector == SelectorType::GlobalISel) {
1020 TM->setFastISel(false);
1021 TM->setGlobalISel(true);
1022 }
1023
1024 // FIXME: Injecting into the DAGISel pipeline seems to cause issues with
1025 // analyses needing to be re-run. This can result in being unable to
1026 // schedule passes (particularly with 'Function Alias Analysis
1027 // Results'). It's not entirely clear why but AFAICT this seems to be
1028 // due to one FunctionPassManager not being able to use analyses from a
1029 // previous one. As we're injecting a ModulePass we break the usual
1030 // pass manager into two. GlobalISel with the fallback path disabled
1031 // and -run-pass seem to be unaffected. The majority of GlobalISel
1032 // testing uses -run-pass so this probably isn't too bad.
1033 SaveAndRestore SavedDebugifyIsSafe(DebugifyIsSafe);
1034 if (Selector != SelectorType::GlobalISel || !isGlobalISelAbortEnabled())
1035 DebugifyIsSafe = false;
1036
1037 // Add instruction selector passes for global isel if enabled.
1038 if (Selector == SelectorType::GlobalISel) {
1039 SaveAndRestore SavedAddingMachinePasses(AddingMachinePasses, true);
1040 if (addIRTranslator())
1041 return true;
1042
1043 addPreLegalizeMachineIR();
1044
1045 if (addLegalizeMachineIR())
1046 return true;
1047
1048 // Before running the register bank selector, ask the target if it
1049 // wants to run some passes.
1050 addPreRegBankSelect();
1051
1052 if (addRegBankSelect())
1053 return true;
1054
1055 addPreGlobalInstructionSelect();
1056
1057 if (addGlobalInstructionSelect())
1058 return true;
1059 }
1060
1061 // Pass to reset the MachineFunction if the ISel failed. Outside of the above
1062 // if so that the verifier is not added to it.
1063 if (Selector == SelectorType::GlobalISel)
1064 addPass(P: createResetMachineFunctionPass(
1065 EmitFallbackDiag: reportDiagnosticWhenGlobalISelFallback(), AbortOnFailedISel: isGlobalISelAbortEnabled()));
1066
1067 // Run the SDAG InstSelector, providing a fallback path when we do not want to
1068 // abort on not-yet-supported input.
1069 if (Selector != SelectorType::GlobalISel || !isGlobalISelAbortEnabled())
1070 if (addInstSelector())
1071 return true;
1072
1073 // Expand pseudo-instructions emitted by ISel. Don't run the verifier before
1074 // FinalizeISel.
1075 addPass(PassID: &FinalizeISelID);
1076
1077 // Print the instruction selected machine code...
1078 printAndVerify(Banner: "After Instruction Selection");
1079
1080 return false;
1081}
1082
1083bool TargetPassConfig::addISelPasses() {
1084 if (TM->useEmulatedTLS())
1085 addPass(P: createLowerEmuTLSPass());
1086
1087 PM->add(P: createTargetTransformInfoWrapperPass(TIRA: TM->getTargetIRAnalysis()));
1088 // ObjCARCContract operates on ObjC intrinsics and must run before
1089 // PreISelIntrinsicLowering.
1090 if (getOptLevel() != CodeGenOptLevel::None)
1091 addPass(P: createObjCARCContractPass());
1092 addPass(P: createPreISelIntrinsicLoweringPass());
1093 addPass(P: createExpandIRInstsPass(getOptLevel()));
1094 addIRPasses();
1095
1096 if (TriggerCrash)
1097 addPass(P: createTriggerCrashFunctionPass());
1098
1099 addCodeGenPrepare();
1100 addPassesToHandleExceptions();
1101 addISelPrepare();
1102
1103 return addCoreISelPasses();
1104}
1105
1106/// -regalloc=... command line option.
1107static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
1108static cl::opt<RegisterRegAlloc::FunctionPassCtor, false,
1109 RegisterPassParser<RegisterRegAlloc>>
1110 RegAlloc("regalloc", cl::Hidden, cl::init(Val: &useDefaultRegisterAllocator),
1111 cl::desc("Register allocator to use"));
1112
1113/// Add the complete set of target-independent postISel code generator passes.
1114///
1115/// This can be read as the standard order of major LLVM CodeGen stages. Stages
1116/// with nontrivial configuration or multiple passes are broken out below in
1117/// add%Stage routines.
1118///
1119/// Any TargetPassConfig::addXX routine may be overriden by the Target. The
1120/// addPre/Post methods with empty header implementations allow injecting
1121/// target-specific fixups just before or after major stages. Additionally,
1122/// targets have the flexibility to change pass order within a stage by
1123/// overriding default implementation of add%Stage routines below. Each
1124/// technique has maintainability tradeoffs because alternate pass orders are
1125/// not well supported. addPre/Post works better if the target pass is easily
1126/// tied to a common pass. But if it has subtle dependencies on multiple passes,
1127/// the target should override the stage instead.
1128///
1129/// TODO: We could use a single addPre/Post(ID) hook to allow pass injection
1130/// before/after any target-independent pass. But it's currently overkill.
1131void TargetPassConfig::addMachinePasses() {
1132 AddingMachinePasses = true;
1133
1134 // Add passes that optimize machine instructions in SSA form.
1135 if (getOptLevel() != CodeGenOptLevel::None) {
1136 addMachineSSAOptimization();
1137 } else {
1138 // If the target requests it, assign local variables to stack slots relative
1139 // to one another and simplify frame index references where possible.
1140 addPass(PassID: &LocalStackSlotAllocationID);
1141 }
1142
1143 if (TM->Options.EnableIPRA)
1144 addPass(P: createRegUsageInfoPropPass());
1145
1146 // Run pre-ra passes.
1147 addPreRegAlloc();
1148
1149 // Debugifying the register allocator passes seems to provoke some
1150 // non-determinism that affects CodeGen and there doesn't seem to be a point
1151 // where it becomes safe again so stop debugifying here.
1152 DebugifyIsSafe = false;
1153
1154 // Add a FSDiscriminator pass right before RA, so that we could get
1155 // more precise SampleFDO profile for RA.
1156 if (EnableFSDiscriminator) {
1157 addPass(P: createMIRAddFSDiscriminatorsPass(
1158 P: sampleprof::FSDiscriminatorPass::Pass1));
1159 const std::string ProfileFile = getFSProfileFile(TM);
1160 if (!ProfileFile.empty() && !DisableRAFSProfileLoader)
1161 addPass(P: createMIRProfileLoaderPass(File: ProfileFile, RemappingFile: getFSRemappingFile(TM),
1162 P: sampleprof::FSDiscriminatorPass::Pass1,
1163 FS: nullptr));
1164 }
1165
1166 // Run register allocation and passes that are tightly coupled with it,
1167 // including phi elimination and scheduling.
1168 if (getOptimizeRegAlloc())
1169 addOptimizedRegAlloc();
1170 else
1171 addFastRegAlloc();
1172
1173 // Run post-ra passes.
1174 addPostRegAlloc();
1175
1176 addPass(PassID: &RemoveRedundantDebugValuesID);
1177
1178 addPass(PassID: &FixupStatepointCallerSavedID);
1179
1180 // Insert prolog/epilog code. Eliminate abstract frame index references...
1181 if (getOptLevel() != CodeGenOptLevel::None) {
1182 addPass(PassID: &PostRAMachineSinkingID);
1183 addPass(PassID: &ShrinkWrapID);
1184 }
1185
1186 // Prolog/Epilog inserter needs a TargetMachine to instantiate. But only
1187 // do so if it hasn't been disabled, substituted, or overridden.
1188 if (!isPassSubstitutedOrOverridden(ID: &PrologEpilogCodeInserterID))
1189 addPass(P: createPrologEpilogInserterPass());
1190
1191 /// Add passes that optimize machine instructions after register allocation.
1192 if (getOptLevel() != CodeGenOptLevel::None)
1193 addMachineLateOptimization();
1194
1195 // Expand pseudo instructions before second scheduling pass.
1196 addPass(PassID: &ExpandPostRAPseudosID);
1197
1198 // Run pre-sched2 passes.
1199 addPreSched2();
1200
1201 if (EnableImplicitNullChecks)
1202 addPass(PassID: &ImplicitNullChecksID);
1203
1204 // Second pass scheduler.
1205 // Let Target optionally insert this pass by itself at some other
1206 // point.
1207 if (getOptLevel() != CodeGenOptLevel::None &&
1208 !TM->targetSchedulesPostRAScheduling()) {
1209 if (MISchedPostRA)
1210 addPass(PassID: &PostMachineSchedulerID);
1211 else
1212 addPass(PassID: &PostRASchedulerID);
1213 }
1214
1215 // GC
1216 addGCPasses();
1217
1218 // Basic block placement.
1219 if (getOptLevel() != CodeGenOptLevel::None)
1220 addBlockPlacement();
1221
1222 // Insert before XRay Instrumentation.
1223 addPass(PassID: &FEntryInserterID);
1224
1225 addPass(PassID: &XRayInstrumentationID);
1226 addPass(PassID: &PatchableFunctionID);
1227
1228 addPreEmitPass();
1229
1230 if (TM->Options.EnableIPRA)
1231 // Collect register usage information and produce a register mask of
1232 // clobbered registers, to be used to optimize call sites.
1233 addPass(P: createRegUsageInfoCollector());
1234
1235 // FIXME: Some backends are incompatible with running the verifier after
1236 // addPreEmitPass. Maybe only pass "false" here for those targets?
1237 addPass(PassID: &FuncletLayoutID);
1238
1239 addPass(PassID: &RemoveLoadsIntoFakeUsesID);
1240 addPass(PassID: &StackMapLivenessID);
1241 addPass(PassID: &LiveDebugValuesID);
1242 addPass(PassID: &MachineSanitizerBinaryMetadataID);
1243
1244 if (TM->Options.EnableMachineOutliner &&
1245 getOptLevel() != CodeGenOptLevel::None &&
1246 EnableMachineOutliner != RunOutliner::NeverOutline) {
1247 if (EnableMachineOutliner != RunOutliner::TargetDefault ||
1248 TM->Options.SupportsDefaultOutlining)
1249 addPass(P: createMachineOutlinerPass(RunOutlinerMode: EnableMachineOutliner));
1250 }
1251
1252 if (EnableGCEmptyBlocks)
1253 addPass(P: llvm::createGCEmptyBasicBlocksLegacyPass());
1254
1255 if (EnableFSDiscriminator)
1256 addPass(P: createMIRAddFSDiscriminatorsPass(
1257 P: sampleprof::FSDiscriminatorPass::PassLast));
1258
1259 if (TM->Options.EnableMachineFunctionSplitter ||
1260 EnableMachineFunctionSplitter || SplitStaticData ||
1261 TM->Options.EnableStaticDataPartitioning) {
1262 const std::string ProfileFile = getFSProfileFile(TM);
1263 if (!ProfileFile.empty()) {
1264 if (EnableFSDiscriminator) {
1265 addPass(P: createMIRProfileLoaderPass(
1266 File: ProfileFile, RemappingFile: getFSRemappingFile(TM),
1267 P: sampleprof::FSDiscriminatorPass::PassLast, FS: nullptr));
1268 } else {
1269 // Sample profile is given, but FSDiscriminator is not
1270 // enabled, this may result in performance regression.
1271 WithColor::warning()
1272 << "Using AutoFDO without FSDiscriminator for MFS may regress "
1273 "performance.\n";
1274 }
1275 }
1276 }
1277
1278 // Machine function splitter uses the basic block sections feature.
1279 // When used along with `-basic-block-sections=`, the basic-block-sections
1280 // feature takes precedence. This means functions eligible for
1281 // basic-block-sections optimizations (`=all`, or `=list=` with function
1282 // included in the list profile) will get that optimization instead.
1283 if (TM->Options.EnableMachineFunctionSplitter ||
1284 EnableMachineFunctionSplitter)
1285 addPass(P: createMachineFunctionSplitterPass());
1286
1287 if (SplitStaticData || TM->Options.EnableStaticDataPartitioning) {
1288 // The static data splitter pass is a machine function pass. and
1289 // static data annotator pass is a module-wide pass. See the file comment
1290 // in StaticDataAnnotator.cpp for the motivation.
1291 addPass(P: createStaticDataSplitterPass());
1292 addPass(P: createStaticDataAnnotatorPass());
1293 }
1294 // We run the BasicBlockSections pass if either we need BB sections or BB
1295 // address map (or both).
1296 if (TM->getBBSectionsType() != llvm::BasicBlockSection::None ||
1297 TM->Options.BBAddrMap) {
1298 if (EmitBBHash || BasicBlockSectionMatchInfer)
1299 addPass(P: llvm::createMachineBlockHashInfoPass());
1300 if (TM->getBBSectionsType() == llvm::BasicBlockSection::List) {
1301 addPass(P: llvm::createBasicBlockSectionsProfileReaderWrapperPass(
1302 Buf: TM->getBBSectionsFuncListBuf()));
1303 if (BasicBlockSectionMatchInfer)
1304 addPass(P: llvm::createBasicBlockMatchingAndInferencePass());
1305 else {
1306 addPass(P: llvm::createBasicBlockPathCloningPass());
1307 addPass(P: llvm::createInsertCodePrefetchPass());
1308 }
1309 }
1310 addPass(P: llvm::createBasicBlockSectionsPass());
1311 }
1312
1313 addPostBBSections();
1314
1315 if (!DisableCFIFixup && TM->Options.EnableCFIFixup)
1316 addPass(P: createCFIFixupLegacy());
1317
1318 PM->add(P: createStackFrameLayoutAnalysisPass());
1319
1320 // Add passes that directly emit MI after all other MI passes.
1321 addPreEmitPass2();
1322
1323 AddingMachinePasses = false;
1324}
1325
1326/// Add passes that optimize machine instructions in SSA form.
1327void TargetPassConfig::addMachineSSAOptimization() {
1328 // Pre-ra tail duplication.
1329 addPass(PassID: &EarlyTailDuplicateLegacyID);
1330
1331 // Optimize PHIs before DCE: removing dead PHI cycles may make more
1332 // instructions dead.
1333 addPass(PassID: &OptimizePHIsLegacyID);
1334
1335 // This pass merges large allocas. StackSlotColoring is a different pass
1336 // which merges spill slots.
1337 addPass(PassID: &StackColoringLegacyID);
1338
1339 // If the target requests it, assign local variables to stack slots relative
1340 // to one another and simplify frame index references where possible.
1341 addPass(PassID: &LocalStackSlotAllocationID);
1342
1343 // With optimization, dead code should already be eliminated. However
1344 // there is one known exception: lowered code for arguments that are only
1345 // used by tail calls, where the tail calls reuse the incoming stack
1346 // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
1347 addPass(PassID: &DeadMachineInstructionElimID);
1348
1349 // Allow targets to insert passes that improve instruction level parallelism,
1350 // like if-conversion. Such passes will typically need dominator trees and
1351 // loop info, just like LICM and CSE below.
1352 addILPOpts();
1353
1354 addPass(PassID: &EarlyMachineLICMID);
1355 addPass(PassID: &MachineCSELegacyID);
1356
1357 addPass(PassID: &MachineSinkingLegacyID);
1358
1359 addPass(PassID: &PeepholeOptimizerLegacyID);
1360 // Clean-up the dead code that may have been generated by peephole
1361 // rewriting.
1362 addPass(PassID: &DeadMachineInstructionElimID);
1363}
1364
1365//===---------------------------------------------------------------------===//
1366/// Register Allocation Pass Configuration
1367//===---------------------------------------------------------------------===//
1368
1369bool TargetPassConfig::getOptimizeRegAlloc() const {
1370 switch (OptimizeRegAlloc) {
1371 case cl::boolOrDefault::BOU_UNSET:
1372 return getOptLevel() != CodeGenOptLevel::None;
1373 case cl::boolOrDefault::BOU_TRUE:
1374 return true;
1375 case cl::boolOrDefault::BOU_FALSE:
1376 return false;
1377 }
1378 llvm_unreachable("Invalid optimize-regalloc state");
1379}
1380
1381/// A dummy default pass factory indicates whether the register allocator is
1382/// overridden on the command line.
1383static llvm::once_flag InitializeDefaultRegisterAllocatorFlag;
1384
1385static RegisterRegAlloc
1386defaultRegAlloc("default",
1387 "pick register allocator based on -O option",
1388 useDefaultRegisterAllocator);
1389
1390static void initializeDefaultRegisterAllocatorOnce() {
1391 if (!RegisterRegAlloc::getDefault())
1392 RegisterRegAlloc::setDefault(RegAlloc);
1393}
1394
1395/// Instantiate the default register allocator pass for this target for either
1396/// the optimized or unoptimized allocation path. This will be added to the pass
1397/// manager by addFastRegAlloc in the unoptimized case or addOptimizedRegAlloc
1398/// in the optimized case.
1399///
1400/// A target that uses the standard regalloc pass order for fast or optimized
1401/// allocation may still override this for per-target regalloc
1402/// selection. But -regalloc=... always takes precedence.
1403FunctionPass *TargetPassConfig::createTargetRegisterAllocator(bool Optimized) {
1404 if (Optimized)
1405 return createGreedyRegisterAllocator();
1406 else
1407 return createFastRegisterAllocator();
1408}
1409
1410/// Find and instantiate the register allocation pass requested by this target
1411/// at the current optimization level. Different register allocators are
1412/// defined as separate passes because they may require different analysis.
1413///
1414/// This helper ensures that the regalloc= option is always available,
1415/// even for targets that override the default allocator.
1416///
1417/// FIXME: When MachinePassRegistry register pass IDs instead of function ptrs,
1418/// this can be folded into addPass.
1419FunctionPass *TargetPassConfig::createRegAllocPass(bool Optimized) {
1420 // Initialize the global default.
1421 llvm::call_once(flag&: InitializeDefaultRegisterAllocatorFlag,
1422 F&: initializeDefaultRegisterAllocatorOnce);
1423
1424 RegisterRegAlloc::FunctionPassCtor Ctor = RegisterRegAlloc::getDefault();
1425 if (Ctor != useDefaultRegisterAllocator)
1426 return Ctor();
1427
1428 // With no -regalloc= override, ask the target for a regalloc pass.
1429 return createTargetRegisterAllocator(Optimized);
1430}
1431
1432bool TargetPassConfig::isCustomizedRegAlloc() {
1433 return RegAlloc !=
1434 (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator;
1435}
1436
1437bool TargetPassConfig::addRegAssignAndRewriteFast() {
1438 if (RegAlloc != (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator &&
1439 RegAlloc != (RegisterRegAlloc::FunctionPassCtor)&createFastRegisterAllocator)
1440 reportFatalUsageError(
1441 reason: "Must use fast (default) register allocator for unoptimized regalloc.");
1442
1443 addPass(P: createRegAllocPass(Optimized: false));
1444
1445 // Allow targets to change the register assignments after
1446 // fast register allocation.
1447 addPostFastRegAllocRewrite();
1448 return true;
1449}
1450
1451bool TargetPassConfig::addRegAssignAndRewriteOptimized() {
1452 // Add the selected register allocation pass.
1453 addPass(P: createRegAllocPass(Optimized: true));
1454
1455 // Allow targets to change the register assignments before rewriting.
1456 addPreRewrite();
1457
1458 // Finally rewrite virtual registers.
1459 addPass(PassID: &VirtRegRewriterID);
1460
1461 // Regalloc scoring for ML-driven eviction - noop except when learning a new
1462 // eviction policy.
1463 addPass(P: createRegAllocScoringPass());
1464 return true;
1465}
1466
1467/// Return true if the default global register allocator is in use and
1468/// has not be overriden on the command line with '-regalloc=...'
1469bool TargetPassConfig::usingDefaultRegAlloc() const {
1470 return RegAlloc.getNumOccurrences() == 0;
1471}
1472
1473/// Add the minimum set of target-independent passes that are required for
1474/// register allocation. No coalescing or scheduling.
1475void TargetPassConfig::addFastRegAlloc() {
1476 addPass(PassID: &PHIEliminationID);
1477 addPass(PassID: &TwoAddressInstructionPassID);
1478
1479 addRegAssignAndRewriteFast();
1480}
1481
1482/// Add standard target-independent passes that are tightly coupled with
1483/// optimized register allocation, including coalescing, machine instruction
1484/// scheduling, and register allocation itself.
1485void TargetPassConfig::addOptimizedRegAlloc() {
1486 addPass(PassID: &DetectDeadLanesID);
1487
1488 addPass(PassID: &InitUndefID);
1489
1490 addPass(PassID: &ProcessImplicitDefsID);
1491
1492 // LiveVariables currently requires pure SSA form.
1493 //
1494 // FIXME: Once TwoAddressInstruction pass no longer uses kill flags,
1495 // LiveVariables can be removed completely, and LiveIntervals can be directly
1496 // computed. (We still either need to regenerate kill flags after regalloc, or
1497 // preferably fix the scavenger to not depend on them).
1498 // FIXME: UnreachableMachineBlockElim is a dependant pass of LiveVariables.
1499 // When LiveVariables is removed this has to be removed/moved either.
1500 // Explicit addition of UnreachableMachineBlockElim allows stopping before or
1501 // after it with -stop-before/-stop-after.
1502 addPass(PassID: &UnreachableMachineBlockElimID);
1503 addPass(PassID: &LiveVariablesID);
1504
1505 // Edge splitting is smarter with machine loop info.
1506 addPass(PassID: &MachineLoopInfoID);
1507 addPass(PassID: &PHIEliminationID);
1508
1509 // Eventually, we want to run LiveIntervals before PHI elimination.
1510 if (EarlyLiveIntervals)
1511 addPass(PassID: &LiveIntervalsID);
1512
1513 addPass(PassID: &TwoAddressInstructionPassID);
1514 addPass(PassID: &RegisterCoalescerID);
1515
1516 // The machine scheduler may accidentally create disconnected components
1517 // when moving subregister definitions around, avoid this by splitting them to
1518 // separate vregs before. Splitting can also improve reg. allocation quality.
1519 addPass(PassID: &RenameIndependentSubregsID);
1520
1521 // PreRA instruction scheduling.
1522 addPass(PassID: &MachineSchedulerID);
1523
1524 if (addRegAssignAndRewriteOptimized()) {
1525 // Perform stack slot coloring and post-ra machine LICM.
1526 addPass(PassID: &StackSlotColoringID);
1527
1528 // Allow targets to expand pseudo instructions depending on the choice of
1529 // registers before MachineCopyPropagation.
1530 addPostRewrite();
1531
1532 // Copy propagate to forward register uses and try to eliminate COPYs that
1533 // were not coalesced.
1534 addPass(PassID: &MachineCopyPropagationID);
1535
1536 // Run post-ra machine LICM to hoist reloads / remats.
1537 //
1538 // FIXME: can this move into MachineLateOptimization?
1539 addPass(PassID: &MachineLICMID);
1540 }
1541}
1542
1543//===---------------------------------------------------------------------===//
1544/// Post RegAlloc Pass Configuration
1545//===---------------------------------------------------------------------===//
1546
1547/// Add passes that optimize machine instructions after register allocation.
1548void TargetPassConfig::addMachineLateOptimization() {
1549 // Cleanup of redundant immediate/address loads.
1550 addPass(PassID: &MachineLateInstrsCleanupID);
1551
1552 // Branch folding must be run after regalloc and prolog/epilog insertion.
1553 addPass(PassID: &BranchFolderPassID);
1554
1555 // Tail duplication.
1556 // Note that duplicating tail just increases code size and degrades
1557 // performance for targets that require Structured Control Flow.
1558 // In addition it can also make CFG irreducible. Thus we disable it.
1559 if (!TM->requiresStructuredCFG())
1560 addPass(PassID: &TailDuplicateLegacyID);
1561
1562 // Copy propagation.
1563 addPass(PassID: &MachineCopyPropagationID);
1564}
1565
1566/// Add standard GC passes.
1567bool TargetPassConfig::addGCPasses() {
1568 addPass(PassID: &GCMachineCodeAnalysisID);
1569 return true;
1570}
1571
1572/// Add standard basic block placement passes.
1573void TargetPassConfig::addBlockPlacement() {
1574 if (EnableFSDiscriminator) {
1575 addPass(P: createMIRAddFSDiscriminatorsPass(
1576 P: sampleprof::FSDiscriminatorPass::Pass2));
1577 const std::string ProfileFile = getFSProfileFile(TM);
1578 if (!ProfileFile.empty() && !DisableLayoutFSProfileLoader)
1579 addPass(P: createMIRProfileLoaderPass(File: ProfileFile, RemappingFile: getFSRemappingFile(TM),
1580 P: sampleprof::FSDiscriminatorPass::Pass2,
1581 FS: nullptr));
1582 }
1583 if (addPass(PassID: &MachineBlockPlacementID)) {
1584 // Run a separate pass to collect block placement statistics.
1585 if (EnableBlockPlacementStats)
1586 addPass(PassID: &MachineBlockPlacementStatsID);
1587 }
1588}
1589
1590//===---------------------------------------------------------------------===//
1591/// GlobalISel Configuration
1592//===---------------------------------------------------------------------===//
1593bool TargetPassConfig::isGlobalISelAbortEnabled() const {
1594 return TM->Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
1595}
1596
1597bool TargetPassConfig::reportDiagnosticWhenGlobalISelFallback() const {
1598 return TM->Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag;
1599}
1600
1601bool TargetPassConfig::isGISelCSEEnabled() const {
1602 return true;
1603}
1604
1605std::unique_ptr<CSEConfigBase> TargetPassConfig::getCSEConfig() const {
1606 return std::make_unique<CSEConfigBase>();
1607}
1608