1//===- TargetPassConfig.cpp - Target independent code generation passes ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines interfaces to access the target independent code
10// generation passes provided by the LLVM backend.
11//
12//===---------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/TargetPassConfig.h"
15#include "llvm/ADT/DenseMap.h"
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/Analysis/BasicAliasAnalysis.h"
19#include "llvm/Analysis/CallGraphSCCPass.h"
20#include "llvm/Analysis/ScopedNoAliasAA.h"
21#include "llvm/Analysis/TargetTransformInfo.h"
22#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
23#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
24#include "llvm/CodeGen/CSEConfigBase.h"
25#include "llvm/CodeGen/CodeGenTargetMachineImpl.h"
26#include "llvm/CodeGen/MachineFunctionPass.h"
27#include "llvm/CodeGen/MachinePassRegistry.h"
28#include "llvm/CodeGen/Passes.h"
29#include "llvm/CodeGen/RegAllocRegistry.h"
30#include "llvm/IR/IRPrintingPasses.h"
31#include "llvm/IR/LegacyPassManager.h"
32#include "llvm/IR/PassInstrumentation.h"
33#include "llvm/IR/Verifier.h"
34#include "llvm/InitializePasses.h"
35#include "llvm/MC/MCAsmInfo.h"
36#include "llvm/MC/MCTargetOptions.h"
37#include "llvm/Pass.h"
38#include "llvm/Support/CodeGen.h"
39#include "llvm/Support/CommandLine.h"
40#include "llvm/Support/Compiler.h"
41#include "llvm/Support/Debug.h"
42#include "llvm/Support/Discriminator.h"
43#include "llvm/Support/ErrorHandling.h"
44#include "llvm/Support/SaveAndRestore.h"
45#include "llvm/Support/Threading.h"
46#include "llvm/Support/VirtualFileSystem.h"
47#include "llvm/Support/WithColor.h"
48#include "llvm/Target/CGPassBuilderOption.h"
49#include "llvm/Target/TargetMachine.h"
50#include "llvm/Transforms/ObjCARC.h"
51#include "llvm/Transforms/Scalar.h"
52#include "llvm/Transforms/Utils.h"
53#include "llvm/Transforms/Utils/TriggerCrashPass.h"
54#include <cassert>
55#include <optional>
56#include <string>
57
58using namespace llvm;
59
60static cl::opt<bool>
61 EnableIPRA("enable-ipra", cl::init(Val: false), cl::Hidden,
62 cl::desc("Enable interprocedural register allocation "
63 "to reduce load/store at procedure calls."));
64static cl::opt<bool> DisablePostRASched("disable-post-ra", cl::Hidden,
65 cl::desc("Disable Post Regalloc Scheduler"));
66static cl::opt<bool> DisableBranchFold("disable-branch-fold", cl::Hidden,
67 cl::desc("Disable branch folding"));
68static cl::opt<bool> DisableTailDuplicate("disable-tail-duplicate", cl::Hidden,
69 cl::desc("Disable tail duplication"));
70static cl::opt<bool> DisableEarlyTailDup("disable-early-taildup", cl::Hidden,
71 cl::desc("Disable pre-register allocation tail duplication"));
72static cl::opt<bool> DisableBlockPlacement("disable-block-placement",
73 cl::Hidden, cl::desc("Disable probability-driven block placement"));
74static cl::opt<bool> EnableBlockPlacementStats("enable-block-placement-stats",
75 cl::Hidden, cl::desc("Collect probability-driven block placement stats"));
76static cl::opt<bool> DisableSSC("disable-ssc", cl::Hidden,
77 cl::desc("Disable Stack Slot Coloring"));
78static cl::opt<bool> DisableMachineDCE("disable-machine-dce", cl::Hidden,
79 cl::desc("Disable Machine Dead Code Elimination"));
80static cl::opt<bool> DisableEarlyIfConversion("disable-early-ifcvt", cl::Hidden,
81 cl::desc("Disable Early If-conversion"));
82static cl::opt<bool> DisableMachineLICM("disable-machine-licm", cl::Hidden,
83 cl::desc("Disable Machine LICM"));
84static cl::opt<bool> DisableMachineCSE("disable-machine-cse", cl::Hidden,
85 cl::desc("Disable Machine Common Subexpression Elimination"));
86static cl::opt<bool> DisablePostRAMachineLICM("disable-postra-machine-licm",
87 cl::Hidden,
88 cl::desc("Disable Machine LICM"));
89static cl::opt<bool> DisableMachineSink("disable-machine-sink", cl::Hidden,
90 cl::desc("Disable Machine Sinking"));
91static cl::opt<bool> DisablePostRAMachineSink("disable-postra-machine-sink",
92 cl::Hidden,
93 cl::desc("Disable PostRA Machine Sinking"));
94static cl::opt<bool> DisableLSR("disable-lsr", cl::Hidden,
95 cl::desc("Disable Loop Strength Reduction Pass"));
96static cl::opt<bool> DisableConstantHoisting("disable-constant-hoisting",
97 cl::Hidden, cl::desc("Disable ConstantHoisting"));
98static cl::opt<bool> DisableCGP("disable-cgp", cl::Hidden,
99 cl::desc("Disable Codegen Prepare"));
100
101static cl::opt<bool>
102 TriggerCrash("codegen-pipeline-trigger-crash", cl::init(Val: false), cl::Hidden,
103 cl::desc("Trigger crash in codegen pipeline"));
104
105static cl::opt<bool> DisableCopyProp("disable-copyprop", cl::Hidden,
106 cl::desc("Disable Copy Propagation pass"));
107static cl::opt<bool> DisablePartialLibcallInlining("disable-partial-libcall-inlining",
108 cl::Hidden, cl::desc("Disable Partial Libcall Inlining"));
109static cl::opt<bool> DisableAtExitBasedGlobalDtorLowering(
110 "disable-atexit-based-global-dtor-lowering", cl::Hidden,
111 cl::desc("For MachO, disable atexit()-based global destructor lowering"));
112static cl::opt<bool> EnableImplicitNullChecks(
113 "enable-implicit-null-checks",
114 cl::desc("Fold null checks into faulting memory operations"),
115 cl::init(Val: false), cl::Hidden);
116static cl::opt<bool>
117 PrintISelInput("print-isel-input", cl::Hidden,
118 cl::desc("Print LLVM IR input to isel pass"));
119cl::opt<bool>
120 PrintRegUsage("print-regusage", cl::Hidden,
121 cl::desc("Print register usage details collected for IPRA"));
122static cl::opt<cl::boolOrDefault>
123 VerifyMachineCode("verify-machineinstrs", cl::Hidden,
124 cl::desc("Verify generated machine code"));
125static cl::opt<cl::boolOrDefault>
126 DebugifyAndStripAll("debugify-and-strip-all-safe", cl::Hidden,
127 cl::desc("Debugify MIR before and Strip debug after "
128 "each pass except those known to be unsafe "
129 "when debug info is present"));
130static cl::opt<cl::boolOrDefault> DebugifyCheckAndStripAll(
131 "debugify-check-and-strip-all-safe", cl::Hidden,
132 cl::desc(
133 "Debugify MIR before, by checking and stripping the debug info after, "
134 "each pass except those known to be unsafe when debug info is "
135 "present"));
136// Enable or disable the MachineOutliner.
137static cl::opt<RunOutliner> EnableMachineOutliner(
138 "enable-machine-outliner", cl::desc("Enable the machine outliner"),
139 cl::Hidden, cl::ValueOptional, cl::init(Val: RunOutliner::TargetDefault),
140 cl::values(
141 clEnumValN(RunOutliner::AlwaysOutline, "always",
142 "Run on all functions guaranteed to be beneficial"),
143 clEnumValN(RunOutliner::OptimisticPGO, "optimistic-pgo",
144 "Outline cold code only. If a code block does not have "
145 "profile data, optimistically assume it is cold."),
146 clEnumValN(RunOutliner::ConservativePGO, "conservative-pgo",
147 "Outline cold code only. If a code block does not have "
148 "profile, data, conservatively assume it is hot."),
149 clEnumValN(RunOutliner::NeverOutline, "never", "Disable all outlining"),
150 // Sentinel value for unspecified option.
151 clEnumValN(RunOutliner::AlwaysOutline, "", "")));
152static cl::opt<bool> EnableGlobalMergeFunc(
153 "enable-global-merge-func", cl::Hidden,
154 cl::desc("Enable global merge functions that are based on hash function"));
155// Disable the pass to fix unwind information. Whether the pass is included in
156// the pipeline is controlled via the target options, this option serves as
157// manual override.
158static cl::opt<bool> DisableCFIFixup("disable-cfi-fixup", cl::Hidden,
159 cl::desc("Disable the CFI fixup pass"));
160// Enable or disable FastISel. Both options are needed, because
161// FastISel is enabled by default with -fast, and we wish to be
162// able to enable or disable fast-isel independently from -O0.
163static cl::opt<cl::boolOrDefault>
164EnableFastISelOption("fast-isel", cl::Hidden,
165 cl::desc("Enable the \"fast\" instruction selector"));
166
167static cl::opt<cl::boolOrDefault> EnableGlobalISelOption(
168 "global-isel", cl::Hidden,
169 cl::desc("Enable the \"global\" instruction selector"));
170
171// FIXME: remove this after switching to NPM or GlobalISel, whichever gets there
172// first...
173static cl::opt<bool>
174 PrintAfterISel("print-after-isel", cl::init(Val: false), cl::Hidden,
175 cl::desc("Print machine instrs after ISel"));
176
177static cl::opt<GlobalISelAbortMode> EnableGlobalISelAbort(
178 "global-isel-abort", cl::Hidden,
179 cl::desc("Enable abort calls when \"global\" instruction selection "
180 "fails to lower/select an instruction"),
181 cl::values(
182 clEnumValN(GlobalISelAbortMode::Disable, "0", "Disable the abort"),
183 clEnumValN(GlobalISelAbortMode::Enable, "1", "Enable the abort"),
184 clEnumValN(GlobalISelAbortMode::DisableWithDiag, "2",
185 "Disable the abort but emit a diagnostic on failure")));
186
187// Disable MIRProfileLoader before RegAlloc. This is for for debugging and
188// tuning purpose.
189static cl::opt<bool> DisableRAFSProfileLoader(
190 "disable-ra-fsprofile-loader", cl::init(Val: false), cl::Hidden,
191 cl::desc("Disable MIRProfileLoader before RegAlloc"));
192// Disable MIRProfileLoader before BloackPlacement. This is for for debugging
193// and tuning purpose.
194static cl::opt<bool> DisableLayoutFSProfileLoader(
195 "disable-layout-fsprofile-loader", cl::init(Val: false), cl::Hidden,
196 cl::desc("Disable MIRProfileLoader before BlockPlacement"));
197// Specify FSProfile file name.
198static cl::opt<std::string>
199 FSProfileFile("fs-profile-file", cl::init(Val: ""), cl::value_desc("filename"),
200 cl::desc("Flow Sensitive profile file name."), cl::Hidden);
201// Specify Remapping file for FSProfile.
202static cl::opt<std::string> FSRemappingFile(
203 "fs-remapping-file", cl::init(Val: ""), cl::value_desc("filename"),
204 cl::desc("Flow Sensitive profile remapping file name."), cl::Hidden);
205
206// Temporary option to allow experimenting with MachineScheduler as a post-RA
207// scheduler. Targets can "properly" enable this with
208// substitutePass(&PostRASchedulerID, &PostMachineSchedulerID).
209// Targets can return true in targetSchedulesPostRAScheduling() and
210// insert a PostRA scheduling pass wherever it wants.
211static cl::opt<bool> MISchedPostRA(
212 "misched-postra", cl::Hidden,
213 cl::desc(
214 "Run MachineScheduler post regalloc (independent of preRA sched)"));
215
216// Experimental option to run live interval analysis early.
217static cl::opt<bool> EarlyLiveIntervals("early-live-intervals", cl::Hidden,
218 cl::desc("Run live interval analysis earlier in the pipeline"));
219
220static cl::opt<bool> DisableReplaceWithVecLib(
221 "disable-replace-with-vec-lib", cl::Hidden,
222 cl::desc("Disable replace with vector math call pass"));
223
224/// Option names for limiting the codegen pipeline.
225/// Those are used in error reporting and we didn't want
226/// to duplicate their names all over the place.
227static const char StartAfterOptName[] = "start-after";
228static const char StartBeforeOptName[] = "start-before";
229static const char StopAfterOptName[] = "stop-after";
230static const char StopBeforeOptName[] = "stop-before";
231
232static cl::opt<std::string>
233 StartAfterOpt(StringRef(StartAfterOptName),
234 cl::desc("Resume compilation after a specific pass"),
235 cl::value_desc("pass-name"), cl::init(Val: ""), cl::Hidden);
236
237static cl::opt<std::string>
238 StartBeforeOpt(StringRef(StartBeforeOptName),
239 cl::desc("Resume compilation before a specific pass"),
240 cl::value_desc("pass-name"), cl::init(Val: ""), cl::Hidden);
241
242static cl::opt<std::string>
243 StopAfterOpt(StringRef(StopAfterOptName),
244 cl::desc("Stop compilation after a specific pass"),
245 cl::value_desc("pass-name"), cl::init(Val: ""), cl::Hidden);
246
247static cl::opt<std::string>
248 StopBeforeOpt(StringRef(StopBeforeOptName),
249 cl::desc("Stop compilation before a specific pass"),
250 cl::value_desc("pass-name"), cl::init(Val: ""), cl::Hidden);
251
252/// Enable the machine function splitter pass.
253static cl::opt<bool> EnableMachineFunctionSplitter(
254 "enable-split-machine-functions", cl::Hidden,
255 cl::desc("Split out cold blocks from machine functions based on profile "
256 "information."));
257
258/// Disable the expand reductions pass for testing.
259static cl::opt<bool> DisableExpandReductions(
260 "disable-expand-reductions", cl::init(Val: false), cl::Hidden,
261 cl::desc("Disable the expand reduction intrinsics pass from running"));
262
263/// Disable the select optimization pass.
264static cl::opt<bool> DisableSelectOptimize(
265 "disable-select-optimize", cl::init(Val: true), cl::Hidden,
266 cl::desc("Disable the select-optimization pass from running"));
267
268/// Enable garbage-collecting empty basic blocks.
269static cl::opt<bool> EnableGCEmptyBlocks(
270 "enable-gc-empty-basic-blocks", cl::init(Val: false), cl::Hidden,
271 cl::desc("Enable garbage-collecting empty basic blocks"));
272
273static cl::opt<bool>
274 SplitStaticData("split-static-data", cl::Hidden, cl::init(Val: false),
275 cl::desc("Split static data sections into hot and cold "
276 "sections using profile information"));
277
278/// Enable matching and inference when using propeller.
279static cl::opt<bool> BasicBlockSectionMatchInfer(
280 "basic-block-section-match-infer",
281 cl::desc(
282 "Enable matching and inference when generating basic block sections"),
283 cl::init(Val: false), cl::Optional);
284
285cl::opt<bool> EmitBBHash(
286 "emit-bb-hash",
287 cl::desc(
288 "Emit the hash of basic block in the SHT_LLVM_BB_ADDR_MAP section."),
289 cl::init(Val: false), cl::Optional);
290
291/// Allow standard passes to be disabled by command line options. This supports
292/// simple binary flags that either suppress the pass or do nothing.
293/// i.e. -disable-mypass=false has no effect.
294/// These should be converted to boolOrDefault in order to use applyOverride.
295static IdentifyingPassPtr applyDisable(IdentifyingPassPtr PassID,
296 bool Override) {
297 if (Override)
298 return IdentifyingPassPtr();
299 return PassID;
300}
301
302/// Allow standard passes to be disabled by the command line, regardless of who
303/// is adding the pass.
304///
305/// StandardID is the pass identified in the standard pass pipeline and provided
306/// to addPass(). It may be a target-specific ID in the case that the target
307/// directly adds its own pass, but in that case we harmlessly fall through.
308///
309/// TargetID is the pass that the target has configured to override StandardID.
310///
311/// StandardID may be a pseudo ID. In that case TargetID is the name of the real
312/// pass to run. This allows multiple options to control a single pass depending
313/// on where in the pipeline that pass is added.
314static IdentifyingPassPtr overridePass(AnalysisID StandardID,
315 IdentifyingPassPtr TargetID) {
316 if (StandardID == &PostRASchedulerID)
317 return applyDisable(PassID: TargetID, Override: DisablePostRASched);
318
319 if (StandardID == &BranchFolderPassID)
320 return applyDisable(PassID: TargetID, Override: DisableBranchFold);
321
322 if (StandardID == &TailDuplicateLegacyID)
323 return applyDisable(PassID: TargetID, Override: DisableTailDuplicate);
324
325 if (StandardID == &EarlyTailDuplicateLegacyID)
326 return applyDisable(PassID: TargetID, Override: DisableEarlyTailDup);
327
328 if (StandardID == &MachineBlockPlacementID)
329 return applyDisable(PassID: TargetID, Override: DisableBlockPlacement);
330
331 if (StandardID == &StackSlotColoringID)
332 return applyDisable(PassID: TargetID, Override: DisableSSC);
333
334 if (StandardID == &DeadMachineInstructionElimID)
335 return applyDisable(PassID: TargetID, Override: DisableMachineDCE);
336
337 if (StandardID == &EarlyIfConverterLegacyID)
338 return applyDisable(PassID: TargetID, Override: DisableEarlyIfConversion);
339
340 if (StandardID == &EarlyMachineLICMID)
341 return applyDisable(PassID: TargetID, Override: DisableMachineLICM);
342
343 if (StandardID == &MachineCSELegacyID)
344 return applyDisable(PassID: TargetID, Override: DisableMachineCSE);
345
346 if (StandardID == &MachineLICMID)
347 return applyDisable(PassID: TargetID, Override: DisablePostRAMachineLICM);
348
349 if (StandardID == &MachineSinkingLegacyID)
350 return applyDisable(PassID: TargetID, Override: DisableMachineSink);
351
352 if (StandardID == &PostRAMachineSinkingID)
353 return applyDisable(PassID: TargetID, Override: DisablePostRAMachineSink);
354
355 if (StandardID == &MachineCopyPropagationID)
356 return applyDisable(PassID: TargetID, Override: DisableCopyProp);
357
358 return TargetID;
359}
360
361// Find the FSProfile file name. The internal option takes the precedence
362// before getting from TargetMachine.
363static std::string getFSProfileFile(const TargetMachine *TM) {
364 if (!FSProfileFile.empty())
365 return FSProfileFile.getValue();
366 const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
367 if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
368 return std::string();
369 return PGOOpt->ProfileFile;
370}
371
372// Find the Profile remapping file name. The internal option takes the
373// precedence before getting from TargetMachine.
374static std::string getFSRemappingFile(const TargetMachine *TM) {
375 if (!FSRemappingFile.empty())
376 return FSRemappingFile.getValue();
377 const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
378 if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
379 return std::string();
380 return PGOOpt->ProfileRemappingFile;
381}
382
383//===---------------------------------------------------------------------===//
384/// TargetPassConfig
385//===---------------------------------------------------------------------===//
386
387INITIALIZE_PASS(TargetPassConfig, "targetpassconfig",
388 "Target Pass Configuration", false, false)
389char TargetPassConfig::ID = 0;
390
391namespace {
392
393struct InsertedPass {
394 AnalysisID TargetPassID;
395 IdentifyingPassPtr InsertedPassID;
396
397 InsertedPass(AnalysisID TargetPassID, IdentifyingPassPtr InsertedPassID)
398 : TargetPassID(TargetPassID), InsertedPassID(InsertedPassID) {}
399
400 Pass *getInsertedPass() const {
401 assert(InsertedPassID.isValid() && "Illegal Pass ID!");
402 if (InsertedPassID.isInstance())
403 return InsertedPassID.getInstance();
404 Pass *NP = Pass::createPass(ID: InsertedPassID.getID());
405 assert(NP && "Pass ID not registered");
406 return NP;
407 }
408};
409
410} // end anonymous namespace
411
412namespace llvm {
413
414class PassConfigImpl {
415public:
416 // List of passes explicitly substituted by this target. Normally this is
417 // empty, but it is a convenient way to suppress or replace specific passes
418 // that are part of a standard pass pipeline without overridding the entire
419 // pipeline. This mechanism allows target options to inherit a standard pass's
420 // user interface. For example, a target may disable a standard pass by
421 // default by substituting a pass ID of zero, and the user may still enable
422 // that standard pass with an explicit command line option.
423 DenseMap<AnalysisID,IdentifyingPassPtr> TargetPasses;
424
425 /// Store the pairs of <AnalysisID, AnalysisID> of which the second pass
426 /// is inserted after each instance of the first one.
427 SmallVector<InsertedPass, 4> InsertedPasses;
428};
429
430} // end namespace llvm
431
432// Out of line virtual method.
433TargetPassConfig::~TargetPassConfig() {
434 delete Impl;
435}
436
437static const PassInfo *getPassInfo(StringRef PassName) {
438 if (PassName.empty())
439 return nullptr;
440
441 const PassRegistry &PR = *PassRegistry::getPassRegistry();
442 const PassInfo *PI = PR.getPassInfo(Arg: PassName);
443 if (!PI)
444 reportFatalUsageError(reason: Twine('\"') + Twine(PassName) +
445 Twine("\" pass is not registered."));
446 return PI;
447}
448
449static AnalysisID getPassIDFromName(StringRef PassName) {
450 const PassInfo *PI = getPassInfo(PassName);
451 return PI ? PI->getTypeInfo() : nullptr;
452}
453
454static std::pair<StringRef, unsigned>
455getPassNameAndInstanceNum(StringRef PassName) {
456 StringRef Name, InstanceNumStr;
457 std::tie(args&: Name, args&: InstanceNumStr) = PassName.split(Separator: ',');
458
459 unsigned InstanceNum = 0;
460 if (!InstanceNumStr.empty() && InstanceNumStr.getAsInteger(Radix: 10, Result&: InstanceNum))
461 reportFatalUsageError(reason: "invalid pass instance specifier " + PassName);
462
463 return std::make_pair(x&: Name, y&: InstanceNum);
464}
465
466void TargetPassConfig::setStartStopPasses() {
467 StringRef StartBeforeName;
468 std::tie(args&: StartBeforeName, args&: StartBeforeInstanceNum) =
469 getPassNameAndInstanceNum(PassName: StartBeforeOpt);
470
471 StringRef StartAfterName;
472 std::tie(args&: StartAfterName, args&: StartAfterInstanceNum) =
473 getPassNameAndInstanceNum(PassName: StartAfterOpt);
474
475 StringRef StopBeforeName;
476 std::tie(args&: StopBeforeName, args&: StopBeforeInstanceNum)
477 = getPassNameAndInstanceNum(PassName: StopBeforeOpt);
478
479 StringRef StopAfterName;
480 std::tie(args&: StopAfterName, args&: StopAfterInstanceNum)
481 = getPassNameAndInstanceNum(PassName: StopAfterOpt);
482
483 StartBefore = getPassIDFromName(PassName: StartBeforeName);
484 StartAfter = getPassIDFromName(PassName: StartAfterName);
485 StopBefore = getPassIDFromName(PassName: StopBeforeName);
486 StopAfter = getPassIDFromName(PassName: StopAfterName);
487 if (StartBefore && StartAfter)
488 reportFatalUsageError(reason: Twine(StartBeforeOptName) + Twine(" and ") +
489 Twine(StartAfterOptName) + Twine(" specified!"));
490 if (StopBefore && StopAfter)
491 reportFatalUsageError(reason: Twine(StopBeforeOptName) + Twine(" and ") +
492 Twine(StopAfterOptName) + Twine(" specified!"));
493 Started = (StartAfter == nullptr) && (StartBefore == nullptr);
494}
495
496CGPassBuilderOption llvm::getCGPassBuilderOption() {
497 CGPassBuilderOption Opt;
498
499#define SET_OPTION_IF_PRESENT(Option) \
500 if (Option.getNumOccurrences()) \
501 Opt.Option = Option;
502
503 SET_OPTION_IF_PRESENT(EnableGlobalISelAbort)
504 SET_OPTION_IF_PRESENT(EnableIPRA)
505
506#define SET_OPTION(Option) Opt.Option = Option;
507
508 SET_OPTION(EnableFastISelOption)
509 SET_OPTION(EnableGlobalISelOption)
510 SET_OPTION(VerifyMachineCode)
511 SET_OPTION(DisableAtExitBasedGlobalDtorLowering)
512 SET_OPTION(DisableExpandReductions)
513 SET_OPTION(PrintAfterISel)
514 SET_OPTION(FSProfileFile)
515 SET_OPTION(EnableGCEmptyBlocks)
516 SET_OPTION(EarlyLiveIntervals)
517 SET_OPTION(EnableBlockPlacementStats)
518 SET_OPTION(EnableGlobalMergeFunc)
519 SET_OPTION(EnableImplicitNullChecks)
520 SET_OPTION(EnableMachineOutliner)
521 SET_OPTION(MISchedPostRA)
522 SET_OPTION(DisableLSR)
523 SET_OPTION(DisableConstantHoisting)
524 SET_OPTION(DisableCGP)
525 SET_OPTION(DisablePartialLibcallInlining)
526 SET_OPTION(DisableSelectOptimize)
527 SET_OPTION(PrintISelInput)
528 SET_OPTION(PrintRegUsage)
529 SET_OPTION(DebugifyAndStripAll)
530 SET_OPTION(DebugifyCheckAndStripAll)
531 SET_OPTION(DisableRAFSProfileLoader)
532 SET_OPTION(DisableCFIFixup)
533 SET_OPTION(EnableMachineFunctionSplitter)
534
535 return Opt;
536}
537
538void llvm::registerCodeGenCallback(PassInstrumentationCallbacks &PIC,
539 TargetMachine &TM) {
540
541 // Register a callback for disabling passes.
542 PIC.registerShouldRunOptionalPassCallback(C: [](StringRef P, IRUnitRef) {
543
544#define DISABLE_PASS(Option, Name) \
545 if (Option && P.contains(#Name)) \
546 return false;
547 DISABLE_PASS(DisableBlockPlacement, MachineBlockPlacementPass)
548 DISABLE_PASS(DisableBranchFold, BranchFolderPass)
549 DISABLE_PASS(DisableCopyProp, MachineCopyPropagationPass)
550 DISABLE_PASS(DisableEarlyIfConversion, EarlyIfConverterLegacyPass)
551 DISABLE_PASS(DisableEarlyTailDup, EarlyTailDuplicatePass)
552 DISABLE_PASS(DisableMachineCSE, MachineCSELegacyPass)
553 DISABLE_PASS(DisableMachineDCE, DeadMachineInstructionElimPass)
554 DISABLE_PASS(DisableMachineLICM, EarlyMachineLICMPass)
555 DISABLE_PASS(DisableMachineSink, MachineSinkingPass)
556 DISABLE_PASS(DisablePostRAMachineLICM, MachineLICMPass)
557 DISABLE_PASS(DisablePostRAMachineSink, PostRAMachineSinkingPass)
558 DISABLE_PASS(DisablePostRASched, PostRASchedulerPass)
559 DISABLE_PASS(DisableSSC, StackSlotColoringPass)
560 DISABLE_PASS(DisableTailDuplicate, TailDuplicatePass)
561
562 return true;
563 });
564}
565
566Expected<TargetPassConfig::StartStopInfo>
567TargetPassConfig::getStartStopInfo(PassInstrumentationCallbacks &PIC) {
568 auto [StartBefore, StartBeforeInstanceNum] =
569 getPassNameAndInstanceNum(PassName: StartBeforeOpt);
570 auto [StartAfter, StartAfterInstanceNum] =
571 getPassNameAndInstanceNum(PassName: StartAfterOpt);
572 auto [StopBefore, StopBeforeInstanceNum] =
573 getPassNameAndInstanceNum(PassName: StopBeforeOpt);
574 auto [StopAfter, StopAfterInstanceNum] =
575 getPassNameAndInstanceNum(PassName: StopAfterOpt);
576
577 if (!StartBefore.empty() && !StartAfter.empty())
578 return make_error<StringError>(
579 Args: Twine(StartBeforeOptName) + " and " + StartAfterOptName + " specified!",
580 Args: std::make_error_code(e: std::errc::invalid_argument));
581 if (!StopBefore.empty() && !StopAfter.empty())
582 return make_error<StringError>(
583 Args: Twine(StopBeforeOptName) + " and " + StopAfterOptName + " specified!",
584 Args: std::make_error_code(e: std::errc::invalid_argument));
585
586 StartStopInfo Result;
587 Result.StartPass = StartBefore.empty() ? StartAfter : StartBefore;
588 Result.StopPass = StopBefore.empty() ? StopAfter : StopBefore;
589 Result.StartInstanceNum =
590 StartBefore.empty() ? StartAfterInstanceNum : StartBeforeInstanceNum;
591 Result.StopInstanceNum =
592 StopBefore.empty() ? StopAfterInstanceNum : StopBeforeInstanceNum;
593 Result.StartAfter = !StartAfter.empty();
594 Result.StopAfter = !StopAfter.empty();
595 Result.StartInstanceNum += Result.StartInstanceNum == 0;
596 Result.StopInstanceNum += Result.StopInstanceNum == 0;
597 return Result;
598}
599
600// Out of line constructor provides default values for pass options and
601// registers all common codegen passes.
602TargetPassConfig::TargetPassConfig(TargetMachine &TM, PassManagerBase &PM)
603 : ImmutablePass(ID), PM(&PM), TM(&TM) {
604 Impl = new PassConfigImpl();
605
606 PassRegistry &PR = *PassRegistry::getPassRegistry();
607 // Register all target independent codegen passes to activate their PassIDs,
608 // including this pass itself.
609 initializeCodeGen(PR);
610
611 initializeLibcallLoweringInfoWrapperPass(PR);
612
613 // Also register alias analysis passes required by codegen passes.
614 initializeBasicAAWrapperPassPass(PR);
615 initializeAAResultsWrapperPassPass(PR);
616
617 if (EnableIPRA.getNumOccurrences()) {
618 TM.Options.EnableIPRA = EnableIPRA;
619 } else {
620 // If not explicitly specified, use target default.
621 TM.Options.EnableIPRA |= TM.useIPRA();
622 }
623
624 if (TM.Options.EnableIPRA)
625 setRequiresCodeGenSCCOrder();
626
627 if (EnableGlobalISelAbort.getNumOccurrences())
628 TM.Options.GlobalISelAbort = EnableGlobalISelAbort;
629
630 setStartStopPasses();
631}
632
633CodeGenOptLevel TargetPassConfig::getOptLevel() const {
634 return TM->getOptLevel();
635}
636
637/// Insert InsertedPassID pass after TargetPassID.
638void TargetPassConfig::insertPass(AnalysisID TargetPassID,
639 IdentifyingPassPtr InsertedPassID) {
640 assert(((!InsertedPassID.isInstance() &&
641 TargetPassID != InsertedPassID.getID()) ||
642 (InsertedPassID.isInstance() &&
643 TargetPassID != InsertedPassID.getInstance()->getPassID())) &&
644 "Insert a pass after itself!");
645 Impl->InsertedPasses.emplace_back(Args&: TargetPassID, Args&: InsertedPassID);
646}
647
648/// createPassConfig - Create a pass configuration object to be used by
649/// addPassToEmitX methods for generating a pipeline of CodeGen passes.
650///
651/// Targets may override this to extend TargetPassConfig.
652TargetPassConfig *
653CodeGenTargetMachineImpl::createPassConfig(PassManagerBase &PM) {
654 return new TargetPassConfig(*this, PM);
655}
656
657TargetPassConfig::TargetPassConfig()
658 : ImmutablePass(ID) {
659 reportFatalUsageError(reason: "trying to construct TargetPassConfig without a target "
660 "machine. Scheduling a CodeGen pass without a target "
661 "triple set?");
662}
663
664bool TargetPassConfig::willCompleteCodeGenPipeline() {
665 return StopBeforeOpt.empty() && StopAfterOpt.empty();
666}
667
668bool TargetPassConfig::hasLimitedCodeGenPipeline() {
669 return !StartBeforeOpt.empty() || !StartAfterOpt.empty() ||
670 !willCompleteCodeGenPipeline();
671}
672
673std::string TargetPassConfig::getLimitedCodeGenPipelineReason() {
674 if (!hasLimitedCodeGenPipeline())
675 return std::string();
676 std::string Res;
677 static cl::opt<std::string> *PassNames[] = {&StartAfterOpt, &StartBeforeOpt,
678 &StopAfterOpt, &StopBeforeOpt};
679 static const char *OptNames[] = {StartAfterOptName, StartBeforeOptName,
680 StopAfterOptName, StopBeforeOptName};
681 bool IsFirst = true;
682 for (int Idx = 0; Idx < 4; ++Idx)
683 if (!PassNames[Idx]->empty()) {
684 if (!IsFirst)
685 Res += " and ";
686 IsFirst = false;
687 Res += OptNames[Idx];
688 }
689 return Res;
690}
691
692// Helper to verify the analysis is really immutable.
693void TargetPassConfig::setOpt(bool &Opt, bool Val) {
694 assert(!Initialized && "PassConfig is immutable");
695 Opt = Val;
696}
697
698void TargetPassConfig::substitutePass(AnalysisID StandardID,
699 IdentifyingPassPtr TargetID) {
700 Impl->TargetPasses[StandardID] = TargetID;
701}
702
703IdentifyingPassPtr TargetPassConfig::getPassSubstitution(AnalysisID ID) const {
704 DenseMap<AnalysisID, IdentifyingPassPtr>::const_iterator
705 I = Impl->TargetPasses.find(Val: ID);
706 if (I == Impl->TargetPasses.end())
707 return ID;
708 return I->second;
709}
710
711bool TargetPassConfig::isPassSubstitutedOrOverridden(AnalysisID ID) const {
712 IdentifyingPassPtr TargetID = getPassSubstitution(ID);
713 IdentifyingPassPtr FinalPtr = overridePass(StandardID: ID, TargetID);
714 return !FinalPtr.isValid() || FinalPtr.isInstance() ||
715 FinalPtr.getID() != ID;
716}
717
718/// Add a pass to the PassManager if that pass is supposed to be run. If the
719/// Started/Stopped flags indicate either that the compilation should start at
720/// a later pass or that it should stop after an earlier pass, then do not add
721/// the pass. Finally, compare the current pass against the StartAfter
722/// and StopAfter options and change the Started/Stopped flags accordingly.
723void TargetPassConfig::addPass(Pass *P) {
724 assert(!Initialized && "PassConfig is immutable");
725
726 // Cache the Pass ID here in case the pass manager finds this pass is
727 // redundant with ones already scheduled / available, and deletes it.
728 // Fundamentally, once we add the pass to the manager, we no longer own it
729 // and shouldn't reference it.
730 AnalysisID PassID = P->getPassID();
731
732 if (StartBefore == PassID && StartBeforeCount++ == StartBeforeInstanceNum)
733 Started = true;
734 if (StopBefore == PassID && StopBeforeCount++ == StopBeforeInstanceNum)
735 Stopped = true;
736 if (Started && !Stopped) {
737 if (AddingMachinePasses) {
738 // Construct banner message before PM->add() as that may delete the pass.
739 std::string Banner =
740 std::string("After ") + std::string(P->getPassName());
741 addMachinePrePasses();
742 PM->add(P);
743 addMachinePostPasses(Banner);
744 } else {
745 PM->add(P);
746 }
747
748 // Add the passes after the pass P if there is any.
749 for (const auto &IP : Impl->InsertedPasses)
750 if (IP.TargetPassID == PassID)
751 addPass(P: IP.getInsertedPass());
752 } else {
753 delete P;
754 }
755
756 if (StopAfter == PassID && StopAfterCount++ == StopAfterInstanceNum)
757 Stopped = true;
758
759 if (StartAfter == PassID && StartAfterCount++ == StartAfterInstanceNum)
760 Started = true;
761 if (Stopped && !Started)
762 reportFatalUsageError(reason: "Cannot stop compilation after pass that is not run");
763}
764
765/// Add a CodeGen pass at this point in the pipeline after checking for target
766/// and command line overrides.
767///
768/// addPass cannot return a pointer to the pass instance because is internal the
769/// PassManager and the instance we create here may already be freed.
770AnalysisID TargetPassConfig::addPass(AnalysisID PassID) {
771 IdentifyingPassPtr TargetID = getPassSubstitution(ID: PassID);
772 IdentifyingPassPtr FinalPtr = overridePass(StandardID: PassID, TargetID);
773 if (!FinalPtr.isValid())
774 return nullptr;
775
776 Pass *P;
777 if (FinalPtr.isInstance())
778 P = FinalPtr.getInstance();
779 else {
780 P = Pass::createPass(ID: FinalPtr.getID());
781 if (!P)
782 llvm_unreachable("Pass ID not registered");
783 }
784 AnalysisID FinalID = P->getPassID();
785 addPass(P); // Ends the lifetime of P.
786
787 return FinalID;
788}
789
790void TargetPassConfig::printAndVerify(const std::string &Banner) {
791 addPrintPass(Banner);
792 addVerifyPass(Banner);
793}
794
795void TargetPassConfig::addPrintPass(const std::string &Banner) {
796 if (PrintAfterISel)
797 PM->add(P: createMachineFunctionPrinterPass(OS&: dbgs(), Banner));
798}
799
800void TargetPassConfig::addVerifyPass(const std::string &Banner) {
801 bool Verify = VerifyMachineCode == cl::boolOrDefault::BOU_TRUE;
802#ifdef EXPENSIVE_CHECKS
803 if (VerifyMachineCode == cl::boolOrDefault::BOU_UNSET)
804 Verify = TM->isMachineVerifierClean();
805#endif
806 if (Verify)
807 PM->add(P: createMachineVerifierPass(Banner));
808}
809
810void TargetPassConfig::addDebugifyPass() {
811 PM->add(P: createDebugifyMachineModulePass());
812}
813
814void TargetPassConfig::addStripDebugPass() {
815 PM->add(P: createStripDebugMachineModuleLegacyPass(/*OnlyDebugified=*/true));
816}
817
818void TargetPassConfig::addCheckDebugPass() {
819 PM->add(P: createCheckDebugMachineModuleLegacyPass());
820}
821
822void TargetPassConfig::addMachinePrePasses(bool AllowDebugify) {
823 if (AllowDebugify && DebugifyIsSafe &&
824 (DebugifyAndStripAll == cl::boolOrDefault::BOU_TRUE ||
825 DebugifyCheckAndStripAll == cl::boolOrDefault::BOU_TRUE))
826 addDebugifyPass();
827}
828
829void TargetPassConfig::addMachinePostPasses(const std::string &Banner) {
830 if (DebugifyIsSafe) {
831 if (DebugifyCheckAndStripAll == cl::boolOrDefault::BOU_TRUE) {
832 addCheckDebugPass();
833 addStripDebugPass();
834 } else if (DebugifyAndStripAll == cl::boolOrDefault::BOU_TRUE)
835 addStripDebugPass();
836 }
837 addVerifyPass(Banner);
838}
839
840/// Add common target configurable passes that perform LLVM IR to IR transforms
841/// following machine independent optimization.
842void TargetPassConfig::addIRPasses() {
843 // Before running any passes, run the verifier to determine if the input
844 // coming from the front-end and/or optimizer is valid.
845 if (!DisableVerify)
846 addPass(P: createVerifierPass());
847
848 if (getOptLevel() != CodeGenOptLevel::None) {
849 // Basic AliasAnalysis support.
850 // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that
851 // BasicAliasAnalysis wins if they disagree. This is intended to help
852 // support "obvious" type-punning idioms.
853 addPass(P: createTypeBasedAAWrapperPass());
854 addPass(P: createScopedNoAliasAAWrapperPass());
855 addPass(P: createBasicAAWrapperPass());
856
857 // Run loop strength reduction before anything else.
858 if (!DisableLSR) {
859 addPass(P: createCanonicalizeFreezeInLoopsPass());
860 addPass(P: createLoopStrengthReducePass());
861 if (EnableLoopTermFold)
862 addPass(P: createLoopTermFoldPass());
863 }
864 }
865
866 // Run GC lowering passes for builtin collectors
867 // TODO: add a pass insertion point here
868 addPass(PassID: &GCLoweringID);
869 addPass(PassID: &ShadowStackGCLoweringID);
870
871 // For MachO, lower @llvm.global_dtors into @llvm.global_ctors with
872 // __cxa_atexit() calls to avoid emitting the deprecated __mod_term_func.
873 if (TM->getTargetTriple().isOSBinFormatMachO() &&
874 !DisableAtExitBasedGlobalDtorLowering)
875 addPass(P: createLowerGlobalDtorsLegacyPass());
876
877 // Make sure that no unreachable blocks are instruction selected.
878 addPass(P: createUnreachableBlockEliminationPass());
879
880 // Prepare expensive constants for SelectionDAG.
881 if (getOptLevel() != CodeGenOptLevel::None && !DisableConstantHoisting)
882 addPass(P: createConstantHoistingPass());
883
884 if (getOptLevel() != CodeGenOptLevel::None && !DisableReplaceWithVecLib)
885 addPass(P: createReplaceWithVeclibLegacyPass());
886
887 if (getOptLevel() != CodeGenOptLevel::None && !DisablePartialLibcallInlining)
888 addPass(P: createPartiallyInlineLibCallsPass());
889
890 // Instrument function entry after all inlining.
891 addPass(P: createPostInlineEntryExitInstrumenterPass());
892
893 // Add scalarization of target's unsupported masked memory intrinsics pass.
894 // the unsupported intrinsic will be replaced with a chain of basic blocks,
895 // that stores/loads element one-by-one if the appropriate mask bit is set.
896 addPass(P: createScalarizeMaskedMemIntrinLegacyPass());
897
898 // Expand reduction intrinsics into shuffle sequences if the target wants to.
899 // Allow disabling it for testing purposes.
900 if (!DisableExpandReductions)
901 addPass(P: createExpandReductionsPass());
902
903 // Convert conditional moves to conditional jumps when profitable.
904 if (getOptLevel() != CodeGenOptLevel::None && !DisableSelectOptimize)
905 addPass(P: createSelectOptimizePass());
906
907 if (EnableGlobalMergeFunc)
908 addPass(P: createGlobalMergeFuncPass());
909
910 if (TM->getTargetTriple().isOSWindows())
911 addPass(P: createWindowsSecureHotPatchingPass());
912}
913
914/// Turn exception handling constructs into something the code generators can
915/// handle.
916void TargetPassConfig::addPassesToHandleExceptions() {
917 const MCAsmInfo &MCAI = TM->getMCAsmInfo();
918 switch (MCAI.getExceptionHandlingType()) {
919 case ExceptionHandling::SjLj:
920 // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both
921 // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise,
922 // catch info can get misplaced when a selector ends up more than one block
923 // removed from the parent invoke(s). This could happen when a landing
924 // pad is shared by multiple invokes and is also a target of a normal
925 // edge from elsewhere.
926 addPass(P: createSjLjEHPreparePass(TM));
927 [[fallthrough]];
928 case ExceptionHandling::DwarfCFI:
929 case ExceptionHandling::ARM:
930 case ExceptionHandling::AIX:
931 case ExceptionHandling::ZOS:
932 addPass(P: createDwarfEHPass(OptLevel: getOptLevel()));
933 break;
934 case ExceptionHandling::WinEH:
935 // We support using both GCC-style and MSVC-style exceptions on Windows, so
936 // add both preparation passes. Each pass will only actually run if it
937 // recognizes the personality function.
938 addPass(P: createWinEHPass());
939 addPass(P: createDwarfEHPass(OptLevel: getOptLevel()));
940 break;
941 case ExceptionHandling::Wasm:
942 // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs
943 // on catchpads and cleanuppads because it does not outline them into
944 // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we
945 // should remove PHIs there.
946 addPass(P: createWinEHPass(/*DemoteCatchSwitchPHIOnly=*/true));
947 addPass(P: createWasmEHPass());
948 break;
949 case ExceptionHandling::None:
950 addPass(P: createLowerInvokePass());
951
952 // The lower invoke pass may create unreachable code. Remove it.
953 addPass(P: createUnreachableBlockEliminationPass());
954 break;
955 }
956}
957
958/// Add pass to prepare the LLVM IR for code generation. This should be done
959/// before exception handling preparation passes.
960void TargetPassConfig::addCodeGenPrepare() {
961 if (getOptLevel() != CodeGenOptLevel::None && !DisableCGP)
962 addPass(P: createCodeGenPrepareLegacyPass());
963}
964
965/// Add common passes that perform LLVM IR to IR transforms in preparation for
966/// instruction selection.
967void TargetPassConfig::addISelPrepare() {
968 addPreISel();
969
970 // Force codegen to run according to the callgraph.
971 if (requiresCodeGenSCCOrder())
972 addPass(P: new DummyCGSCCPass);
973
974 addPass(P: createInlineAsmPreparePass());
975
976 // Add both the safe stack and the stack protection passes: each of them will
977 // only protect functions that have corresponding attributes.
978 addPass(P: createSafeStackPass());
979 addPass(P: createStackProtectorPass());
980
981 if (PrintISelInput)
982 addPass(P: createPrintFunctionPass(
983 OS&: dbgs(), Banner: "\n\n*** Final LLVM Code input to ISel ***\n"));
984
985 // All passes which modify the LLVM IR are now complete; run the verifier
986 // to ensure that the IR is valid.
987 if (!DisableVerify)
988 addPass(P: createVerifierPass());
989}
990
991bool TargetPassConfig::addCoreISelPasses() {
992 // Enable FastISel with -fast-isel, but allow that to be overridden.
993 TM->setO0WantsFastISel(EnableFastISelOption != cl::boolOrDefault::BOU_FALSE);
994
995 // Determine an instruction selector.
996 enum class SelectorType { SelectionDAG, FastISel, GlobalISel };
997 SelectorType Selector;
998
999 if (EnableFastISelOption == cl::boolOrDefault::BOU_TRUE)
1000 Selector = SelectorType::FastISel;
1001 else if (EnableGlobalISelOption == cl::boolOrDefault::BOU_TRUE ||
1002 (TM->Options.EnableGlobalISel &&
1003 EnableGlobalISelOption != cl::boolOrDefault::BOU_FALSE))
1004 Selector = SelectorType::GlobalISel;
1005 else if (TM->getOptLevel() == CodeGenOptLevel::None &&
1006 TM->getO0WantsFastISel())
1007 Selector = SelectorType::FastISel;
1008 else
1009 Selector = SelectorType::SelectionDAG;
1010
1011 // Set consistently TM->Options.EnableFastISel and EnableGlobalISel.
1012 if (Selector == SelectorType::FastISel) {
1013 TM->setFastISel(true);
1014 TM->setGlobalISel(false);
1015 } else if (Selector == SelectorType::GlobalISel) {
1016 TM->setFastISel(false);
1017 TM->setGlobalISel(true);
1018 }
1019
1020 // FIXME: Injecting into the DAGISel pipeline seems to cause issues with
1021 // analyses needing to be re-run. This can result in being unable to
1022 // schedule passes (particularly with 'Function Alias Analysis
1023 // Results'). It's not entirely clear why but AFAICT this seems to be
1024 // due to one FunctionPassManager not being able to use analyses from a
1025 // previous one. As we're injecting a ModulePass we break the usual
1026 // pass manager into two. GlobalISel with the fallback path disabled
1027 // and -run-pass seem to be unaffected. The majority of GlobalISel
1028 // testing uses -run-pass so this probably isn't too bad.
1029 SaveAndRestore SavedDebugifyIsSafe(DebugifyIsSafe);
1030 if (Selector != SelectorType::GlobalISel || !isGlobalISelAbortEnabled())
1031 DebugifyIsSafe = false;
1032
1033 // Add instruction selector passes for global isel if enabled.
1034 if (Selector == SelectorType::GlobalISel) {
1035 SaveAndRestore SavedAddingMachinePasses(AddingMachinePasses, true);
1036 if (addIRTranslator())
1037 return true;
1038
1039 addPreLegalizeMachineIR();
1040
1041 if (addLegalizeMachineIR())
1042 return true;
1043
1044 // Before running the register bank selector, ask the target if it
1045 // wants to run some passes.
1046 addPreRegBankSelect();
1047
1048 if (addRegBankSelect())
1049 return true;
1050
1051 addPreGlobalInstructionSelect();
1052
1053 if (addGlobalInstructionSelect())
1054 return true;
1055 }
1056
1057 // Pass to reset the MachineFunction if the ISel failed. Outside of the above
1058 // if so that the verifier is not added to it.
1059 if (Selector == SelectorType::GlobalISel)
1060 addPass(P: createResetMachineFunctionPass(
1061 EmitFallbackDiag: reportDiagnosticWhenGlobalISelFallback(), AbortOnFailedISel: isGlobalISelAbortEnabled()));
1062
1063 // Run the SDAG InstSelector, providing a fallback path when we do not want to
1064 // abort on not-yet-supported input.
1065 if (Selector != SelectorType::GlobalISel || !isGlobalISelAbortEnabled())
1066 if (addInstSelector())
1067 return true;
1068
1069 // Expand pseudo-instructions emitted by ISel. Don't run the verifier before
1070 // FinalizeISel.
1071 addPass(PassID: &FinalizeISelID);
1072
1073 // Print the instruction selected machine code...
1074 printAndVerify(Banner: "After Instruction Selection");
1075
1076 return false;
1077}
1078
1079bool TargetPassConfig::addISelPasses() {
1080 if (TM->useEmulatedTLS())
1081 addPass(P: createLowerEmuTLSPass());
1082
1083 PM->add(P: createTargetTransformInfoWrapperPass(TIRA: TM->getTargetIRAnalysis()));
1084 // ObjCARCContract operates on ObjC intrinsics and must run before
1085 // PreISelIntrinsicLowering.
1086 if (getOptLevel() != CodeGenOptLevel::None)
1087 addPass(P: createObjCARCContractPass());
1088 addPass(P: createPreISelIntrinsicLoweringPass());
1089 addPass(P: createExpandIRInstsPass(getOptLevel()));
1090 addIRPasses();
1091
1092 if (TriggerCrash)
1093 addPass(P: createTriggerCrashFunctionPass());
1094
1095 addCodeGenPrepare();
1096 addPassesToHandleExceptions();
1097 addISelPrepare();
1098
1099 return addCoreISelPasses();
1100}
1101
1102/// -regalloc=... command line option.
1103static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
1104static cl::opt<RegisterRegAlloc::FunctionPassCtor, false,
1105 RegisterPassParser<RegisterRegAlloc>>
1106 RegAlloc("regalloc", cl::Hidden, cl::init(Val: &useDefaultRegisterAllocator),
1107 cl::desc("Register allocator to use"));
1108
1109/// Add the complete set of target-independent postISel code generator passes.
1110///
1111/// This can be read as the standard order of major LLVM CodeGen stages. Stages
1112/// with nontrivial configuration or multiple passes are broken out below in
1113/// add%Stage routines.
1114///
1115/// Any TargetPassConfig::addXX routine may be overriden by the Target. The
1116/// addPre/Post methods with empty header implementations allow injecting
1117/// target-specific fixups just before or after major stages. Additionally,
1118/// targets have the flexibility to change pass order within a stage by
1119/// overriding default implementation of add%Stage routines below. Each
1120/// technique has maintainability tradeoffs because alternate pass orders are
1121/// not well supported. addPre/Post works better if the target pass is easily
1122/// tied to a common pass. But if it has subtle dependencies on multiple passes,
1123/// the target should override the stage instead.
1124///
1125/// TODO: We could use a single addPre/Post(ID) hook to allow pass injection
1126/// before/after any target-independent pass. But it's currently overkill.
1127void TargetPassConfig::addMachinePasses() {
1128 AddingMachinePasses = true;
1129
1130 // Add passes that optimize machine instructions in SSA form.
1131 if (getOptLevel() != CodeGenOptLevel::None) {
1132 addMachineSSAOptimization();
1133 } else {
1134 // If the target requests it, assign local variables to stack slots relative
1135 // to one another and simplify frame index references where possible.
1136 addPass(PassID: &LocalStackSlotAllocationID);
1137 }
1138
1139 if (TM->Options.EnableIPRA)
1140 addPass(P: createRegUsageInfoPropPass());
1141
1142 // Run pre-ra passes.
1143 addPreRegAlloc();
1144
1145 // Debugifying the register allocator passes seems to provoke some
1146 // non-determinism that affects CodeGen and there doesn't seem to be a point
1147 // where it becomes safe again so stop debugifying here.
1148 DebugifyIsSafe = false;
1149
1150 // Add a FSDiscriminator pass right before RA, so that we could get
1151 // more precise SampleFDO profile for RA.
1152 if (EnableFSDiscriminator) {
1153 addPass(P: createMIRAddFSDiscriminatorsPass(
1154 P: sampleprof::FSDiscriminatorPass::Pass1));
1155 const std::string ProfileFile = getFSProfileFile(TM);
1156 if (!ProfileFile.empty() && !DisableRAFSProfileLoader)
1157 addPass(P: createMIRProfileLoaderPass(File: ProfileFile, RemappingFile: getFSRemappingFile(TM),
1158 P: sampleprof::FSDiscriminatorPass::Pass1,
1159 FS: nullptr));
1160 }
1161
1162 // Run register allocation and passes that are tightly coupled with it,
1163 // including phi elimination and scheduling.
1164 if (getOptimizeRegAlloc())
1165 addOptimizedRegAlloc();
1166 else
1167 addFastRegAlloc();
1168
1169 // Run post-ra passes.
1170 addPostRegAlloc();
1171
1172 addPass(PassID: &RemoveRedundantDebugValuesID);
1173
1174 addPass(PassID: &FixupStatepointCallerSavedID);
1175
1176 // Insert prolog/epilog code. Eliminate abstract frame index references...
1177 if (getOptLevel() != CodeGenOptLevel::None) {
1178 addPass(PassID: &PostRAMachineSinkingID);
1179 addPass(PassID: &ShrinkWrapID);
1180 }
1181
1182 // Prolog/Epilog inserter needs a TargetMachine to instantiate. But only
1183 // do so if it hasn't been disabled, substituted, or overridden.
1184 if (!isPassSubstitutedOrOverridden(ID: &PrologEpilogCodeInserterID))
1185 addPass(P: createPrologEpilogInserterPass());
1186
1187 /// Add passes that optimize machine instructions after register allocation.
1188 if (getOptLevel() != CodeGenOptLevel::None)
1189 addMachineLateOptimization();
1190
1191 // Expand pseudo instructions before second scheduling pass.
1192 addPass(PassID: &ExpandPostRAPseudosID);
1193
1194 // Run pre-sched2 passes.
1195 addPreSched2();
1196
1197 if (EnableImplicitNullChecks)
1198 addPass(PassID: &ImplicitNullChecksID);
1199
1200 // Second pass scheduler.
1201 // Let Target optionally insert this pass by itself at some other
1202 // point.
1203 if (getOptLevel() != CodeGenOptLevel::None &&
1204 !TM->targetSchedulesPostRAScheduling()) {
1205 if (MISchedPostRA)
1206 addPass(PassID: &PostMachineSchedulerID);
1207 else
1208 addPass(PassID: &PostRASchedulerID);
1209 }
1210
1211 // GC
1212 addGCPasses();
1213
1214 // Basic block placement.
1215 if (getOptLevel() != CodeGenOptLevel::None)
1216 addBlockPlacement();
1217
1218 // Insert before XRay Instrumentation.
1219 addPass(PassID: &FEntryInserterID);
1220
1221 addPass(PassID: &XRayInstrumentationID);
1222 addPass(PassID: &PatchableFunctionID);
1223
1224 addPreEmitPass();
1225
1226 if (TM->Options.EnableIPRA)
1227 // Collect register usage information and produce a register mask of
1228 // clobbered registers, to be used to optimize call sites.
1229 addPass(P: createRegUsageInfoCollector());
1230
1231 // FIXME: Some backends are incompatible with running the verifier after
1232 // addPreEmitPass. Maybe only pass "false" here for those targets?
1233 addPass(PassID: &FuncletLayoutID);
1234
1235 addPass(PassID: &RemoveLoadsIntoFakeUsesID);
1236 addPass(PassID: &StackMapLivenessID);
1237 addPass(PassID: &LiveDebugValuesID);
1238 addPass(PassID: &MachineSanitizerBinaryMetadataID);
1239
1240 if (TM->Options.EnableMachineOutliner &&
1241 getOptLevel() != CodeGenOptLevel::None &&
1242 EnableMachineOutliner != RunOutliner::NeverOutline) {
1243 if (EnableMachineOutliner != RunOutliner::TargetDefault ||
1244 TM->Options.SupportsDefaultOutlining)
1245 addPass(P: createMachineOutlinerPass(RunOutlinerMode: EnableMachineOutliner));
1246 }
1247
1248 if (EnableGCEmptyBlocks)
1249 addPass(P: llvm::createGCEmptyBasicBlocksLegacyPass());
1250
1251 if (EnableFSDiscriminator)
1252 addPass(P: createMIRAddFSDiscriminatorsPass(
1253 P: sampleprof::FSDiscriminatorPass::PassLast));
1254
1255 if (TM->Options.EnableMachineFunctionSplitter ||
1256 EnableMachineFunctionSplitter || SplitStaticData ||
1257 TM->Options.EnableStaticDataPartitioning) {
1258 const std::string ProfileFile = getFSProfileFile(TM);
1259 if (!ProfileFile.empty()) {
1260 if (EnableFSDiscriminator) {
1261 addPass(P: createMIRProfileLoaderPass(
1262 File: ProfileFile, RemappingFile: getFSRemappingFile(TM),
1263 P: sampleprof::FSDiscriminatorPass::PassLast, FS: nullptr));
1264 } else {
1265 // Sample profile is given, but FSDiscriminator is not
1266 // enabled, this may result in performance regression.
1267 WithColor::warning()
1268 << "Using AutoFDO without FSDiscriminator for MFS may regress "
1269 "performance.\n";
1270 }
1271 }
1272 }
1273
1274 // Machine function splitter uses the basic block sections feature.
1275 // When used along with `-basic-block-sections=`, the basic-block-sections
1276 // feature takes precedence. This means functions eligible for
1277 // basic-block-sections optimizations (`=all`, or `=list=` with function
1278 // included in the list profile) will get that optimization instead.
1279 if (TM->Options.EnableMachineFunctionSplitter ||
1280 EnableMachineFunctionSplitter)
1281 addPass(P: createMachineFunctionSplitterPass());
1282
1283 if (SplitStaticData || TM->Options.EnableStaticDataPartitioning) {
1284 // The static data splitter pass is a machine function pass. and
1285 // static data annotator pass is a module-wide pass. See the file comment
1286 // in StaticDataAnnotator.cpp for the motivation.
1287 addPass(P: createStaticDataSplitterLegacyPass());
1288 addPass(P: createStaticDataAnnotatorLegacyPass());
1289 }
1290 // We run the BasicBlockSections pass if either we need BB sections or BB
1291 // address map (or both).
1292 if (TM->getBBSectionsType() != llvm::BasicBlockSection::None ||
1293 TM->Options.BBAddrMap) {
1294 if (EmitBBHash || BasicBlockSectionMatchInfer)
1295 addPass(P: llvm::createMachineBlockHashInfoPass());
1296 if (TM->getBBSectionsType() == llvm::BasicBlockSection::List) {
1297 addPass(P: llvm::createBasicBlockSectionsProfileReaderWrapperPass(
1298 Buf: TM->getBBSectionsFuncListBuf()));
1299 if (BasicBlockSectionMatchInfer)
1300 addPass(P: llvm::createBasicBlockMatchingAndInferencePass());
1301 else {
1302 addPass(P: llvm::createBasicBlockPathCloningPass());
1303 addPass(P: llvm::createInsertCodePrefetchPass());
1304 }
1305 }
1306 addPass(P: llvm::createBasicBlockSectionsPass());
1307 }
1308
1309 addPostBBSections();
1310
1311 if (!DisableCFIFixup && TM->Options.EnableCFIFixup)
1312 addPass(P: createCFIFixupLegacy());
1313
1314 PM->add(P: createStackFrameLayoutAnalysisPass());
1315
1316 // Add passes that directly emit MI after all other MI passes.
1317 addPreEmitPass2();
1318
1319 AddingMachinePasses = false;
1320}
1321
1322/// Add passes that optimize machine instructions in SSA form.
1323void TargetPassConfig::addMachineSSAOptimization() {
1324 // Pre-ra tail duplication.
1325 addPass(PassID: &EarlyTailDuplicateLegacyID);
1326
1327 // Optimize PHIs before DCE: removing dead PHI cycles may make more
1328 // instructions dead.
1329 addPass(PassID: &OptimizePHIsLegacyID);
1330
1331 // This pass merges large allocas. StackSlotColoring is a different pass
1332 // which merges spill slots.
1333 addPass(PassID: &StackColoringLegacyID);
1334
1335 // If the target requests it, assign local variables to stack slots relative
1336 // to one another and simplify frame index references where possible.
1337 addPass(PassID: &LocalStackSlotAllocationID);
1338
1339 // With optimization, dead code should already be eliminated. However
1340 // there is one known exception: lowered code for arguments that are only
1341 // used by tail calls, where the tail calls reuse the incoming stack
1342 // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
1343 addPass(PassID: &DeadMachineInstructionElimID);
1344
1345 // Allow targets to insert passes that improve instruction level parallelism,
1346 // like if-conversion. Such passes will typically need dominator trees and
1347 // loop info, just like LICM and CSE below.
1348 addILPOpts();
1349
1350 addPass(PassID: &EarlyMachineLICMID);
1351 addPass(PassID: &MachineCSELegacyID);
1352
1353 addPass(PassID: &MachineSinkingLegacyID);
1354
1355 addPass(PassID: &PeepholeOptimizerLegacyID);
1356 // Clean-up the dead code that may have been generated by peephole
1357 // rewriting.
1358 addPass(PassID: &DeadMachineInstructionElimID);
1359}
1360
1361//===---------------------------------------------------------------------===//
1362/// Register Allocation Pass Configuration
1363//===---------------------------------------------------------------------===//
1364
1365/// A dummy default pass factory indicates whether the register allocator is
1366/// overridden on the command line.
1367static llvm::once_flag InitializeDefaultRegisterAllocatorFlag;
1368
1369static RegisterRegAlloc
1370defaultRegAlloc("default",
1371 "pick register allocator based on -O option",
1372 useDefaultRegisterAllocator);
1373
1374static void initializeDefaultRegisterAllocatorOnce() {
1375 if (!RegisterRegAlloc::getDefault())
1376 RegisterRegAlloc::setDefault(RegAlloc);
1377}
1378
1379bool TargetPassConfig::getOptimizeRegAlloc() const {
1380 // An explicit -regalloc choice implies its pipeline: only the fast
1381 // allocator uses the unoptimized one.
1382 llvm::call_once(flag&: InitializeDefaultRegisterAllocatorFlag,
1383 F&: initializeDefaultRegisterAllocatorOnce);
1384 RegisterRegAlloc::FunctionPassCtor Ctor = RegisterRegAlloc::getDefault();
1385 if (Ctor != (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator)
1386 return Ctor !=
1387 (RegisterRegAlloc::FunctionPassCtor)&createFastRegisterAllocator;
1388 return getOptLevel() != CodeGenOptLevel::None;
1389}
1390
1391/// Instantiate the default register allocator pass for this target for either
1392/// the optimized or unoptimized allocation path. This will be added to the pass
1393/// manager by addFastRegAlloc in the unoptimized case or addOptimizedRegAlloc
1394/// in the optimized case.
1395///
1396/// A target that uses the standard regalloc pass order for fast or optimized
1397/// allocation may still override this for per-target regalloc
1398/// selection. But -regalloc=... always takes precedence.
1399FunctionPass *TargetPassConfig::createTargetRegisterAllocator(bool Optimized) {
1400 if (Optimized)
1401 return createGreedyRegisterAllocator();
1402 else
1403 return createFastRegisterAllocator();
1404}
1405
1406/// Find and instantiate the register allocation pass requested by this target
1407/// at the current optimization level. Different register allocators are
1408/// defined as separate passes because they may require different analysis.
1409///
1410/// This helper ensures that the regalloc= option is always available,
1411/// even for targets that override the default allocator.
1412///
1413/// FIXME: When MachinePassRegistry register pass IDs instead of function ptrs,
1414/// this can be folded into addPass.
1415FunctionPass *TargetPassConfig::createRegAllocPass(bool Optimized) {
1416 // getOptimizeRegAlloc, called before the pipeline branches, has initialized
1417 // the global default.
1418 RegisterRegAlloc::FunctionPassCtor Ctor = RegisterRegAlloc::getDefault();
1419 if (Ctor != useDefaultRegisterAllocator)
1420 return Ctor();
1421
1422 // With no -regalloc= override, ask the target for a regalloc pass.
1423 return createTargetRegisterAllocator(Optimized);
1424}
1425
1426bool TargetPassConfig::isCustomizedRegAlloc() {
1427 return RegAlloc !=
1428 (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator;
1429}
1430
1431bool TargetPassConfig::addRegAssignAndRewriteFast() {
1432 if (RegAlloc != (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator &&
1433 RegAlloc != (RegisterRegAlloc::FunctionPassCtor)&createFastRegisterAllocator)
1434 reportFatalUsageError(
1435 reason: "Must use fast (default) register allocator for unoptimized regalloc.");
1436
1437 addPass(P: createRegAllocPass(Optimized: false));
1438
1439 // Allow targets to change the register assignments after
1440 // fast register allocation.
1441 addPostFastRegAllocRewrite();
1442 return true;
1443}
1444
1445bool TargetPassConfig::addRegAssignAndRewriteOptimized() {
1446 // Add the selected register allocation pass.
1447 addPass(P: createRegAllocPass(Optimized: true));
1448
1449 // Allow targets to change the register assignments before rewriting.
1450 addPreRewrite();
1451
1452 // Finally rewrite virtual registers.
1453 addPass(PassID: &VirtRegRewriterID);
1454
1455 // Regalloc scoring for ML-driven eviction - noop except when learning a new
1456 // eviction policy.
1457 addPass(P: createRegAllocScoringPass());
1458 return true;
1459}
1460
1461/// Return true if the default global register allocator is in use and
1462/// has not be overriden on the command line with '-regalloc=...'
1463bool TargetPassConfig::usingDefaultRegAlloc() const {
1464 return RegAlloc.getNumOccurrences() == 0;
1465}
1466
1467/// Add the minimum set of target-independent passes that are required for
1468/// register allocation. No coalescing or scheduling.
1469void TargetPassConfig::addFastRegAlloc() {
1470 addPass(PassID: &PHIEliminationID);
1471 addPass(PassID: &TwoAddressInstructionPassID);
1472
1473 addRegAssignAndRewriteFast();
1474}
1475
1476/// Add standard target-independent passes that are tightly coupled with
1477/// optimized register allocation, including coalescing, machine instruction
1478/// scheduling, and register allocation itself.
1479void TargetPassConfig::addOptimizedRegAlloc() {
1480 addPass(PassID: &DetectDeadLanesID);
1481
1482 addPass(PassID: &InitUndefID);
1483
1484 addPass(PassID: &ProcessImplicitDefsID);
1485
1486 // LiveVariables currently requires pure SSA form.
1487 //
1488 // FIXME: Once TwoAddressInstruction pass no longer uses kill flags,
1489 // LiveVariables can be removed completely, and LiveIntervals can be directly
1490 // computed. (We still either need to regenerate kill flags after regalloc, or
1491 // preferably fix the scavenger to not depend on them).
1492 // FIXME: UnreachableMachineBlockElim is a dependant pass of LiveVariables.
1493 // When LiveVariables is removed this has to be removed/moved either.
1494 // Explicit addition of UnreachableMachineBlockElim allows stopping before or
1495 // after it with -stop-before/-stop-after.
1496 addPass(PassID: &UnreachableMachineBlockElimID);
1497 addPass(PassID: &LiveVariablesID);
1498
1499 // Edge splitting is smarter with machine loop info.
1500 addPass(PassID: &MachineLoopInfoID);
1501 addPass(PassID: &PHIEliminationID);
1502
1503 // Eventually, we want to run LiveIntervals before PHI elimination.
1504 if (EarlyLiveIntervals)
1505 addPass(PassID: &LiveIntervalsID);
1506
1507 addPass(PassID: &TwoAddressInstructionPassID);
1508 addPass(PassID: &RegisterCoalescerID);
1509
1510 // The machine scheduler may accidentally create disconnected components
1511 // when moving subregister definitions around, avoid this by splitting them to
1512 // separate vregs before. Splitting can also improve reg. allocation quality.
1513 addPass(PassID: &RenameIndependentSubregsID);
1514
1515 // PreRA instruction scheduling.
1516 addPass(PassID: &MachineSchedulerID);
1517
1518 if (addRegAssignAndRewriteOptimized()) {
1519 // Perform stack slot coloring and post-ra machine LICM.
1520 addPass(PassID: &StackSlotColoringID);
1521
1522 // Allow targets to expand pseudo instructions depending on the choice of
1523 // registers before MachineCopyPropagation.
1524 addPostRewrite();
1525
1526 // Copy propagate to forward register uses and try to eliminate COPYs that
1527 // were not coalesced.
1528 addPass(PassID: &MachineCopyPropagationID);
1529
1530 // Run post-ra machine LICM to hoist reloads / remats.
1531 //
1532 // FIXME: can this move into MachineLateOptimization?
1533 addPass(PassID: &MachineLICMID);
1534 }
1535}
1536
1537//===---------------------------------------------------------------------===//
1538/// Post RegAlloc Pass Configuration
1539//===---------------------------------------------------------------------===//
1540
1541/// Add passes that optimize machine instructions after register allocation.
1542void TargetPassConfig::addMachineLateOptimization() {
1543 // Cleanup of redundant immediate/address loads.
1544 addPass(PassID: &MachineLateInstrsCleanupID);
1545
1546 // Branch folding must be run after regalloc and prolog/epilog insertion.
1547 addPass(PassID: &BranchFolderPassID);
1548
1549 // Tail duplication.
1550 // Note that duplicating tail just increases code size and degrades
1551 // performance for targets that require Structured Control Flow.
1552 // In addition it can also make CFG irreducible. Thus we disable it.
1553 if (!TM->requiresStructuredCFG())
1554 addPass(PassID: &TailDuplicateLegacyID);
1555
1556 // Copy propagation.
1557 addPass(PassID: &MachineCopyPropagationID);
1558}
1559
1560/// Add standard GC passes.
1561bool TargetPassConfig::addGCPasses() {
1562 addPass(PassID: &GCMachineCodeAnalysisID);
1563 return true;
1564}
1565
1566/// Add standard basic block placement passes.
1567void TargetPassConfig::addBlockPlacement() {
1568 if (EnableFSDiscriminator) {
1569 addPass(P: createMIRAddFSDiscriminatorsPass(
1570 P: sampleprof::FSDiscriminatorPass::Pass2));
1571 const std::string ProfileFile = getFSProfileFile(TM);
1572 if (!ProfileFile.empty() && !DisableLayoutFSProfileLoader)
1573 addPass(P: createMIRProfileLoaderPass(File: ProfileFile, RemappingFile: getFSRemappingFile(TM),
1574 P: sampleprof::FSDiscriminatorPass::Pass2,
1575 FS: nullptr));
1576 }
1577 if (addPass(PassID: &MachineBlockPlacementID)) {
1578 // Run a separate pass to collect block placement statistics.
1579 if (EnableBlockPlacementStats)
1580 addPass(PassID: &MachineBlockPlacementStatsID);
1581 }
1582}
1583
1584//===---------------------------------------------------------------------===//
1585/// GlobalISel Configuration
1586//===---------------------------------------------------------------------===//
1587bool TargetPassConfig::isGlobalISelAbortEnabled() const {
1588 return TM->Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
1589}
1590
1591bool TargetPassConfig::reportDiagnosticWhenGlobalISelFallback() const {
1592 return TM->Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag;
1593}
1594
1595std::unique_ptr<CSEConfigBase> TargetPassConfig::getCSEConfig() const {
1596 return std::make_unique<CSEConfigBase>();
1597}
1598