1//===- TargetPassConfig.cpp - Target independent code generation passes ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines interfaces to access the target independent code
10// generation passes provided by the LLVM backend.
11//
12//===---------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/TargetPassConfig.h"
15#include "CodeGenOptions.h"
16#include "llvm/ADT/DenseMap.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/StringRef.h"
19#include "llvm/Analysis/BasicAliasAnalysis.h"
20#include "llvm/Analysis/CallGraphSCCPass.h"
21#include "llvm/Analysis/ScopedNoAliasAA.h"
22#include "llvm/Analysis/TargetTransformInfo.h"
23#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
24#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
25#include "llvm/CodeGen/CSEConfigBase.h"
26#include "llvm/CodeGen/CodeGenTargetMachineImpl.h"
27#include "llvm/CodeGen/MachineBlockHashInfo.h"
28#include "llvm/CodeGen/MachineFunctionPass.h"
29#include "llvm/CodeGen/MachinePassRegistry.h"
30#include "llvm/CodeGen/Passes.h"
31#include "llvm/CodeGen/RegAllocRegistry.h"
32#include "llvm/IR/IRPrintingPasses.h"
33#include "llvm/IR/LegacyPassManager.h"
34#include "llvm/IR/PassInstrumentation.h"
35#include "llvm/IR/Verifier.h"
36#include "llvm/InitializePasses.h"
37#include "llvm/MC/MCAsmInfo.h"
38#include "llvm/MC/MCTargetOptions.h"
39#include "llvm/Pass.h"
40#include "llvm/Support/CodeGen.h"
41#include "llvm/Support/CommandLine.h"
42#include "llvm/Support/Compiler.h"
43#include "llvm/Support/Debug.h"
44#include "llvm/Support/Discriminator.h"
45#include "llvm/Support/ErrorHandling.h"
46#include "llvm/Support/SaveAndRestore.h"
47#include "llvm/Support/Threading.h"
48#include "llvm/Support/VirtualFileSystem.h"
49#include "llvm/Support/WithColor.h"
50#include "llvm/Target/CGPassBuilderOption.h"
51#include "llvm/Target/TargetMachine.h"
52#include "llvm/Transforms/ObjCARC.h"
53#include "llvm/Transforms/Scalar.h"
54#include "llvm/Transforms/Utils.h"
55#include <cassert>
56#include <optional>
57#include <string>
58
59using namespace llvm;
60
61static const CodeGenOptions &getOpts(const TargetMachine &TM) {
62 return cl::LibraryOptionsContext::get<CodeGenOptions>(
63 C: TM.Options.MCOptions.LibraryOptions);
64}
65
66namespace {
67class TriggerCrashFunctionLegacyPass : public FunctionPass {
68public:
69 static char ID;
70 TriggerCrashFunctionLegacyPass() : FunctionPass(ID) {}
71 bool runOnFunction(Function &F) override {
72 abort();
73 return false;
74 }
75 StringRef getPassName() const override { return "TriggerCrashFunctionPass"; }
76};
77} // namespace
78
79char TriggerCrashFunctionLegacyPass::ID = 0;
80
81// Enable or disable the MachineOutliner.
82static cl::opt<RunOutliner> EnableMachineOutliner(
83 "enable-machine-outliner", cl::desc("Enable the machine outliner"),
84 cl::Hidden, cl::ValueOptional, cl::init(Val: RunOutliner::TargetDefault),
85 cl::values(
86 clEnumValN(RunOutliner::AlwaysOutline, "always",
87 "Run on all functions guaranteed to be beneficial"),
88 clEnumValN(RunOutliner::OptimisticPGO, "optimistic-pgo",
89 "Outline cold code only. If a code block does not have "
90 "profile data, optimistically assume it is cold."),
91 clEnumValN(RunOutliner::ConservativePGO, "conservative-pgo",
92 "Outline cold code only. If a code block does not have "
93 "profile, data, conservatively assume it is hot."),
94 clEnumValN(RunOutliner::NeverOutline, "never", "Disable all outlining"),
95 // Sentinel value for unspecified option.
96 clEnumValN(RunOutliner::AlwaysOutline, "", "")));
97
98/// Option names for limiting the codegen pipeline.
99/// Those are used in error reporting and we didn't want
100/// to duplicate their names all over the place.
101static const char StartAfterOptName[] = "start-after";
102static const char StartBeforeOptName[] = "start-before";
103static const char StopAfterOptName[] = "stop-after";
104static const char StopBeforeOptName[] = "stop-before";
105
106/// Allow standard passes to be disabled by command line options. This supports
107/// simple binary flags that either suppress the pass or do nothing.
108/// i.e. -disable-mypass=false has no effect.
109/// These should be converted to boolOrDefault in order to use applyOverride.
110static IdentifyingPassPtr applyDisable(IdentifyingPassPtr PassID,
111 bool Override) {
112 if (Override)
113 return IdentifyingPassPtr();
114 return PassID;
115}
116
117/// Allow standard passes to be disabled by the command line, regardless of who
118/// is adding the pass.
119///
120/// StandardID is the pass identified in the standard pass pipeline and provided
121/// to addPass(). It may be a target-specific ID in the case that the target
122/// directly adds its own pass, but in that case we harmlessly fall through.
123///
124/// TargetID is the pass that the target has configured to override StandardID.
125///
126/// StandardID may be a pseudo ID. In that case TargetID is the name of the real
127/// pass to run. This allows multiple options to control a single pass depending
128/// on where in the pipeline that pass is added.
129static IdentifyingPassPtr overridePass(const CodeGenOptions &Opts,
130 AnalysisID StandardID,
131 IdentifyingPassPtr TargetID) {
132 if (StandardID == &PostRASchedulerID)
133 return applyDisable(PassID: TargetID, Override: Opts.disable_post_ra);
134
135 if (StandardID == &BranchFolderPassID)
136 return applyDisable(PassID: TargetID, Override: Opts.disable_branch_fold);
137
138 if (StandardID == &TailDuplicateLegacyID)
139 return applyDisable(PassID: TargetID, Override: Opts.disable_tail_duplicate);
140
141 if (StandardID == &EarlyTailDuplicateLegacyID)
142 return applyDisable(PassID: TargetID, Override: Opts.disable_early_taildup);
143
144 if (StandardID == &MachineBlockPlacementID)
145 return applyDisable(PassID: TargetID, Override: Opts.disable_block_placement);
146
147 if (StandardID == &StackSlotColoringID)
148 return applyDisable(PassID: TargetID, Override: Opts.disable_ssc);
149
150 if (StandardID == &DeadMachineInstructionElimID)
151 return applyDisable(PassID: TargetID, Override: Opts.disable_machine_dce);
152
153 if (StandardID == &EarlyIfConverterLegacyID)
154 return applyDisable(PassID: TargetID, Override: Opts.disable_early_ifcvt);
155
156 if (StandardID == &EarlyMachineLICMID)
157 return applyDisable(PassID: TargetID, Override: Opts.disable_machine_licm);
158
159 if (StandardID == &MachineCSELegacyID)
160 return applyDisable(PassID: TargetID, Override: Opts.disable_machine_cse);
161
162 if (StandardID == &MachineLICMID)
163 return applyDisable(PassID: TargetID, Override: Opts.disable_postra_machine_licm);
164
165 if (StandardID == &MachineSinkingLegacyID)
166 return applyDisable(PassID: TargetID, Override: Opts.disable_machine_sink);
167
168 if (StandardID == &PostRAMachineSinkingID)
169 return applyDisable(PassID: TargetID, Override: Opts.disable_postra_machine_sink);
170
171 if (StandardID == &MachineCopyPropagationID)
172 return applyDisable(PassID: TargetID, Override: Opts.disable_copyprop);
173
174 return TargetID;
175}
176
177// Find the FSProfile file name. The internal option takes the precedence
178// before getting from TargetMachine.
179static std::string getFSProfileFile(const CodeGenOptions &Opts,
180 const TargetMachine *TM) {
181 if (!Opts.fs_profile_file.empty())
182 return Opts.fs_profile_file.str();
183 const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
184 if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
185 return std::string();
186 return PGOOpt->ProfileFile;
187}
188
189// Find the Profile remapping file name. The internal option takes the
190// precedence before getting from TargetMachine.
191static std::string getFSRemappingFile(const CodeGenOptions &Opts,
192 const TargetMachine *TM) {
193 if (!Opts.fs_remapping_file.empty())
194 return Opts.fs_remapping_file.str();
195 const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
196 if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
197 return std::string();
198 return PGOOpt->ProfileRemappingFile;
199}
200
201//===---------------------------------------------------------------------===//
202/// TargetPassConfig
203//===---------------------------------------------------------------------===//
204
205INITIALIZE_PASS(TargetPassConfig, "targetpassconfig",
206 "Target Pass Configuration", false, false)
207char TargetPassConfig::ID = 0;
208
209namespace {
210
211struct InsertedPass {
212 AnalysisID TargetPassID;
213 IdentifyingPassPtr InsertedPassID;
214
215 InsertedPass(AnalysisID TargetPassID, IdentifyingPassPtr InsertedPassID)
216 : TargetPassID(TargetPassID), InsertedPassID(InsertedPassID) {}
217
218 Pass *getInsertedPass() const {
219 assert(InsertedPassID.isValid() && "Illegal Pass ID!");
220 if (InsertedPassID.isInstance())
221 return InsertedPassID.getInstance();
222 Pass *NP = Pass::createPass(ID: InsertedPassID.getID());
223 assert(NP && "Pass ID not registered");
224 return NP;
225 }
226};
227
228} // end anonymous namespace
229
230namespace llvm {
231
232class PassConfigImpl {
233public:
234 // List of passes explicitly substituted by this target. Normally this is
235 // empty, but it is a convenient way to suppress or replace specific passes
236 // that are part of a standard pass pipeline without overridding the entire
237 // pipeline. This mechanism allows target options to inherit a standard pass's
238 // user interface. For example, a target may disable a standard pass by
239 // default by substituting a pass ID of zero, and the user may still enable
240 // that standard pass with an explicit command line option.
241 DenseMap<AnalysisID,IdentifyingPassPtr> TargetPasses;
242
243 /// Store the pairs of <AnalysisID, AnalysisID> of which the second pass
244 /// is inserted after each instance of the first one.
245 SmallVector<InsertedPass, 4> InsertedPasses;
246};
247
248} // end namespace llvm
249
250// Out of line virtual method.
251TargetPassConfig::~TargetPassConfig() {
252 delete Impl;
253}
254
255static const PassInfo *getPassInfo(StringRef PassName) {
256 if (PassName.empty())
257 return nullptr;
258
259 const PassRegistry &PR = *PassRegistry::getPassRegistry();
260 const PassInfo *PI = PR.getPassInfo(Arg: PassName);
261 if (!PI)
262 reportFatalUsageError(reason: Twine('\"') + Twine(PassName) +
263 Twine("\" pass is not registered."));
264 return PI;
265}
266
267static AnalysisID getPassIDFromName(StringRef PassName) {
268 const PassInfo *PI = getPassInfo(PassName);
269 return PI ? PI->getTypeInfo() : nullptr;
270}
271
272static std::pair<StringRef, unsigned>
273getPassNameAndInstanceNum(StringRef PassName) {
274 StringRef Name, InstanceNumStr;
275 std::tie(args&: Name, args&: InstanceNumStr) = PassName.split(Separator: ',');
276
277 unsigned InstanceNum = 0;
278 if (!InstanceNumStr.empty() && InstanceNumStr.getAsInteger(Radix: 10, Result&: InstanceNum))
279 reportFatalUsageError(reason: "invalid pass instance specifier " + PassName);
280
281 return std::make_pair(x&: Name, y&: InstanceNum);
282}
283
284void TargetPassConfig::setStartStopPasses() {
285 StringRef StartBeforeName;
286 std::tie(args&: StartBeforeName, args&: StartBeforeInstanceNum) =
287 getPassNameAndInstanceNum(PassName: Opts.start_before);
288
289 StringRef StartAfterName;
290 std::tie(args&: StartAfterName, args&: StartAfterInstanceNum) =
291 getPassNameAndInstanceNum(PassName: Opts.start_after);
292
293 StringRef StopBeforeName;
294 std::tie(args&: StopBeforeName, args&: StopBeforeInstanceNum) =
295 getPassNameAndInstanceNum(PassName: Opts.stop_before);
296
297 StringRef StopAfterName;
298 std::tie(args&: StopAfterName, args&: StopAfterInstanceNum) =
299 getPassNameAndInstanceNum(PassName: Opts.stop_after);
300
301 StartBefore = getPassIDFromName(PassName: StartBeforeName);
302 StartAfter = getPassIDFromName(PassName: StartAfterName);
303 StopBefore = getPassIDFromName(PassName: StopBeforeName);
304 StopAfter = getPassIDFromName(PassName: StopAfterName);
305 if (StartBefore && StartAfter)
306 reportFatalUsageError(reason: Twine(StartBeforeOptName) + Twine(" and ") +
307 Twine(StartAfterOptName) + Twine(" specified!"));
308 if (StopBefore && StopAfter)
309 reportFatalUsageError(reason: Twine(StopBeforeOptName) + Twine(" and ") +
310 Twine(StopAfterOptName) + Twine(" specified!"));
311 Started = (StartAfter == nullptr) && (StartBefore == nullptr);
312}
313
314static cl::boolOrDefault toBoolOrDefault(BoolOrDefault B) {
315 if (B == BoolOrDefault::Default)
316 return cl::boolOrDefault::BOU_UNSET;
317 return B == BoolOrDefault::True ? cl::boolOrDefault::BOU_TRUE
318 : cl::boolOrDefault::BOU_FALSE;
319}
320
321static MachineVerifierMode
322computeMachineVerifierMode(const CodeGenOptions &Opts, bool VerifyEach) {
323 if (VerifyEach)
324 return MachineVerifierMode::Each;
325 return Opts.disable_mir_output_verify ? MachineVerifierMode::None
326 : MachineVerifierMode::End;
327}
328
329CGPassBuilderOption llvm::getCGPassBuilderOption(const TargetMachine &TM) {
330 const CodeGenOptions &Opts = getOpts(TM);
331 CGPassBuilderOption Opt;
332 Opt.EnableGlobalISelAbort = Opts.global_isel_abort;
333 if (Opts.enable_ipra != BoolOrDefault::Default)
334 Opt.EnableIPRA = Opts.enable_ipra == BoolOrDefault::True;
335 Opt.EnableFastISelOption = toBoolOrDefault(B: Opts.fast_isel);
336 Opt.EnableRegAllocFastTied = toBoolOrDefault(B: Opts.regalloc_fast_tied);
337 Opt.EnableGlobalISelOption = toBoolOrDefault(B: Opts.global_isel);
338 Opt.VerifyMachineCode = computeMachineVerifierMode(
339 Opts, VerifyEach: valueOr(X: Opts.verify_machineinstrs, Default: false));
340 Opt.DisableAtExitBasedGlobalDtorLowering =
341 Opts.disable_atexit_based_global_dtor_lowering;
342 Opt.DisableExpandReductions = Opts.disable_expand_reductions;
343 Opt.PrintAfterISel = Opts.print_after_isel;
344 Opt.FSProfileFile = Opts.fs_profile_file.str();
345 Opt.EnableGCEmptyBlocks = Opts.enable_gc_empty_basic_blocks;
346 Opt.EnableBlockPlacementStats = Opts.enable_block_placement_stats;
347 Opt.EnableGlobalMergeFunc = Opts.enable_global_merge_func;
348 Opt.EnableImplicitNullChecks = Opts.enable_implicit_null_checks;
349 Opt.EnableMachineOutliner = EnableMachineOutliner;
350 Opt.MISchedPostRA = Opts.misched_postra;
351 Opt.DisableLSR = Opts.disable_lsr;
352 Opt.DisableConstantHoisting = Opts.disable_constant_hoisting;
353 Opt.DisableCGP = Opts.disable_cgp;
354 Opt.DisablePartialLibcallInlining = Opts.disable_partial_libcall_inlining;
355 Opt.DisableSelectOptimize = Opts.disable_select_optimize;
356 Opt.PrintISelInput = Opts.print_isel_input;
357 Opt.PrintRegUsage = Opts.print_regusage;
358 Opt.DebugifyAndStripAll = toBoolOrDefault(B: Opts.debugify_and_strip_all_safe);
359 Opt.DebugifyCheckAndStripAll =
360 toBoolOrDefault(B: Opts.debugify_check_and_strip_all_safe);
361 Opt.DisableRAFSProfileLoader = Opts.disable_ra_fsprofile_loader;
362 Opt.DisableCFIFixup = Opts.disable_cfi_fixup;
363 return Opt;
364}
365
366void llvm::registerCodeGenCallback(PassInstrumentationCallbacks &PIC,
367 TargetMachine &TM) {
368 const CodeGenOptions &Opts = getOpts(TM);
369
370 // Register a callback for disabling passes.
371 PIC.registerShouldRunOptionalPassCallback(C: [&Opts](StringRef P, IRUnitRef) {
372
373#define DISABLE_PASS(Option, Name) \
374 if (Option && P.contains(#Name)) \
375 return false;
376 DISABLE_PASS(Opts.disable_block_placement, MachineBlockPlacementPass)
377 DISABLE_PASS(Opts.disable_branch_fold, BranchFolderPass)
378 DISABLE_PASS(Opts.disable_copyprop, MachineCopyPropagationPass)
379 DISABLE_PASS(Opts.disable_early_ifcvt, EarlyIfConverterLegacyPass)
380 DISABLE_PASS(Opts.disable_early_taildup, EarlyTailDuplicatePass)
381 DISABLE_PASS(Opts.disable_machine_cse, MachineCSELegacyPass)
382 DISABLE_PASS(Opts.disable_machine_dce, DeadMachineInstructionElimPass)
383 DISABLE_PASS(Opts.disable_machine_licm, EarlyMachineLICMPass)
384 DISABLE_PASS(Opts.disable_machine_sink, MachineSinkingPass)
385 DISABLE_PASS(Opts.disable_postra_machine_licm, MachineLICMPass)
386 DISABLE_PASS(Opts.disable_postra_machine_sink, PostRAMachineSinkingPass)
387 DISABLE_PASS(Opts.disable_post_ra, PostRASchedulerPass)
388 DISABLE_PASS(Opts.disable_ssc, StackSlotColoringPass)
389 DISABLE_PASS(Opts.disable_tail_duplicate, TailDuplicatePass)
390
391 return true;
392 });
393}
394
395Expected<TargetPassConfig::StartStopInfo>
396TargetPassConfig::getStartStopInfo(const TargetMachine &TM,
397 PassInstrumentationCallbacks &PIC) {
398 const CodeGenOptions &Opts = getOpts(TM);
399 auto [StartBefore, StartBeforeInstanceNum] =
400 getPassNameAndInstanceNum(PassName: Opts.start_before);
401 auto [StartAfter, StartAfterInstanceNum] =
402 getPassNameAndInstanceNum(PassName: Opts.start_after);
403 auto [StopBefore, StopBeforeInstanceNum] =
404 getPassNameAndInstanceNum(PassName: Opts.stop_before);
405 auto [StopAfter, StopAfterInstanceNum] =
406 getPassNameAndInstanceNum(PassName: Opts.stop_after);
407
408 if (!StartBefore.empty() && !StartAfter.empty())
409 return make_error<StringError>(
410 Args: Twine(StartBeforeOptName) + " and " + StartAfterOptName + " specified!",
411 Args: std::make_error_code(e: std::errc::invalid_argument));
412 if (!StopBefore.empty() && !StopAfter.empty())
413 return make_error<StringError>(
414 Args: Twine(StopBeforeOptName) + " and " + StopAfterOptName + " specified!",
415 Args: std::make_error_code(e: std::errc::invalid_argument));
416
417 StartStopInfo Result;
418 Result.StartPass = StartBefore.empty() ? StartAfter : StartBefore;
419 Result.StopPass = StopBefore.empty() ? StopAfter : StopBefore;
420 Result.StartInstanceNum =
421 StartBefore.empty() ? StartAfterInstanceNum : StartBeforeInstanceNum;
422 Result.StopInstanceNum =
423 StopBefore.empty() ? StopAfterInstanceNum : StopBeforeInstanceNum;
424 Result.StartAfter = !StartAfter.empty();
425 Result.StopAfter = !StopAfter.empty();
426 Result.StartInstanceNum += Result.StartInstanceNum == 0;
427 Result.StopInstanceNum += Result.StopInstanceNum == 0;
428 return Result;
429}
430
431// Out of line constructor provides default values for pass options and
432// registers all common codegen passes.
433TargetPassConfig::TargetPassConfig(TargetMachine &TM, PassManagerBase &PM)
434 : ImmutablePass(ID), PM(&PM), Opts(getOpts(TM)), TM(&TM) {
435 Impl = new PassConfigImpl();
436
437 PassRegistry &PR = *PassRegistry::getPassRegistry();
438 // Register all target independent codegen passes to activate their PassIDs,
439 // including this pass itself.
440 initializeCodeGen(PR);
441
442 initializeLibcallLoweringInfoWrapperPass(PR);
443
444 // Also register alias analysis passes required by codegen passes.
445 initializeBasicAAWrapperPassPass(PR);
446 initializeAAResultsWrapperPassPass(PR);
447
448 if (Opts.enable_ipra != BoolOrDefault::Default) {
449 TM.Options.EnableIPRA = Opts.enable_ipra == BoolOrDefault::True;
450 } else {
451 // If not explicitly specified, use target default.
452 TM.Options.EnableIPRA |= TM.useIPRA();
453 }
454
455 if (TM.Options.EnableIPRA)
456 setRequiresCodeGenSCCOrder();
457
458 if (Opts.regalloc_fast_tied != BoolOrDefault::Default)
459 TM.setEnableTiedFastRegAlloc(Opts.regalloc_fast_tied ==
460 BoolOrDefault::True);
461
462 if (Opts.global_isel_abort)
463 TM.Options.GlobalISelAbort = *Opts.global_isel_abort;
464
465 bool VerifyEach = valueOr(X: Opts.verify_machineinstrs, Default: false);
466#ifdef EXPENSIVE_CHECKS
467 if (Opts.verify_machineinstrs == BoolOrDefault::Default)
468 VerifyEach = TM.isMachineVerifierClean();
469#endif
470 VerifyMode = computeMachineVerifierMode(Opts, VerifyEach);
471
472 if (Opts.function_splitting)
473 TM.Options.FunctionSplitting = *Opts.function_splitting;
474
475 setStartStopPasses();
476}
477
478CodeGenOptLevel TargetPassConfig::getOptLevel() const {
479 return TM->getOptLevel();
480}
481
482/// Insert InsertedPassID pass after TargetPassID.
483void TargetPassConfig::insertPass(AnalysisID TargetPassID,
484 IdentifyingPassPtr InsertedPassID) {
485 assert(((!InsertedPassID.isInstance() &&
486 TargetPassID != InsertedPassID.getID()) ||
487 (InsertedPassID.isInstance() &&
488 TargetPassID != InsertedPassID.getInstance()->getPassID())) &&
489 "Insert a pass after itself!");
490 Impl->InsertedPasses.emplace_back(Args&: TargetPassID, Args&: InsertedPassID);
491}
492
493/// createPassConfig - Create a pass configuration object to be used by
494/// addPassToEmitX methods for generating a pipeline of CodeGen passes.
495///
496/// Targets may override this to extend TargetPassConfig.
497TargetPassConfig *
498CodeGenTargetMachineImpl::createPassConfig(PassManagerBase &PM) {
499 return new TargetPassConfig(*this, PM);
500}
501
502template <> Pass *llvm::callDefaultCtor<TargetPassConfig>() {
503 reportFatalUsageError(reason: "trying to construct TargetPassConfig without a target "
504 "machine. Scheduling a CodeGen pass without a target "
505 "triple set?");
506}
507
508bool TargetPassConfig::willCompleteCodeGenPipeline() const {
509 return Opts.stop_before.empty() && Opts.stop_after.empty();
510}
511
512bool TargetPassConfig::hasLimitedCodeGenPipeline() const {
513 return !Opts.start_before.empty() || !Opts.start_after.empty() ||
514 !willCompleteCodeGenPipeline();
515}
516
517static std::string formatLimitReason(StringRef StartAfter,
518 StringRef StartBefore, StringRef StopAfter,
519 StringRef StopBefore) {
520 std::string Res;
521 StringRef PassNames[] = {StartAfter, StartBefore, StopAfter, StopBefore};
522 static const char *OptNames[] = {StartAfterOptName, StartBeforeOptName,
523 StopAfterOptName, StopBeforeOptName};
524 bool IsFirst = true;
525 for (int Idx = 0; Idx < 4; ++Idx)
526 if (!PassNames[Idx].empty()) {
527 if (!IsFirst)
528 Res += " and ";
529 IsFirst = false;
530 Res += OptNames[Idx];
531 }
532 return Res;
533}
534
535std::string TargetPassConfig::getLimitedCodeGenPipelineReason() const {
536 return formatLimitReason(StartAfter: Opts.start_after, StartBefore: Opts.start_before, StopAfter: Opts.stop_after,
537 StopBefore: Opts.stop_before);
538}
539
540std::string TargetPassConfig::StartStopInfo::getLimitReason() const {
541 return formatLimitReason(
542 StartAfter: StartAfter ? StartPass : "", StartBefore: StartAfter ? "" : StartPass,
543 StopAfter: StopAfter ? StopPass : "", StopBefore: StopAfter ? "" : StopPass);
544}
545
546// Helper to verify the analysis is really immutable.
547void TargetPassConfig::setOpt(bool &Opt, bool Val) {
548 assert(!Initialized && "PassConfig is immutable");
549 Opt = Val;
550}
551
552void TargetPassConfig::substitutePass(AnalysisID StandardID,
553 IdentifyingPassPtr TargetID) {
554 Impl->TargetPasses[StandardID] = TargetID;
555}
556
557IdentifyingPassPtr TargetPassConfig::getPassSubstitution(AnalysisID ID) const {
558 DenseMap<AnalysisID, IdentifyingPassPtr>::const_iterator
559 I = Impl->TargetPasses.find(Val: ID);
560 if (I == Impl->TargetPasses.end())
561 return ID;
562 return I->second;
563}
564
565bool TargetPassConfig::isPassSubstitutedOrOverridden(AnalysisID ID) const {
566 IdentifyingPassPtr TargetID = getPassSubstitution(ID);
567 IdentifyingPassPtr FinalPtr = overridePass(Opts, StandardID: ID, TargetID);
568 return !FinalPtr.isValid() || FinalPtr.isInstance() ||
569 FinalPtr.getID() != ID;
570}
571
572/// Add a pass to the PassManager if that pass is supposed to be run. If the
573/// Started/Stopped flags indicate either that the compilation should start at
574/// a later pass or that it should stop after an earlier pass, then do not add
575/// the pass. Finally, compare the current pass against the StartAfter
576/// and StopAfter options and change the Started/Stopped flags accordingly.
577void TargetPassConfig::addPass(Pass *P) {
578 assert(!Initialized && "PassConfig is immutable");
579
580 // Cache the Pass ID here in case the pass manager finds this pass is
581 // redundant with ones already scheduled / available, and deletes it.
582 // Fundamentally, once we add the pass to the manager, we no longer own it
583 // and shouldn't reference it.
584 AnalysisID PassID = P->getPassID();
585
586 if (StartBefore == PassID && StartBeforeCount++ == StartBeforeInstanceNum)
587 Started = true;
588 if (StopBefore == PassID && StopBeforeCount++ == StopBeforeInstanceNum)
589 Stopped = true;
590 if (Started && !Stopped) {
591 if (AddingMachinePasses) {
592 // Construct banner message before PM->add() as that may delete the pass.
593 std::string Banner =
594 std::string("After ") + std::string(P->getPassName());
595 addMachinePrePasses();
596 PM->add(P);
597 addMachinePostPasses(Banner);
598 } else {
599 PM->add(P);
600 }
601
602 // Add the passes after the pass P if there is any.
603 for (const auto &IP : Impl->InsertedPasses)
604 if (IP.TargetPassID == PassID)
605 addPass(P: IP.getInsertedPass());
606 } else {
607 delete P;
608 }
609
610 if (StopAfter == PassID && StopAfterCount++ == StopAfterInstanceNum)
611 Stopped = true;
612
613 if (StartAfter == PassID && StartAfterCount++ == StartAfterInstanceNum)
614 Started = true;
615 if (Stopped && !Started)
616 reportFatalUsageError(reason: "Cannot stop compilation after pass that is not run");
617}
618
619/// Add a CodeGen pass at this point in the pipeline after checking for target
620/// and command line overrides.
621///
622/// addPass cannot return a pointer to the pass instance because is internal the
623/// PassManager and the instance we create here may already be freed.
624AnalysisID TargetPassConfig::addPass(AnalysisID PassID) {
625 IdentifyingPassPtr TargetID = getPassSubstitution(ID: PassID);
626 IdentifyingPassPtr FinalPtr = overridePass(Opts, StandardID: PassID, TargetID);
627 if (!FinalPtr.isValid())
628 return nullptr;
629
630 Pass *P;
631 if (FinalPtr.isInstance())
632 P = FinalPtr.getInstance();
633 else {
634 P = Pass::createPass(ID: FinalPtr.getID());
635 if (!P)
636 llvm_unreachable("Pass ID not registered");
637 }
638 AnalysisID FinalID = P->getPassID();
639 addPass(P); // Ends the lifetime of P.
640
641 return FinalID;
642}
643
644void TargetPassConfig::printAndVerify(const std::string &Banner) {
645 addPrintPass(Banner);
646 addVerifyPass(Banner);
647}
648
649void TargetPassConfig::addPrintPass(const std::string &Banner) {
650 if (Opts.print_after_isel)
651 PM->add(P: createMachineFunctionPrinterPass(OS&: dbgs(), Banner));
652}
653
654void TargetPassConfig::addVerifyPass(const std::string &Banner) {
655 if (VerifyMode == MachineVerifierMode::Each)
656 PM->add(P: createMachineVerifierPass(Banner));
657}
658
659void TargetPassConfig::addDebugifyPass() {
660 PM->add(P: createDebugifyMachineModulePass());
661}
662
663void TargetPassConfig::addStripDebugPass() {
664 PM->add(P: createStripDebugMachineModuleLegacyPass(/*OnlyDebugified=*/true));
665}
666
667void TargetPassConfig::addCheckDebugPass() {
668 PM->add(P: createCheckDebugMachineModuleLegacyPass());
669}
670
671void TargetPassConfig::addMachinePrePasses(bool AllowDebugify) {
672 if (AllowDebugify && DebugifyIsSafe &&
673 (valueOr(X: Opts.debugify_and_strip_all_safe, Default: false) ||
674 valueOr(X: Opts.debugify_check_and_strip_all_safe, Default: false)))
675 addDebugifyPass();
676}
677
678void TargetPassConfig::addMachinePostPasses(const std::string &Banner) {
679 if (DebugifyIsSafe) {
680 if (valueOr(X: Opts.debugify_check_and_strip_all_safe, Default: false)) {
681 addCheckDebugPass();
682 addStripDebugPass();
683 } else if (valueOr(X: Opts.debugify_and_strip_all_safe, Default: false))
684 addStripDebugPass();
685 }
686 addVerifyPass(Banner);
687}
688
689/// Add common target configurable passes that perform LLVM IR to IR transforms
690/// following machine independent optimization.
691void TargetPassConfig::addIRPasses() {
692 // Before running any passes, run the verifier to determine if the input
693 // coming from the front-end and/or optimizer is valid.
694 if (!DisableVerify)
695 addPass(P: createVerifierPass());
696
697 if (getOptLevel() != CodeGenOptLevel::None) {
698 // Basic AliasAnalysis support.
699 // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that
700 // BasicAliasAnalysis wins if they disagree. This is intended to help
701 // support "obvious" type-punning idioms.
702 addPass(P: createTypeBasedAAWrapperPass());
703 addPass(P: createScopedNoAliasAAWrapperPass());
704 addPass(P: createBasicAAWrapperPass());
705
706 // Run loop strength reduction before anything else.
707 if (!Opts.disable_lsr) {
708 addPass(P: createCanonicalizeFreezeInLoopsPass());
709 addPass(P: createLoopStrengthReducePass());
710 if (EnableLoopTermFold)
711 addPass(P: createLoopTermFoldPass());
712 }
713 }
714
715 // Run GC lowering passes for builtin collectors
716 // TODO: add a pass insertion point here
717 addPass(PassID: &GCLoweringID);
718 addPass(PassID: &ShadowStackGCLoweringID);
719
720 // For MachO, lower @llvm.global_dtors into @llvm.global_ctors with
721 // __cxa_atexit() calls to avoid emitting the deprecated __mod_term_func.
722 if (TM->getTargetTriple().isOSBinFormatMachO() &&
723 !Opts.disable_atexit_based_global_dtor_lowering)
724 addPass(P: createLowerGlobalDtorsLegacyPass());
725
726 // Make sure that no unreachable blocks are instruction selected.
727 addPass(P: createUnreachableBlockEliminationPass());
728
729 // Prepare expensive constants for SelectionDAG.
730 if (getOptLevel() != CodeGenOptLevel::None && !Opts.disable_constant_hoisting)
731 addPass(P: createConstantHoistingPass());
732
733 if (getOptLevel() != CodeGenOptLevel::None &&
734 !Opts.disable_replace_with_vec_lib)
735 addPass(P: createReplaceWithVeclibLegacyPass());
736
737 if (getOptLevel() != CodeGenOptLevel::None &&
738 !Opts.disable_partial_libcall_inlining)
739 addPass(P: createPartiallyInlineLibCallsPass());
740
741 // Instrument function entry after all inlining.
742 addPass(P: createPostInlineEntryExitInstrumenterPass());
743
744 // Add scalarization of target's unsupported masked memory intrinsics pass.
745 // the unsupported intrinsic will be replaced with a chain of basic blocks,
746 // that stores/loads element one-by-one if the appropriate mask bit is set.
747 addPass(P: createScalarizeMaskedMemIntrinLegacyPass());
748
749 // Expand reduction intrinsics into shuffle sequences if the target wants to.
750 // Allow disabling it for testing purposes.
751 if (!Opts.disable_expand_reductions)
752 addPass(P: createExpandReductionsPass());
753
754 // Convert conditional moves to conditional jumps when profitable.
755 if (getOptLevel() != CodeGenOptLevel::None && !Opts.disable_select_optimize)
756 addPass(P: createSelectOptimizePass());
757
758 if (Opts.enable_global_merge_func)
759 addPass(P: createGlobalMergeFuncPass());
760
761 if (TM->getTargetTriple().isOSWindows())
762 addPass(P: createWindowsSecureHotPatchingPass());
763}
764
765/// Turn exception handling constructs into something the code generators can
766/// handle.
767void TargetPassConfig::addPassesToHandleExceptions() {
768 const MCAsmInfo &MCAI = TM->getMCAsmInfo();
769 switch (MCAI.getExceptionHandlingType()) {
770 case ExceptionHandling::SjLj:
771 // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both
772 // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise,
773 // catch info can get misplaced when a selector ends up more than one block
774 // removed from the parent invoke(s). This could happen when a landing
775 // pad is shared by multiple invokes and is also a target of a normal
776 // edge from elsewhere.
777 addPass(P: createSjLjEHPreparePass(TM));
778 [[fallthrough]];
779 case ExceptionHandling::DwarfCFI:
780 case ExceptionHandling::ARM:
781 case ExceptionHandling::AIX:
782 case ExceptionHandling::ZOS:
783 addPass(P: createDwarfEHPass(OptLevel: getOptLevel()));
784 break;
785 case ExceptionHandling::WinEH:
786 // We support using both GCC-style and MSVC-style exceptions on Windows, so
787 // add both preparation passes. Each pass will only actually run if it
788 // recognizes the personality function.
789 addPass(P: createWinEHPass());
790 addPass(P: createDwarfEHPass(OptLevel: getOptLevel()));
791 break;
792 case ExceptionHandling::Wasm:
793 // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs
794 // on catchpads and cleanuppads because it does not outline them into
795 // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we
796 // should remove PHIs there. WinEHPrepare derives this from the Wasm
797 // personality, so no explicit flag is needed here.
798 addPass(P: createWinEHPass());
799 break;
800 case ExceptionHandling::Default:
801 case ExceptionHandling::None:
802 case ExceptionHandling::Emscripten:
803 // Emscripten EH is lowered earlier by WebAssemblyLowerEmscriptenEHSjLj, so
804 // by this point it needs no generic EH preparation, like the None case.
805 addPass(P: createLowerInvokePass());
806
807 // The lower invoke pass may create unreachable code. Remove it.
808 addPass(P: createUnreachableBlockEliminationPass());
809 break;
810 }
811}
812
813/// Add pass to prepare the LLVM IR for code generation. This should be done
814/// before exception handling preparation passes.
815void TargetPassConfig::addCodeGenPrepare() {
816 if (getOptLevel() != CodeGenOptLevel::None && !Opts.disable_cgp)
817 addPass(P: createCodeGenPrepareLegacyPass());
818}
819
820/// Add common passes that perform LLVM IR to IR transforms in preparation for
821/// instruction selection.
822void TargetPassConfig::addISelPrepare() {
823 addPreISel();
824
825 // Force codegen to run according to the callgraph.
826 if (requiresCodeGenSCCOrder())
827 addPass(P: new DummyCGSCCPass);
828
829 addPass(P: createInlineAsmPreparePass());
830
831 // Add both the safe stack and the stack protection passes: each of them will
832 // only protect functions that have corresponding attributes.
833 addPass(P: createSafeStackPass());
834 addPass(P: createStackProtectorPass());
835
836 if (Opts.print_isel_input)
837 addPass(P: createPrintFunctionPass(
838 OS&: dbgs(), Banner: "\n\n*** Final LLVM Code input to ISel ***\n"));
839
840 // All passes which modify the LLVM IR are now complete; run the verifier
841 // to ensure that the IR is valid.
842 if (!DisableVerify)
843 addPass(P: createVerifierPass());
844}
845
846bool TargetPassConfig::addCoreISelPasses() {
847 // Enable FastISel with -fast-isel, but allow that to be overridden.
848 TM->setO0WantsFastISel(valueOr(X: Opts.fast_isel, Default: true));
849
850 // Determine an instruction selector.
851 enum class SelectorType { SelectionDAG, FastISel, GlobalISel };
852 SelectorType Selector;
853
854 if (valueOr(X: Opts.fast_isel, Default: false))
855 Selector = SelectorType::FastISel;
856 else if (valueOr(X: Opts.global_isel, Default: false) ||
857 (TM->Options.EnableGlobalISel && valueOr(X: Opts.global_isel, Default: true)))
858 Selector = SelectorType::GlobalISel;
859 else if (TM->getOptLevel() == CodeGenOptLevel::None &&
860 TM->getO0WantsFastISel())
861 Selector = SelectorType::FastISel;
862 else
863 Selector = SelectorType::SelectionDAG;
864
865 // Set consistently TM->Options.EnableFastISel and EnableGlobalISel.
866 if (Selector == SelectorType::FastISel) {
867 TM->setFastISel(true);
868 TM->setGlobalISel(false);
869 } else if (Selector == SelectorType::GlobalISel) {
870 TM->setFastISel(false);
871 TM->setGlobalISel(true);
872 }
873
874 // FIXME: Injecting into the DAGISel pipeline seems to cause issues with
875 // analyses needing to be re-run. This can result in being unable to
876 // schedule passes (particularly with 'Function Alias Analysis
877 // Results'). It's not entirely clear why but AFAICT this seems to be
878 // due to one FunctionPassManager not being able to use analyses from a
879 // previous one. As we're injecting a ModulePass we break the usual
880 // pass manager into two. GlobalISel with the fallback path disabled
881 // and -run-pass seem to be unaffected. The majority of GlobalISel
882 // testing uses -run-pass so this probably isn't too bad.
883 SaveAndRestore SavedDebugifyIsSafe(DebugifyIsSafe);
884 if (Selector != SelectorType::GlobalISel || !isGlobalISelAbortEnabled())
885 DebugifyIsSafe = false;
886
887 // Add instruction selector passes for global isel if enabled.
888 if (Selector == SelectorType::GlobalISel) {
889 SaveAndRestore SavedAddingMachinePasses(AddingMachinePasses, true);
890 if (addIRTranslator())
891 return true;
892
893 addPreLegalizeMachineIR();
894
895 if (addLegalizeMachineIR())
896 return true;
897
898 // Before running the register bank selector, ask the target if it
899 // wants to run some passes.
900 addPreRegBankSelect();
901
902 if (addRegBankSelect())
903 return true;
904
905 addPreGlobalInstructionSelect();
906
907 if (addGlobalInstructionSelect())
908 return true;
909 }
910
911 // Pass to reset the MachineFunction if the ISel failed. Outside of the above
912 // if so that the verifier is not added to it.
913 if (Selector == SelectorType::GlobalISel)
914 addPass(P: createResetMachineFunctionLegacyPass(
915 EmitFallbackDiag: reportDiagnosticWhenGlobalISelFallback(), AbortOnFailedISel: isGlobalISelAbortEnabled()));
916
917 // Run the SDAG InstSelector, providing a fallback path when we do not want to
918 // abort on not-yet-supported input.
919 if (Selector != SelectorType::GlobalISel || !isGlobalISelAbortEnabled())
920 if (addInstSelector())
921 return true;
922
923 // Expand pseudo-instructions emitted by ISel. Don't run the verifier before
924 // FinalizeISel.
925 addPass(PassID: &FinalizeISelID);
926
927 // Print the instruction selected machine code...
928 printAndVerify(Banner: "After Instruction Selection");
929
930 return false;
931}
932
933bool TargetPassConfig::addISelPasses() {
934 if (TM->useEmulatedTLS())
935 addPass(P: createLowerEmuTLSPass());
936
937 PM->add(P: createTargetTransformInfoWrapperPass(TIRA: TM->getTargetIRAnalysis()));
938 // ObjCARCContract operates on ObjC intrinsics and must run before
939 // PreISelIntrinsicLowering.
940 if (getOptLevel() != CodeGenOptLevel::None)
941 addPass(P: createObjCARCContractPass());
942 addPass(P: createPreISelIntrinsicLoweringPass());
943 addPass(P: createExpandIRInstsPass(getOptLevel()));
944 addIRPasses();
945
946 if (Opts.codegen_pipeline_trigger_crash)
947 addPass(P: new TriggerCrashFunctionLegacyPass());
948
949 addCodeGenPrepare();
950 addPassesToHandleExceptions();
951 addISelPrepare();
952
953 return addCoreISelPasses();
954}
955
956/// -regalloc=... command line option.
957static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
958static cl::opt<RegisterRegAlloc::FunctionPassCtor, false,
959 RegisterPassParser<RegisterRegAlloc>>
960 RegAlloc("regalloc", cl::Hidden, cl::init(Val: &useDefaultRegisterAllocator),
961 cl::desc("Register allocator to use"));
962
963/// Add the complete set of target-independent postISel code generator passes.
964///
965/// This can be read as the standard order of major LLVM CodeGen stages. Stages
966/// with nontrivial configuration or multiple passes are broken out below in
967/// add%Stage routines.
968///
969/// Any TargetPassConfig::addXX routine may be overriden by the Target. The
970/// addPre/Post methods with empty header implementations allow injecting
971/// target-specific fixups just before or after major stages. Additionally,
972/// targets have the flexibility to change pass order within a stage by
973/// overriding default implementation of add%Stage routines below. Each
974/// technique has maintainability tradeoffs because alternate pass orders are
975/// not well supported. addPre/Post works better if the target pass is easily
976/// tied to a common pass. But if it has subtle dependencies on multiple passes,
977/// the target should override the stage instead.
978///
979/// TODO: We could use a single addPre/Post(ID) hook to allow pass injection
980/// before/after any target-independent pass. But it's currently overkill.
981void TargetPassConfig::addMachinePasses() {
982 AddingMachinePasses = true;
983
984 // Add passes that optimize machine instructions in SSA form.
985 if (getOptLevel() != CodeGenOptLevel::None) {
986 addMachineSSAOptimization();
987 } else {
988 // If the target requests it, assign local variables to stack slots relative
989 // to one another and simplify frame index references where possible.
990 addPass(PassID: &LocalStackSlotAllocationID);
991 }
992
993 if (TM->Options.EnableIPRA)
994 addPass(P: createRegUsageInfoPropPass());
995
996 // Run pre-ra passes.
997 addPreRegAlloc();
998
999 // Debugifying the register allocator passes seems to provoke some
1000 // non-determinism that affects CodeGen and there doesn't seem to be a point
1001 // where it becomes safe again so stop debugifying here.
1002 DebugifyIsSafe = false;
1003
1004 // Add a FSDiscriminator pass right before RA, so that we could get
1005 // more precise SampleFDO profile for RA.
1006 if (EnableFSDiscriminator) {
1007 addPass(P: createMIRAddFSDiscriminatorsPass(
1008 P: sampleprof::FSDiscriminatorPass::Pass1));
1009 const std::string ProfileFile = getFSProfileFile(Opts, TM);
1010 if (!ProfileFile.empty() && !Opts.disable_ra_fsprofile_loader)
1011 addPass(P: createMIRProfileLoaderPass(
1012 File: ProfileFile, RemappingFile: getFSRemappingFile(Opts, TM),
1013 P: sampleprof::FSDiscriminatorPass::Pass1, FS: nullptr));
1014 }
1015
1016 // Run register allocation and passes that are tightly coupled with it,
1017 // including phi elimination and scheduling.
1018 if (getOptimizeRegAlloc())
1019 addOptimizedRegAlloc();
1020 else
1021 addFastRegAlloc();
1022
1023 // Run post-ra passes.
1024 addPostRegAlloc();
1025
1026 addPass(PassID: &RemoveRedundantDebugValuesID);
1027
1028 addPass(PassID: &FixupStatepointCallerSavedID);
1029
1030 // Insert prolog/epilog code. Eliminate abstract frame index references...
1031 if (getOptLevel() != CodeGenOptLevel::None) {
1032 addPass(PassID: &PostRAMachineSinkingID);
1033 addPass(PassID: &ShrinkWrapID);
1034 }
1035
1036 // Prolog/Epilog inserter needs a TargetMachine to instantiate. But only
1037 // do so if it hasn't been disabled, substituted, or overridden.
1038 if (!isPassSubstitutedOrOverridden(ID: &PrologEpilogCodeInserterID))
1039 addPass(P: createPrologEpilogInserterPass());
1040
1041 /// Add passes that optimize machine instructions after register allocation.
1042 if (getOptLevel() != CodeGenOptLevel::None)
1043 addMachineLateOptimization();
1044
1045 // Expand pseudo instructions before second scheduling pass.
1046 addPass(PassID: &ExpandPostRAPseudosID);
1047
1048 // Run pre-sched2 passes.
1049 addPreSched2();
1050
1051 if (Opts.enable_implicit_null_checks)
1052 addPass(PassID: &ImplicitNullChecksID);
1053
1054 // Second pass scheduler.
1055 // Let Target optionally insert this pass by itself at some other
1056 // point.
1057 if (getOptLevel() != CodeGenOptLevel::None &&
1058 !TM->targetSchedulesPostRAScheduling()) {
1059 if (Opts.misched_postra)
1060 addPass(PassID: &PostMachineSchedulerID);
1061 else
1062 addPass(PassID: &PostRASchedulerID);
1063 }
1064
1065 // GC
1066 addGCPasses();
1067
1068 // Basic block placement.
1069 if (getOptLevel() != CodeGenOptLevel::None)
1070 addBlockPlacement();
1071
1072 // Insert before XRay Instrumentation.
1073 addPass(PassID: &FEntryInserterID);
1074
1075 addPass(PassID: &XRayInstrumentationID);
1076 addPass(PassID: &PatchableFunctionID);
1077
1078 addPreEmitPass();
1079
1080 if (TM->Options.EnableIPRA)
1081 // Collect register usage information and produce a register mask of
1082 // clobbered registers, to be used to optimize call sites.
1083 addPass(P: createRegUsageInfoCollector());
1084
1085 // FIXME: Some backends are incompatible with running the verifier after
1086 // addPreEmitPass. Maybe only pass "false" here for those targets?
1087 addPass(PassID: &FuncletLayoutID);
1088
1089 addPass(PassID: &RemoveLoadsIntoFakeUsesID);
1090 addPass(PassID: &StackMapLivenessID);
1091 addPass(PassID: &LiveDebugValuesID);
1092 addPass(PassID: &MachineSanitizerBinaryMetadataID);
1093
1094 if (TM->Options.EnableMachineOutliner &&
1095 getOptLevel() != CodeGenOptLevel::None &&
1096 EnableMachineOutliner != RunOutliner::NeverOutline) {
1097 if (EnableMachineOutliner != RunOutliner::TargetDefault ||
1098 TM->supportsDefaultOutlining())
1099 addPass(P: createMachineOutlinerPass(RunOutlinerMode: EnableMachineOutliner));
1100 }
1101
1102 if (Opts.enable_gc_empty_basic_blocks)
1103 addPass(P: llvm::createGCEmptyBasicBlocksLegacyPass());
1104
1105 if (EnableFSDiscriminator)
1106 addPass(P: createMIRAddFSDiscriminatorsPass(
1107 P: sampleprof::FSDiscriminatorPass::PassLast));
1108
1109 const bool SplitFunctionsWithPGO =
1110 TM->Options.FunctionSplitting == FunctionSplittingMode::All;
1111
1112 if (SplitFunctionsWithPGO || Opts.split_static_data ||
1113 TM->Options.EnableStaticDataPartitioning) {
1114 const std::string ProfileFile = getFSProfileFile(Opts, TM);
1115 if (!ProfileFile.empty()) {
1116 if (EnableFSDiscriminator) {
1117 addPass(P: createMIRProfileLoaderPass(
1118 File: ProfileFile, RemappingFile: getFSRemappingFile(Opts, TM),
1119 P: sampleprof::FSDiscriminatorPass::PassLast, FS: nullptr));
1120 } else {
1121 // Sample profile is given, but FSDiscriminator is not
1122 // enabled, this may result in performance regression.
1123 WithColor::warning()
1124 << "Using AutoFDO without FSDiscriminator for MFS may regress "
1125 "performance.\n";
1126 }
1127 }
1128 }
1129
1130 if (Opts.split_static_data || TM->Options.EnableStaticDataPartitioning) {
1131 // The static data splitter pass is a machine function pass. and
1132 // static data annotator pass is a module-wide pass. See the file comment
1133 // in StaticDataAnnotator.cpp for the motivation.
1134 addPass(P: createStaticDataSplitterLegacyPass());
1135 addPass(P: createStaticDataAnnotatorLegacyPass());
1136 }
1137 // We run the BasicBlockSections pass if we need BB sections, BB address map,
1138 // or function splitting. With function splitting, functions which are
1139 // eligible for basic-block-sections optimizations (`=all`, or `=list=` with
1140 // function included in the list profile) get that optimization, and the other
1141 // functions are split using their PGO/AutoFDO profile.
1142 if (TM->getBBSectionsType() != llvm::BasicBlockSection::None ||
1143 TM->Options.BBAddrMap || SplitFunctionsWithPGO) {
1144 if (shouldEmitBBHash() || Opts.basic_block_section_match_infer)
1145 addPass(P: llvm::createMachineBlockHashInfoPass());
1146 if (TM->getBBSectionsType() == llvm::BasicBlockSection::List) {
1147 addPass(P: llvm::createBasicBlockSectionsProfileReaderWrapperPass(
1148 Buf: TM->getBBSectionsFuncListBuf()));
1149 if (Opts.basic_block_section_match_infer)
1150 addPass(P: llvm::createBasicBlockMatchingAndInferencePass());
1151 else {
1152 addPass(P: llvm::createBasicBlockPathCloningPass());
1153 addPass(P: llvm::createInsertCodePrefetchPass());
1154 }
1155 }
1156 addPass(P: llvm::createBasicBlockSectionsPass(SplitWithPGO: SplitFunctionsWithPGO));
1157 }
1158
1159 addPostBBSections();
1160
1161 if (!Opts.disable_cfi_fixup && TM->Options.EnableCFIFixup)
1162 addPass(P: createCFIFixupLegacy());
1163
1164 PM->add(P: createStackFrameLayoutAnalysisPass());
1165
1166 // Add passes that directly emit MI after all other MI passes.
1167 addPreEmitPass2();
1168
1169 AddingMachinePasses = false;
1170}
1171
1172/// Add passes that optimize machine instructions in SSA form.
1173void TargetPassConfig::addMachineSSAOptimization() {
1174 // Pre-ra tail duplication.
1175 addPass(PassID: &EarlyTailDuplicateLegacyID);
1176
1177 // Optimize PHIs before DCE: removing dead PHI cycles may make more
1178 // instructions dead.
1179 addPass(PassID: &OptimizePHIsLegacyID);
1180
1181 // This pass merges large allocas. StackSlotColoring is a different pass
1182 // which merges spill slots.
1183 addPass(PassID: &StackColoringLegacyID);
1184
1185 // If the target requests it, assign local variables to stack slots relative
1186 // to one another and simplify frame index references where possible.
1187 addPass(PassID: &LocalStackSlotAllocationID);
1188
1189 // With optimization, dead code should already be eliminated. However
1190 // there is one known exception: lowered code for arguments that are only
1191 // used by tail calls, where the tail calls reuse the incoming stack
1192 // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
1193 addPass(PassID: &DeadMachineInstructionElimID);
1194
1195 // Allow targets to insert passes that improve instruction level parallelism,
1196 // like if-conversion. Such passes will typically need dominator trees and
1197 // loop info, just like LICM and CSE below.
1198 addILPOpts();
1199
1200 addPass(PassID: &EarlyMachineLICMID);
1201 addPass(PassID: &MachineCSELegacyID);
1202
1203 addPass(PassID: &MachineSinkingLegacyID);
1204
1205 addPass(PassID: &PeepholeOptimizerLegacyID);
1206 // Clean-up the dead code that may have been generated by peephole
1207 // rewriting.
1208 addPass(PassID: &DeadMachineInstructionElimID);
1209}
1210
1211//===---------------------------------------------------------------------===//
1212/// Register Allocation Pass Configuration
1213//===---------------------------------------------------------------------===//
1214
1215/// A dummy default pass factory indicates whether the register allocator is
1216/// overridden on the command line.
1217static llvm::once_flag InitializeDefaultRegisterAllocatorFlag;
1218
1219static RegisterRegAlloc
1220defaultRegAlloc("default",
1221 "pick register allocator based on -O option",
1222 useDefaultRegisterAllocator);
1223
1224static void initializeDefaultRegisterAllocatorOnce() {
1225 if (!RegisterRegAlloc::getDefault())
1226 RegisterRegAlloc::setDefault(RegAlloc);
1227}
1228
1229bool TargetPassConfig::getOptimizeRegAlloc() const {
1230 // An explicit -regalloc choice implies its pipeline: only the fast
1231 // allocator uses the unoptimized one.
1232 llvm::call_once(flag&: InitializeDefaultRegisterAllocatorFlag,
1233 F&: initializeDefaultRegisterAllocatorOnce);
1234 RegisterRegAlloc::FunctionPassCtor Ctor = RegisterRegAlloc::getDefault();
1235 if (Ctor != (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator)
1236 return Ctor !=
1237 (RegisterRegAlloc::FunctionPassCtor)&createFastRegisterAllocator;
1238 return getOptLevel() != CodeGenOptLevel::None;
1239}
1240
1241/// Instantiate the default register allocator pass for this target for either
1242/// the optimized or unoptimized allocation path. This will be added to the pass
1243/// manager by addFastRegAlloc in the unoptimized case or addOptimizedRegAlloc
1244/// in the optimized case.
1245///
1246/// A target that uses the standard regalloc pass order for fast or optimized
1247/// allocation may still override this for per-target regalloc
1248/// selection. But -regalloc=... always takes precedence.
1249FunctionPass *TargetPassConfig::createTargetRegisterAllocator(bool Optimized) {
1250 if (Optimized)
1251 return createGreedyRegisterAllocator();
1252 else
1253 return createFastRegisterAllocator();
1254}
1255
1256/// Find and instantiate the register allocation pass requested by this target
1257/// at the current optimization level. Different register allocators are
1258/// defined as separate passes because they may require different analysis.
1259///
1260/// This helper ensures that the regalloc= option is always available,
1261/// even for targets that override the default allocator.
1262///
1263/// FIXME: When MachinePassRegistry register pass IDs instead of function ptrs,
1264/// this can be folded into addPass.
1265FunctionPass *TargetPassConfig::createRegAllocPass(bool Optimized) {
1266 // getOptimizeRegAlloc, called before the pipeline branches, has initialized
1267 // the global default.
1268 RegisterRegAlloc::FunctionPassCtor Ctor = RegisterRegAlloc::getDefault();
1269 if (Ctor != useDefaultRegisterAllocator)
1270 return Ctor();
1271
1272 // With no -regalloc= override, ask the target for a regalloc pass.
1273 return createTargetRegisterAllocator(Optimized);
1274}
1275
1276bool TargetPassConfig::isCustomizedRegAlloc() {
1277 return RegAlloc !=
1278 (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator;
1279}
1280
1281bool TargetPassConfig::addRegAssignAndRewriteFast() {
1282 if (RegAlloc != (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator &&
1283 RegAlloc != (RegisterRegAlloc::FunctionPassCtor)&createFastRegisterAllocator)
1284 reportFatalUsageError(
1285 reason: "Must use fast (default) register allocator for unoptimized regalloc.");
1286
1287 addPass(P: createRegAllocPass(Optimized: false));
1288
1289 // Allow targets to change the register assignments after
1290 // fast register allocation.
1291 addPostFastRegAllocRewrite();
1292 return true;
1293}
1294
1295bool TargetPassConfig::addRegAssignAndRewriteOptimized() {
1296 // Add the selected register allocation pass.
1297 addPass(P: createRegAllocPass(Optimized: true));
1298
1299 // Allow targets to change the register assignments before rewriting.
1300 addPreRewrite();
1301
1302 // Finally rewrite virtual registers.
1303 addPass(PassID: &VirtRegRewriterID);
1304
1305 // Regalloc scoring for ML-driven eviction - noop except when learning a new
1306 // eviction policy.
1307 addPass(P: createRegAllocScoringPass());
1308 return true;
1309}
1310
1311/// Return true if the default global register allocator is in use and
1312/// has not be overriden on the command line with '-regalloc=...'
1313bool TargetPassConfig::usingDefaultRegAlloc() const {
1314 return RegAlloc.getNumOccurrences() == 0;
1315}
1316
1317/// Add the minimum set of target-independent passes that are required for
1318/// register allocation. No coalescing or scheduling.
1319void TargetPassConfig::addFastRegAlloc() {
1320 addPass(PassID: &PHIEliminationID);
1321 if (!TM->enableTiedFastRegAlloc())
1322 addPass(PassID: &TwoAddressInstructionPassID);
1323
1324 addRegAssignAndRewriteFast();
1325}
1326
1327/// Add standard target-independent passes that are tightly coupled with
1328/// optimized register allocation, including coalescing, machine instruction
1329/// scheduling, and register allocation itself.
1330void TargetPassConfig::addOptimizedRegAlloc() {
1331 addPass(PassID: &DetectDeadLanesID);
1332
1333 addPass(PassID: &InitUndefID);
1334
1335 addPass(PassID: &ProcessImplicitDefsID);
1336
1337 // LiveVariables currently requires pure SSA form.
1338 //
1339 // FIXME: Once TwoAddressInstruction pass no longer uses kill flags,
1340 // LiveVariables can be removed completely, and LiveIntervals can be directly
1341 // computed. (We still either need to regenerate kill flags after regalloc, or
1342 // preferably fix the scavenger to not depend on them).
1343 // FIXME: UnreachableMachineBlockElim is a dependant pass of LiveVariables.
1344 // When LiveVariables is removed this has to be removed/moved either.
1345 // Explicit addition of UnreachableMachineBlockElim allows stopping before or
1346 // after it with -stop-before/-stop-after.
1347 addPass(PassID: &UnreachableMachineBlockElimID);
1348 addPass(PassID: &LiveVariablesID);
1349
1350 // Run SSA machine scheduler runs just before PHI elimination.
1351 if (EnableSSAMachineScheduler) {
1352 addPass(PassID: &LiveIntervalsID);
1353 addPass(PassID: &SSAMachineSchedulerID);
1354 }
1355
1356 // LiveIntervals is computed unconditionally before TwoAddressInstruction so
1357 // that pass can rely on it instead of LiveVariables. This is a step toward
1358 // removing LiveVariables entirely.
1359 addPass(PassID: &LiveIntervalsID);
1360
1361 // Edge splitting is smarter with machine loop info.
1362 addPass(PassID: &MachineLoopInfoID);
1363 addPass(PassID: &PHIEliminationID);
1364
1365 addPass(PassID: &TwoAddressInstructionPassID);
1366 addPass(PassID: &RegisterCoalescerID);
1367
1368 // The machine scheduler may accidentally create disconnected components
1369 // when moving subregister definitions around, avoid this by splitting them to
1370 // separate vregs before. Splitting can also improve reg. allocation quality.
1371 addPass(PassID: &RenameIndependentSubregsID);
1372
1373 // PreRA instruction scheduling.
1374 addPass(PassID: &MachineSchedulerID);
1375
1376 if (addRegAssignAndRewriteOptimized()) {
1377 // Perform stack slot coloring and post-ra machine LICM.
1378 addPass(PassID: &StackSlotColoringID);
1379
1380 // Allow targets to expand pseudo instructions depending on the choice of
1381 // registers before MachineCopyPropagation.
1382 addPostRewrite();
1383
1384 // Copy propagate to forward register uses and try to eliminate COPYs that
1385 // were not coalesced.
1386 addPass(PassID: &MachineCopyPropagationID);
1387
1388 // Run post-ra machine LICM to hoist reloads / remats.
1389 //
1390 // FIXME: can this move into MachineLateOptimization?
1391 addPass(PassID: &MachineLICMID);
1392 }
1393}
1394
1395//===---------------------------------------------------------------------===//
1396/// Post RegAlloc Pass Configuration
1397//===---------------------------------------------------------------------===//
1398
1399/// Add passes that optimize machine instructions after register allocation.
1400void TargetPassConfig::addMachineLateOptimization() {
1401 // Cleanup of redundant immediate/address loads.
1402 addPass(PassID: &MachineLateInstrsCleanupID);
1403
1404 // Branch folding must be run after regalloc and prolog/epilog insertion.
1405 addPass(PassID: &BranchFolderPassID);
1406
1407 // Tail duplication.
1408 // Note that duplicating tail just increases code size and degrades
1409 // performance for targets that require Structured Control Flow.
1410 // In addition it can also make CFG irreducible. Thus we disable it.
1411 if (!TM->requiresStructuredCFG())
1412 addPass(PassID: &TailDuplicateLegacyID);
1413
1414 // Copy propagation.
1415 addPass(PassID: &MachineCopyPropagationID);
1416}
1417
1418/// Add standard GC passes.
1419bool TargetPassConfig::addGCPasses() {
1420 addPass(PassID: &GCMachineCodeAnalysisID);
1421 return true;
1422}
1423
1424/// Add standard basic block placement passes.
1425void TargetPassConfig::addBlockPlacement() {
1426 if (EnableFSDiscriminator) {
1427 addPass(P: createMIRAddFSDiscriminatorsPass(
1428 P: sampleprof::FSDiscriminatorPass::Pass2));
1429 const std::string ProfileFile = getFSProfileFile(Opts, TM);
1430 if (!ProfileFile.empty() && !Opts.disable_layout_fsprofile_loader)
1431 addPass(P: createMIRProfileLoaderPass(
1432 File: ProfileFile, RemappingFile: getFSRemappingFile(Opts, TM),
1433 P: sampleprof::FSDiscriminatorPass::Pass2, FS: nullptr));
1434 }
1435 if (addPass(PassID: &MachineBlockPlacementID)) {
1436 // Run a separate pass to collect block placement statistics.
1437 if (Opts.enable_block_placement_stats)
1438 addPass(PassID: &MachineBlockPlacementStatsID);
1439 }
1440}
1441
1442//===---------------------------------------------------------------------===//
1443/// GlobalISel Configuration
1444//===---------------------------------------------------------------------===//
1445bool TargetPassConfig::isGlobalISelAbortEnabled() const {
1446 return TM->Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
1447}
1448
1449bool TargetPassConfig::reportDiagnosticWhenGlobalISelFallback() const {
1450 return TM->Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag;
1451}
1452
1453std::unique_ptr<CSEConfigBase> TargetPassConfig::getCSEConfig() const {
1454 return std::make_unique<CSEConfigBase>();
1455}
1456