1//===- TargetPassConfig.cpp - Target independent code generation passes ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines interfaces to access the target independent code
10// generation passes provided by the LLVM backend.
11//
12//===---------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/TargetPassConfig.h"
15#include "CodeGenOptions.h"
16#include "llvm/ADT/DenseMap.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/StringRef.h"
19#include "llvm/Analysis/BasicAliasAnalysis.h"
20#include "llvm/Analysis/CallGraphSCCPass.h"
21#include "llvm/Analysis/ScopedNoAliasAA.h"
22#include "llvm/Analysis/TargetTransformInfo.h"
23#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
24#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
25#include "llvm/CodeGen/CSEConfigBase.h"
26#include "llvm/CodeGen/CodeGenTargetMachineImpl.h"
27#include "llvm/CodeGen/MachineBlockHashInfo.h"
28#include "llvm/CodeGen/MachineFunctionPass.h"
29#include "llvm/CodeGen/MachinePassRegistry.h"
30#include "llvm/CodeGen/Passes.h"
31#include "llvm/CodeGen/RegAllocRegistry.h"
32#include "llvm/IR/IRPrintingPasses.h"
33#include "llvm/IR/LegacyPassManager.h"
34#include "llvm/IR/PassInstrumentation.h"
35#include "llvm/IR/Verifier.h"
36#include "llvm/InitializePasses.h"
37#include "llvm/MC/MCAsmInfo.h"
38#include "llvm/MC/MCTargetOptions.h"
39#include "llvm/Pass.h"
40#include "llvm/Support/CodeGen.h"
41#include "llvm/Support/CommandLine.h"
42#include "llvm/Support/Compiler.h"
43#include "llvm/Support/Debug.h"
44#include "llvm/Support/Discriminator.h"
45#include "llvm/Support/ErrorHandling.h"
46#include "llvm/Support/SaveAndRestore.h"
47#include "llvm/Support/Threading.h"
48#include "llvm/Support/VirtualFileSystem.h"
49#include "llvm/Support/WithColor.h"
50#include "llvm/Target/CGPassBuilderOption.h"
51#include "llvm/Target/TargetMachine.h"
52#include "llvm/Transforms/ObjCARC.h"
53#include "llvm/Transforms/Scalar.h"
54#include "llvm/Transforms/Utils.h"
55#include <cassert>
56#include <optional>
57#include <string>
58
59using namespace llvm;
60
61static const CodeGenOptions &getOpts(const TargetMachine &TM) {
62 return cl::LibraryOptionsContext::get<CodeGenOptions>(
63 C: TM.Options.MCOptions.LibraryOptions);
64}
65
66namespace {
67class TriggerCrashFunctionLegacyPass : public FunctionPass {
68public:
69 static char ID;
70 TriggerCrashFunctionLegacyPass() : FunctionPass(ID) {}
71 bool runOnFunction(Function &F) override {
72 abort();
73 return false;
74 }
75 StringRef getPassName() const override { return "TriggerCrashFunctionPass"; }
76};
77} // namespace
78
79char TriggerCrashFunctionLegacyPass::ID = 0;
80
81// Enable or disable the MachineOutliner.
82static cl::opt<RunOutliner> EnableMachineOutliner(
83 "enable-machine-outliner", cl::desc("Enable the machine outliner"),
84 cl::Hidden, cl::ValueOptional, cl::init(Val: RunOutliner::TargetDefault),
85 cl::values(
86 clEnumValN(RunOutliner::AlwaysOutline, "always",
87 "Run on all functions guaranteed to be beneficial"),
88 clEnumValN(RunOutliner::OptimisticPGO, "optimistic-pgo",
89 "Outline cold code only. If a code block does not have "
90 "profile data, optimistically assume it is cold."),
91 clEnumValN(RunOutliner::ConservativePGO, "conservative-pgo",
92 "Outline cold code only. If a code block does not have "
93 "profile, data, conservatively assume it is hot."),
94 clEnumValN(RunOutliner::NeverOutline, "never", "Disable all outlining"),
95 // Sentinel value for unspecified option.
96 clEnumValN(RunOutliner::AlwaysOutline, "", "")));
97
98/// Option names for limiting the codegen pipeline.
99/// Those are used in error reporting and we didn't want
100/// to duplicate their names all over the place.
101static const char StartAfterOptName[] = "start-after";
102static const char StartBeforeOptName[] = "start-before";
103static const char StopAfterOptName[] = "stop-after";
104static const char StopBeforeOptName[] = "stop-before";
105
106/// Allow standard passes to be disabled by command line options. This supports
107/// simple binary flags that either suppress the pass or do nothing.
108/// i.e. -disable-mypass=false has no effect.
109/// These should be converted to boolOrDefault in order to use applyOverride.
110static IdentifyingPassPtr applyDisable(IdentifyingPassPtr PassID,
111 bool Override) {
112 if (Override)
113 return IdentifyingPassPtr();
114 return PassID;
115}
116
117/// Allow standard passes to be disabled by the command line, regardless of who
118/// is adding the pass.
119///
120/// StandardID is the pass identified in the standard pass pipeline and provided
121/// to addPass(). It may be a target-specific ID in the case that the target
122/// directly adds its own pass, but in that case we harmlessly fall through.
123///
124/// TargetID is the pass that the target has configured to override StandardID.
125///
126/// StandardID may be a pseudo ID. In that case TargetID is the name of the real
127/// pass to run. This allows multiple options to control a single pass depending
128/// on where in the pipeline that pass is added.
129static IdentifyingPassPtr overridePass(const CodeGenOptions &Opts,
130 AnalysisID StandardID,
131 IdentifyingPassPtr TargetID) {
132 if (StandardID == &PostRASchedulerID)
133 return applyDisable(PassID: TargetID, Override: Opts.disable_post_ra);
134
135 if (StandardID == &BranchFolderPassID)
136 return applyDisable(PassID: TargetID, Override: Opts.disable_branch_fold);
137
138 if (StandardID == &TailDuplicateLegacyID)
139 return applyDisable(PassID: TargetID, Override: Opts.disable_tail_duplicate);
140
141 if (StandardID == &EarlyTailDuplicateLegacyID)
142 return applyDisable(PassID: TargetID, Override: Opts.disable_early_taildup);
143
144 if (StandardID == &MachineBlockPlacementID)
145 return applyDisable(PassID: TargetID, Override: Opts.disable_block_placement);
146
147 if (StandardID == &StackSlotColoringID)
148 return applyDisable(PassID: TargetID, Override: Opts.disable_ssc);
149
150 if (StandardID == &DeadMachineInstructionElimID)
151 return applyDisable(PassID: TargetID, Override: Opts.disable_machine_dce);
152
153 if (StandardID == &EarlyIfConverterLegacyID)
154 return applyDisable(PassID: TargetID, Override: Opts.disable_early_ifcvt);
155
156 if (StandardID == &EarlyMachineLICMID)
157 return applyDisable(PassID: TargetID, Override: Opts.disable_machine_licm);
158
159 if (StandardID == &MachineCSELegacyID)
160 return applyDisable(PassID: TargetID, Override: Opts.disable_machine_cse);
161
162 if (StandardID == &MachineLICMID)
163 return applyDisable(PassID: TargetID, Override: Opts.disable_postra_machine_licm);
164
165 if (StandardID == &MachineSinkingLegacyID)
166 return applyDisable(PassID: TargetID, Override: Opts.disable_machine_sink);
167
168 if (StandardID == &PostRAMachineSinkingID)
169 return applyDisable(PassID: TargetID, Override: Opts.disable_postra_machine_sink);
170
171 if (StandardID == &MachineCopyPropagationID)
172 return applyDisable(PassID: TargetID, Override: Opts.disable_copyprop);
173
174 return TargetID;
175}
176
177// Find the FSProfile file name. The internal option takes the precedence
178// before getting from TargetMachine.
179static std::string getFSProfileFile(const CodeGenOptions &Opts,
180 const TargetMachine *TM) {
181 if (!Opts.fs_profile_file.empty())
182 return Opts.fs_profile_file.str();
183 const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
184 if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
185 return std::string();
186 return PGOOpt->ProfileFile;
187}
188
189// Find the Profile remapping file name. The internal option takes the
190// precedence before getting from TargetMachine.
191static std::string getFSRemappingFile(const CodeGenOptions &Opts,
192 const TargetMachine *TM) {
193 if (!Opts.fs_remapping_file.empty())
194 return Opts.fs_remapping_file.str();
195 const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
196 if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
197 return std::string();
198 return PGOOpt->ProfileRemappingFile;
199}
200
201//===---------------------------------------------------------------------===//
202/// TargetPassConfig
203//===---------------------------------------------------------------------===//
204
205INITIALIZE_PASS(TargetPassConfig, "targetpassconfig",
206 "Target Pass Configuration", false, false)
207char TargetPassConfig::ID = 0;
208
209namespace {
210
211struct InsertedPass {
212 AnalysisID TargetPassID;
213 IdentifyingPassPtr InsertedPassID;
214
215 InsertedPass(AnalysisID TargetPassID, IdentifyingPassPtr InsertedPassID)
216 : TargetPassID(TargetPassID), InsertedPassID(InsertedPassID) {}
217
218 Pass *getInsertedPass() const {
219 assert(InsertedPassID.isValid() && "Illegal Pass ID!");
220 if (InsertedPassID.isInstance())
221 return InsertedPassID.getInstance();
222 Pass *NP = Pass::createPass(ID: InsertedPassID.getID());
223 assert(NP && "Pass ID not registered");
224 return NP;
225 }
226};
227
228} // end anonymous namespace
229
230namespace llvm {
231
232class PassConfigImpl {
233public:
234 // List of passes explicitly substituted by this target. Normally this is
235 // empty, but it is a convenient way to suppress or replace specific passes
236 // that are part of a standard pass pipeline without overridding the entire
237 // pipeline. This mechanism allows target options to inherit a standard pass's
238 // user interface. For example, a target may disable a standard pass by
239 // default by substituting a pass ID of zero, and the user may still enable
240 // that standard pass with an explicit command line option.
241 DenseMap<AnalysisID,IdentifyingPassPtr> TargetPasses;
242
243 /// Store the pairs of <AnalysisID, AnalysisID> of which the second pass
244 /// is inserted after each instance of the first one.
245 SmallVector<InsertedPass, 4> InsertedPasses;
246};
247
248} // end namespace llvm
249
250// Out of line virtual method.
251TargetPassConfig::~TargetPassConfig() {
252 delete Impl;
253}
254
255static const PassInfo *getPassInfo(StringRef PassName) {
256 if (PassName.empty())
257 return nullptr;
258
259 const PassRegistry &PR = *PassRegistry::getPassRegistry();
260 const PassInfo *PI = PR.getPassInfo(Arg: PassName);
261 if (!PI)
262 reportFatalUsageError(reason: Twine('\"') + Twine(PassName) +
263 Twine("\" pass is not registered."));
264 return PI;
265}
266
267static AnalysisID getPassIDFromName(StringRef PassName) {
268 const PassInfo *PI = getPassInfo(PassName);
269 return PI ? PI->getTypeInfo() : nullptr;
270}
271
272static std::pair<StringRef, unsigned>
273getPassNameAndInstanceNum(StringRef PassName) {
274 StringRef Name, InstanceNumStr;
275 std::tie(args&: Name, args&: InstanceNumStr) = PassName.split(Separator: ',');
276
277 unsigned InstanceNum = 0;
278 if (!InstanceNumStr.empty() && InstanceNumStr.getAsInteger(Radix: 10, Result&: InstanceNum))
279 reportFatalUsageError(reason: "invalid pass instance specifier " + PassName);
280
281 return std::make_pair(x&: Name, y&: InstanceNum);
282}
283
284void TargetPassConfig::setStartStopPasses() {
285 StringRef StartBeforeName;
286 std::tie(args&: StartBeforeName, args&: StartBeforeInstanceNum) =
287 getPassNameAndInstanceNum(PassName: Opts.start_before);
288
289 StringRef StartAfterName;
290 std::tie(args&: StartAfterName, args&: StartAfterInstanceNum) =
291 getPassNameAndInstanceNum(PassName: Opts.start_after);
292
293 StringRef StopBeforeName;
294 std::tie(args&: StopBeforeName, args&: StopBeforeInstanceNum) =
295 getPassNameAndInstanceNum(PassName: Opts.stop_before);
296
297 StringRef StopAfterName;
298 std::tie(args&: StopAfterName, args&: StopAfterInstanceNum) =
299 getPassNameAndInstanceNum(PassName: Opts.stop_after);
300
301 StartBefore = getPassIDFromName(PassName: StartBeforeName);
302 StartAfter = getPassIDFromName(PassName: StartAfterName);
303 StopBefore = getPassIDFromName(PassName: StopBeforeName);
304 StopAfter = getPassIDFromName(PassName: StopAfterName);
305 if (StartBefore && StartAfter)
306 reportFatalUsageError(reason: Twine(StartBeforeOptName) + Twine(" and ") +
307 Twine(StartAfterOptName) + Twine(" specified!"));
308 if (StopBefore && StopAfter)
309 reportFatalUsageError(reason: Twine(StopBeforeOptName) + Twine(" and ") +
310 Twine(StopAfterOptName) + Twine(" specified!"));
311 Started = (StartAfter == nullptr) && (StartBefore == nullptr);
312}
313
314static cl::boolOrDefault toBoolOrDefault(BoolOrDefault B) {
315 if (B == BoolOrDefault::Default)
316 return cl::boolOrDefault::BOU_UNSET;
317 return B == BoolOrDefault::True ? cl::boolOrDefault::BOU_TRUE
318 : cl::boolOrDefault::BOU_FALSE;
319}
320
321static MachineVerifierMode
322computeMachineVerifierMode(const CodeGenOptions &Opts, bool VerifyEach) {
323 if (VerifyEach)
324 return MachineVerifierMode::Each;
325 return Opts.disable_mir_output_verify ? MachineVerifierMode::None
326 : MachineVerifierMode::End;
327}
328
329CGPassBuilderOption llvm::getCGPassBuilderOption(const TargetMachine &TM) {
330 const CodeGenOptions &Opts = getOpts(TM);
331 CGPassBuilderOption Opt;
332 Opt.EnableGlobalISelAbort = Opts.global_isel_abort;
333 if (Opts.enable_ipra != BoolOrDefault::Default)
334 Opt.EnableIPRA = Opts.enable_ipra == BoolOrDefault::True;
335 Opt.EnableFastISelOption = toBoolOrDefault(B: Opts.fast_isel);
336 Opt.EnableRegAllocFastTied = toBoolOrDefault(B: Opts.regalloc_fast_tied);
337 Opt.EnableMachinePipeliner = toBoolOrDefault(B: Opts.enable_pipeliner);
338 Opt.EnableGlobalISelOption = toBoolOrDefault(B: Opts.global_isel);
339 Opt.VerifyMachineCode = computeMachineVerifierMode(
340 Opts, VerifyEach: valueOr(X: Opts.verify_machineinstrs, Default: false));
341 Opt.DisableAtExitBasedGlobalDtorLowering =
342 Opts.disable_atexit_based_global_dtor_lowering;
343 Opt.DisableExpandReductions = Opts.disable_expand_reductions;
344 Opt.PrintAfterISel = Opts.print_after_isel;
345 Opt.FSProfileFile = Opts.fs_profile_file.str();
346 Opt.EnableGCEmptyBlocks = Opts.enable_gc_empty_basic_blocks;
347 Opt.EnableBlockPlacementStats = Opts.enable_block_placement_stats;
348 Opt.EnableGlobalMergeFunc = Opts.enable_global_merge_func;
349 Opt.EnableImplicitNullChecks = Opts.enable_implicit_null_checks;
350 Opt.EnableMachineOutliner = EnableMachineOutliner;
351 Opt.MISchedPostRA = Opts.misched_postra;
352 Opt.DisableLSR = Opts.disable_lsr;
353 Opt.DisableConstantHoisting = Opts.disable_constant_hoisting;
354 Opt.DisableCGP = Opts.disable_cgp;
355 Opt.DisablePartialLibcallInlining = Opts.disable_partial_libcall_inlining;
356 Opt.DisableSelectOptimize = Opts.disable_select_optimize;
357 Opt.PrintISelInput = Opts.print_isel_input;
358 Opt.PrintRegUsage = Opts.print_regusage;
359 Opt.DebugifyAndStripAll = toBoolOrDefault(B: Opts.debugify_and_strip_all_safe);
360 Opt.DebugifyCheckAndStripAll =
361 toBoolOrDefault(B: Opts.debugify_check_and_strip_all_safe);
362 Opt.DisableRAFSProfileLoader = Opts.disable_ra_fsprofile_loader;
363 Opt.DisableCFIFixup = Opts.disable_cfi_fixup;
364 return Opt;
365}
366
367void llvm::registerCodeGenCallback(PassInstrumentationCallbacks &PIC,
368 TargetMachine &TM) {
369 const CodeGenOptions &Opts = getOpts(TM);
370
371 // Register a callback for disabling passes.
372 PIC.registerShouldRunOptionalPassCallback(C: [&Opts](StringRef P, IRUnitRef) {
373
374#define DISABLE_PASS(Option, Name) \
375 if (Option && P.contains(#Name)) \
376 return false;
377 DISABLE_PASS(Opts.disable_block_placement, MachineBlockPlacementPass)
378 DISABLE_PASS(Opts.disable_branch_fold, BranchFolderPass)
379 DISABLE_PASS(Opts.disable_copyprop, MachineCopyPropagationPass)
380 DISABLE_PASS(Opts.disable_early_ifcvt, EarlyIfConverterLegacyPass)
381 DISABLE_PASS(Opts.disable_early_taildup, EarlyTailDuplicatePass)
382 DISABLE_PASS(Opts.disable_machine_cse, MachineCSELegacyPass)
383 DISABLE_PASS(Opts.disable_machine_dce, DeadMachineInstructionElimPass)
384 DISABLE_PASS(Opts.disable_machine_licm, EarlyMachineLICMPass)
385 DISABLE_PASS(Opts.disable_machine_sink, MachineSinkingPass)
386 DISABLE_PASS(Opts.disable_postra_machine_licm, MachineLICMPass)
387 DISABLE_PASS(Opts.disable_postra_machine_sink, PostRAMachineSinkingPass)
388 DISABLE_PASS(Opts.disable_post_ra, PostRASchedulerPass)
389 DISABLE_PASS(Opts.disable_ssc, StackSlotColoringPass)
390 DISABLE_PASS(Opts.disable_tail_duplicate, TailDuplicatePass)
391
392 return true;
393 });
394}
395
396Expected<TargetPassConfig::StartStopInfo>
397TargetPassConfig::getStartStopInfo(const TargetMachine &TM,
398 PassInstrumentationCallbacks &PIC) {
399 const CodeGenOptions &Opts = getOpts(TM);
400 auto [StartBefore, StartBeforeInstanceNum] =
401 getPassNameAndInstanceNum(PassName: Opts.start_before);
402 auto [StartAfter, StartAfterInstanceNum] =
403 getPassNameAndInstanceNum(PassName: Opts.start_after);
404 auto [StopBefore, StopBeforeInstanceNum] =
405 getPassNameAndInstanceNum(PassName: Opts.stop_before);
406 auto [StopAfter, StopAfterInstanceNum] =
407 getPassNameAndInstanceNum(PassName: Opts.stop_after);
408
409 if (!StartBefore.empty() && !StartAfter.empty())
410 return make_error<StringError>(
411 Args: Twine(StartBeforeOptName) + " and " + StartAfterOptName + " specified!",
412 Args: std::make_error_code(e: std::errc::invalid_argument));
413 if (!StopBefore.empty() && !StopAfter.empty())
414 return make_error<StringError>(
415 Args: Twine(StopBeforeOptName) + " and " + StopAfterOptName + " specified!",
416 Args: std::make_error_code(e: std::errc::invalid_argument));
417
418 StartStopInfo Result;
419 Result.StartPass = StartBefore.empty() ? StartAfter : StartBefore;
420 Result.StopPass = StopBefore.empty() ? StopAfter : StopBefore;
421 Result.StartInstanceNum =
422 StartBefore.empty() ? StartAfterInstanceNum : StartBeforeInstanceNum;
423 Result.StopInstanceNum =
424 StopBefore.empty() ? StopAfterInstanceNum : StopBeforeInstanceNum;
425 Result.StartAfter = !StartAfter.empty();
426 Result.StopAfter = !StopAfter.empty();
427 Result.StartInstanceNum += Result.StartInstanceNum == 0;
428 Result.StopInstanceNum += Result.StopInstanceNum == 0;
429 return Result;
430}
431
432// Out of line constructor provides default values for pass options and
433// registers all common codegen passes.
434TargetPassConfig::TargetPassConfig(TargetMachine &TM, PassManagerBase &PM)
435 : ImmutablePass(ID), PM(&PM), Opts(getOpts(TM)), TM(&TM) {
436 Impl = new PassConfigImpl();
437
438 PassRegistry &PR = *PassRegistry::getPassRegistry();
439 // Register all target independent codegen passes to activate their PassIDs,
440 // including this pass itself.
441 initializeCodeGen(PR);
442
443 initializeLibcallLoweringInfoWrapperPass(PR);
444
445 // Also register alias analysis passes required by codegen passes.
446 initializeBasicAAWrapperPassPass(PR);
447 initializeAAResultsWrapperPassPass(PR);
448
449 if (Opts.enable_ipra != BoolOrDefault::Default) {
450 TM.Options.EnableIPRA = Opts.enable_ipra == BoolOrDefault::True;
451 } else {
452 // If not explicitly specified, use target default.
453 TM.Options.EnableIPRA |= TM.useIPRA();
454 }
455
456 if (TM.Options.EnableIPRA)
457 setRequiresCodeGenSCCOrder();
458
459 if (Opts.regalloc_fast_tied != BoolOrDefault::Default)
460 TM.setEnableTiedFastRegAlloc(Opts.regalloc_fast_tied ==
461 BoolOrDefault::True);
462
463 if (Opts.global_isel_abort)
464 TM.Options.GlobalISelAbort = *Opts.global_isel_abort;
465
466 bool VerifyEach = valueOr(X: Opts.verify_machineinstrs, Default: false);
467#ifdef EXPENSIVE_CHECKS
468 if (Opts.verify_machineinstrs == BoolOrDefault::Default)
469 VerifyEach = TM.isMachineVerifierClean();
470#endif
471 VerifyMode = computeMachineVerifierMode(Opts, VerifyEach);
472
473 if (Opts.function_splitting)
474 TM.Options.FunctionSplitting = *Opts.function_splitting;
475
476 setStartStopPasses();
477}
478
479CodeGenOptLevel TargetPassConfig::getOptLevel() const {
480 return TM->getOptLevel();
481}
482
483/// Insert InsertedPassID pass after TargetPassID.
484void TargetPassConfig::insertPass(AnalysisID TargetPassID,
485 IdentifyingPassPtr InsertedPassID) {
486 assert(((!InsertedPassID.isInstance() &&
487 TargetPassID != InsertedPassID.getID()) ||
488 (InsertedPassID.isInstance() &&
489 TargetPassID != InsertedPassID.getInstance()->getPassID())) &&
490 "Insert a pass after itself!");
491 Impl->InsertedPasses.emplace_back(Args&: TargetPassID, Args&: InsertedPassID);
492}
493
494/// createPassConfig - Create a pass configuration object to be used by
495/// addPassToEmitX methods for generating a pipeline of CodeGen passes.
496///
497/// Targets may override this to extend TargetPassConfig.
498TargetPassConfig *
499CodeGenTargetMachineImpl::createPassConfig(PassManagerBase &PM) {
500 return new TargetPassConfig(*this, PM);
501}
502
503template <> Pass *llvm::callDefaultCtor<TargetPassConfig>() {
504 reportFatalUsageError(reason: "trying to construct TargetPassConfig without a target "
505 "machine. Scheduling a CodeGen pass without a target "
506 "triple set?");
507}
508
509bool TargetPassConfig::willCompleteCodeGenPipeline() const {
510 return Opts.stop_before.empty() && Opts.stop_after.empty();
511}
512
513bool TargetPassConfig::hasLimitedCodeGenPipeline() const {
514 return !Opts.start_before.empty() || !Opts.start_after.empty() ||
515 !willCompleteCodeGenPipeline();
516}
517
518static std::string formatLimitReason(StringRef StartAfter,
519 StringRef StartBefore, StringRef StopAfter,
520 StringRef StopBefore) {
521 std::string Res;
522 StringRef PassNames[] = {StartAfter, StartBefore, StopAfter, StopBefore};
523 static const char *OptNames[] = {StartAfterOptName, StartBeforeOptName,
524 StopAfterOptName, StopBeforeOptName};
525 bool IsFirst = true;
526 for (int Idx = 0; Idx < 4; ++Idx)
527 if (!PassNames[Idx].empty()) {
528 if (!IsFirst)
529 Res += " and ";
530 IsFirst = false;
531 Res += OptNames[Idx];
532 }
533 return Res;
534}
535
536std::string TargetPassConfig::getLimitedCodeGenPipelineReason() const {
537 return formatLimitReason(StartAfter: Opts.start_after, StartBefore: Opts.start_before, StopAfter: Opts.stop_after,
538 StopBefore: Opts.stop_before);
539}
540
541std::string TargetPassConfig::StartStopInfo::getLimitReason() const {
542 return formatLimitReason(
543 StartAfter: StartAfter ? StartPass : "", StartBefore: StartAfter ? "" : StartPass,
544 StopAfter: StopAfter ? StopPass : "", StopBefore: StopAfter ? "" : StopPass);
545}
546
547// Helper to verify the analysis is really immutable.
548void TargetPassConfig::setOpt(bool &Opt, bool Val) {
549 assert(!Initialized && "PassConfig is immutable");
550 Opt = Val;
551}
552
553void TargetPassConfig::substitutePass(AnalysisID StandardID,
554 IdentifyingPassPtr TargetID) {
555 Impl->TargetPasses[StandardID] = TargetID;
556}
557
558IdentifyingPassPtr TargetPassConfig::getPassSubstitution(AnalysisID ID) const {
559 DenseMap<AnalysisID, IdentifyingPassPtr>::const_iterator
560 I = Impl->TargetPasses.find(Val: ID);
561 if (I == Impl->TargetPasses.end())
562 return ID;
563 return I->second;
564}
565
566bool TargetPassConfig::isPassSubstitutedOrOverridden(AnalysisID ID) const {
567 IdentifyingPassPtr TargetID = getPassSubstitution(ID);
568 IdentifyingPassPtr FinalPtr = overridePass(Opts, StandardID: ID, TargetID);
569 return !FinalPtr.isValid() || FinalPtr.isInstance() ||
570 FinalPtr.getID() != ID;
571}
572
573/// Add a pass to the PassManager if that pass is supposed to be run. If the
574/// Started/Stopped flags indicate either that the compilation should start at
575/// a later pass or that it should stop after an earlier pass, then do not add
576/// the pass. Finally, compare the current pass against the StartAfter
577/// and StopAfter options and change the Started/Stopped flags accordingly.
578void TargetPassConfig::addPass(Pass *P) {
579 assert(!Initialized && "PassConfig is immutable");
580
581 // Cache the Pass ID here in case the pass manager finds this pass is
582 // redundant with ones already scheduled / available, and deletes it.
583 // Fundamentally, once we add the pass to the manager, we no longer own it
584 // and shouldn't reference it.
585 AnalysisID PassID = P->getPassID();
586
587 if (StartBefore == PassID && StartBeforeCount++ == StartBeforeInstanceNum)
588 Started = true;
589 if (StopBefore == PassID && StopBeforeCount++ == StopBeforeInstanceNum)
590 Stopped = true;
591 if (Started && !Stopped) {
592 if (AddingMachinePasses) {
593 // Construct banner message before PM->add() as that may delete the pass.
594 std::string Banner =
595 std::string("After ") + std::string(P->getPassName());
596 addMachinePrePasses();
597 PM->add(P);
598 addMachinePostPasses(Banner);
599 } else {
600 PM->add(P);
601 }
602
603 // Add the passes after the pass P if there is any.
604 for (const auto &IP : Impl->InsertedPasses)
605 if (IP.TargetPassID == PassID)
606 addPass(P: IP.getInsertedPass());
607 } else {
608 delete P;
609 }
610
611 if (StopAfter == PassID && StopAfterCount++ == StopAfterInstanceNum)
612 Stopped = true;
613
614 if (StartAfter == PassID && StartAfterCount++ == StartAfterInstanceNum)
615 Started = true;
616 if (Stopped && !Started)
617 reportFatalUsageError(reason: "Cannot stop compilation after pass that is not run");
618}
619
620/// Add a CodeGen pass at this point in the pipeline after checking for target
621/// and command line overrides.
622///
623/// addPass cannot return a pointer to the pass instance because is internal the
624/// PassManager and the instance we create here may already be freed.
625AnalysisID TargetPassConfig::addPass(AnalysisID PassID) {
626 IdentifyingPassPtr TargetID = getPassSubstitution(ID: PassID);
627 IdentifyingPassPtr FinalPtr = overridePass(Opts, StandardID: PassID, TargetID);
628 if (!FinalPtr.isValid())
629 return nullptr;
630
631 Pass *P;
632 if (FinalPtr.isInstance())
633 P = FinalPtr.getInstance();
634 else {
635 P = Pass::createPass(ID: FinalPtr.getID());
636 if (!P)
637 llvm_unreachable("Pass ID not registered");
638 }
639 AnalysisID FinalID = P->getPassID();
640 addPass(P); // Ends the lifetime of P.
641
642 return FinalID;
643}
644
645void TargetPassConfig::printAndVerify(const std::string &Banner) {
646 addPrintPass(Banner);
647 addVerifyPass(Banner);
648}
649
650void TargetPassConfig::addPrintPass(const std::string &Banner) {
651 if (Opts.print_after_isel)
652 PM->add(P: createMachineFunctionPrinterPass(OS&: dbgs(), Banner));
653}
654
655void TargetPassConfig::addVerifyPass(const std::string &Banner) {
656 if (VerifyMode == MachineVerifierMode::Each)
657 PM->add(P: createMachineVerifierPass(Banner));
658}
659
660void TargetPassConfig::addDebugifyPass() {
661 PM->add(P: createDebugifyMachineModulePass());
662}
663
664void TargetPassConfig::addStripDebugPass() {
665 PM->add(P: createStripDebugMachineModuleLegacyPass(/*OnlyDebugified=*/true));
666}
667
668void TargetPassConfig::addCheckDebugPass() {
669 PM->add(P: createCheckDebugMachineModuleLegacyPass());
670}
671
672void TargetPassConfig::addMachinePrePasses(bool AllowDebugify) {
673 if (AllowDebugify && DebugifyIsSafe &&
674 (valueOr(X: Opts.debugify_and_strip_all_safe, Default: false) ||
675 valueOr(X: Opts.debugify_check_and_strip_all_safe, Default: false)))
676 addDebugifyPass();
677}
678
679void TargetPassConfig::addMachinePostPasses(const std::string &Banner) {
680 if (DebugifyIsSafe) {
681 if (valueOr(X: Opts.debugify_check_and_strip_all_safe, Default: false)) {
682 addCheckDebugPass();
683 addStripDebugPass();
684 } else if (valueOr(X: Opts.debugify_and_strip_all_safe, Default: false))
685 addStripDebugPass();
686 }
687 addVerifyPass(Banner);
688}
689
690/// Add common target configurable passes that perform LLVM IR to IR transforms
691/// following machine independent optimization.
692void TargetPassConfig::addIRPasses() {
693 // Before running any passes, run the verifier to determine if the input
694 // coming from the front-end and/or optimizer is valid.
695 if (!DisableVerify)
696 addPass(P: createVerifierPass());
697
698 if (getOptLevel() != CodeGenOptLevel::None) {
699 // Basic AliasAnalysis support.
700 // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that
701 // BasicAliasAnalysis wins if they disagree. This is intended to help
702 // support "obvious" type-punning idioms.
703 addPass(P: createTypeBasedAAWrapperPass());
704 addPass(P: createScopedNoAliasAAWrapperPass());
705 addPass(P: createBasicAAWrapperPass());
706
707 // Run loop strength reduction before anything else.
708 if (EnableLoopStrengthReduce && !Opts.disable_lsr) {
709 addPass(P: createCanonicalizeFreezeInLoopsPass());
710 addPass(P: createLoopStrengthReducePass());
711 if (EnableLoopTermFold)
712 addPass(P: createLoopTermFoldPass());
713 }
714 }
715
716 // Run GC lowering passes for builtin collectors
717 // TODO: add a pass insertion point here
718 addPass(PassID: &GCLoweringID);
719 addPass(PassID: &ShadowStackGCLoweringID);
720
721 // For MachO, lower @llvm.global_dtors into @llvm.global_ctors with
722 // __cxa_atexit() calls to avoid emitting the deprecated __mod_term_func.
723 if (TM->getTargetTriple().isOSBinFormatMachO() &&
724 !Opts.disable_atexit_based_global_dtor_lowering)
725 addPass(P: createLowerGlobalDtorsLegacyPass());
726
727 // Make sure that no unreachable blocks are instruction selected.
728 addPass(P: createUnreachableBlockEliminationPass());
729
730 // Prepare expensive constants for SelectionDAG.
731 if (getOptLevel() != CodeGenOptLevel::None && !Opts.disable_constant_hoisting)
732 addPass(P: createConstantHoistingPass());
733
734 if (getOptLevel() != CodeGenOptLevel::None &&
735 !Opts.disable_replace_with_vec_lib)
736 addPass(P: createReplaceWithVeclibLegacyPass());
737
738 if (getOptLevel() != CodeGenOptLevel::None &&
739 !Opts.disable_partial_libcall_inlining)
740 addPass(P: createPartiallyInlineLibCallsPass());
741
742 // Instrument function entry after all inlining.
743 addPass(P: createPostInlineEntryExitInstrumenterPass());
744
745 // Add scalarization of target's unsupported masked memory intrinsics pass.
746 // the unsupported intrinsic will be replaced with a chain of basic blocks,
747 // that stores/loads element one-by-one if the appropriate mask bit is set.
748 addPass(P: createScalarizeMaskedMemIntrinLegacyPass());
749
750 // Expand reduction intrinsics into shuffle sequences if the target wants to.
751 // Allow disabling it for testing purposes.
752 if (!Opts.disable_expand_reductions)
753 addPass(P: createExpandReductionsPass());
754
755 // Convert conditional moves to conditional jumps when profitable.
756 if (getOptLevel() != CodeGenOptLevel::None && !Opts.disable_select_optimize)
757 addPass(P: createSelectOptimizePass());
758
759 if (Opts.enable_global_merge_func)
760 addPass(P: createGlobalMergeFuncPass());
761
762 if (TM->getTargetTriple().isOSWindows())
763 addPass(P: createWindowsSecureHotPatchingPass());
764}
765
766/// Turn exception handling constructs into something the code generators can
767/// handle.
768void TargetPassConfig::addPassesToHandleExceptions() {
769 // The module's exception model is not known when the pipeline is built, so
770 // schedule the preparation pass for every model the target supports. Each
771 // pass does nothing for other models.
772 bool HasEHPrep = false;
773
774 // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both
775 // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise,
776 // catch info can get misplaced when a selector ends up more than one block
777 // removed from the parent invoke(s). This could happen when a landing
778 // pad is shared by multiple invokes and is also a target of a normal
779 // edge from elsewhere.
780 if (TM->supportsExceptionModel(Model: ExceptionHandling::SjLj)) {
781 addPass(P: createSjLjEHPreparePass());
782 HasEHPrep = true;
783 }
784
785 // We support using both GCC-style and MSVC-style exceptions on Windows, so
786 // WinEHPrepare runs in addition to DwarfEHPrepare below. Each pass will only
787 // actually run if it recognizes the personality function.
788 //
789 // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs
790 // on catchpads and cleanuppads because it does not outline them into
791 // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we
792 // should remove PHIs there. WinEHPrepare derives this from the Wasm
793 // personality, so no explicit flag is needed here.
794 if (TM->supportsExceptionModel(Model: ExceptionHandling::WinEH) ||
795 TM->supportsExceptionModel(Model: ExceptionHandling::Wasm)) {
796 addPass(P: createWinEHPass());
797 HasEHPrep = true;
798 }
799
800 if (TM->supportsExceptionModel(Model: ExceptionHandling::Wasm))
801 addPass(P: createWasmEHPass());
802
803 if (TM->supportsExceptionModel(Model: ExceptionHandling::DwarfCFI)) {
804 addPass(P: createDwarfEHPass(OptLevel: getOptLevel()));
805 HasEHPrep = true;
806 }
807
808 // If the target supports no exception model, nothing else lowers invokes.
809 if (!HasEHPrep) {
810 addPass(P: createLowerInvokePass());
811
812 // The lower invoke pass may create unreachable code. Remove it.
813 addPass(P: createUnreachableBlockEliminationPass());
814 }
815}
816
817/// Add pass to prepare the LLVM IR for code generation. This should be done
818/// before exception handling preparation passes.
819void TargetPassConfig::addCodeGenPrepare() {
820 if (getOptLevel() != CodeGenOptLevel::None && !Opts.disable_cgp)
821 addPass(P: createCodeGenPrepareLegacyPass());
822}
823
824/// Add common passes that perform LLVM IR to IR transforms in preparation for
825/// instruction selection.
826void TargetPassConfig::addISelPrepare() {
827 addPreISel();
828
829 // Force codegen to run according to the callgraph.
830 if (requiresCodeGenSCCOrder())
831 addPass(P: new DummyCGSCCPass);
832
833 addPass(P: createInlineAsmPreparePass());
834
835 // Add both the safe stack and the stack protection passes: each of them will
836 // only protect functions that have corresponding attributes.
837 addPass(P: createSafeStackPass());
838 addPass(P: createStackProtectorPass());
839
840 if (Opts.print_isel_input)
841 addPass(P: createPrintFunctionPass(
842 OS&: dbgs(), Banner: "\n\n*** Final LLVM Code input to ISel ***\n"));
843
844 // All passes which modify the LLVM IR are now complete; run the verifier
845 // to ensure that the IR is valid.
846 if (!DisableVerify)
847 addPass(P: createVerifierPass());
848}
849
850bool TargetPassConfig::addCoreISelPasses() {
851 // Enable FastISel with -fast-isel, but allow that to be overridden.
852 TM->setO0WantsFastISel(valueOr(X: Opts.fast_isel, Default: true));
853
854 // Determine an instruction selector.
855 enum class SelectorType { SelectionDAG, FastISel, GlobalISel };
856 SelectorType Selector;
857
858 if (valueOr(X: Opts.fast_isel, Default: false))
859 Selector = SelectorType::FastISel;
860 else if (valueOr(X: Opts.global_isel, Default: false) ||
861 (TM->Options.EnableGlobalISel && valueOr(X: Opts.global_isel, Default: true)))
862 Selector = SelectorType::GlobalISel;
863 else if (TM->getOptLevel() == CodeGenOptLevel::None &&
864 TM->getO0WantsFastISel())
865 Selector = SelectorType::FastISel;
866 else
867 Selector = SelectorType::SelectionDAG;
868
869 // Set consistently TM->Options.EnableFastISel and EnableGlobalISel.
870 if (Selector == SelectorType::FastISel) {
871 TM->setFastISel(true);
872 TM->setGlobalISel(false);
873 } else if (Selector == SelectorType::GlobalISel) {
874 TM->setFastISel(false);
875 TM->setGlobalISel(true);
876 }
877
878 // FIXME: Injecting into the DAGISel pipeline seems to cause issues with
879 // analyses needing to be re-run. This can result in being unable to
880 // schedule passes (particularly with 'Function Alias Analysis
881 // Results'). It's not entirely clear why but AFAICT this seems to be
882 // due to one FunctionPassManager not being able to use analyses from a
883 // previous one. As we're injecting a ModulePass we break the usual
884 // pass manager into two. GlobalISel with the fallback path disabled
885 // and -run-pass seem to be unaffected. The majority of GlobalISel
886 // testing uses -run-pass so this probably isn't too bad.
887 SaveAndRestore SavedDebugifyIsSafe(DebugifyIsSafe);
888 if (Selector != SelectorType::GlobalISel || !isGlobalISelAbortEnabled())
889 DebugifyIsSafe = false;
890
891 // Add instruction selector passes for global isel if enabled.
892 if (Selector == SelectorType::GlobalISel) {
893 SaveAndRestore SavedAddingMachinePasses(AddingMachinePasses, true);
894 if (addIRTranslator())
895 return true;
896
897 addPreLegalizeMachineIR();
898
899 if (addLegalizeMachineIR())
900 return true;
901
902 // Before running the register bank selector, ask the target if it
903 // wants to run some passes.
904 addPreRegBankSelect();
905
906 if (addRegBankSelect())
907 return true;
908
909 addPreGlobalInstructionSelect();
910
911 if (addGlobalInstructionSelect())
912 return true;
913 }
914
915 // Pass to reset the MachineFunction if the ISel failed. Outside of the above
916 // if so that the verifier is not added to it.
917 if (Selector == SelectorType::GlobalISel)
918 addPass(P: createResetMachineFunctionLegacyPass(
919 EmitFallbackDiag: reportDiagnosticWhenGlobalISelFallback(), AbortOnFailedISel: isGlobalISelAbortEnabled()));
920
921 // Run the SDAG InstSelector, providing a fallback path when we do not want to
922 // abort on not-yet-supported input.
923 if (Selector != SelectorType::GlobalISel || !isGlobalISelAbortEnabled())
924 if (addInstSelector())
925 return true;
926
927 // Expand pseudo-instructions emitted by ISel. Don't run the verifier before
928 // FinalizeISel.
929 addPass(PassID: &FinalizeISelID);
930
931 // Print the instruction selected machine code...
932 printAndVerify(Banner: "After Instruction Selection");
933
934 return false;
935}
936
937bool TargetPassConfig::addISelPasses() {
938 if (TM->useEmulatedTLS())
939 addPass(P: createLowerEmuTLSPass());
940
941 PM->add(P: createTargetTransformInfoWrapperPass(TIRA: TM->getTargetIRAnalysis()));
942 // ObjCARCContract operates on ObjC intrinsics and must run before
943 // PreISelIntrinsicLowering.
944 if (getOptLevel() != CodeGenOptLevel::None)
945 addPass(P: createObjCARCContractPass());
946 addPass(P: createPreISelIntrinsicLoweringPass());
947 addPass(P: createExpandIRInstsPass(getOptLevel()));
948 addIRPasses();
949
950 if (Opts.codegen_pipeline_trigger_crash)
951 addPass(P: new TriggerCrashFunctionLegacyPass());
952
953 addCodeGenPrepare();
954 addPassesToHandleExceptions();
955 addISelPrepare();
956
957 return addCoreISelPasses();
958}
959
960/// -regalloc=... command line option.
961static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
962static cl::opt<RegisterRegAlloc::FunctionPassCtor, false,
963 RegisterPassParser<RegisterRegAlloc>>
964 RegAlloc("regalloc", cl::Hidden, cl::init(Val: &useDefaultRegisterAllocator),
965 cl::desc("Register allocator to use"));
966
967/// Add the complete set of target-independent postISel code generator passes.
968///
969/// This can be read as the standard order of major LLVM CodeGen stages. Stages
970/// with nontrivial configuration or multiple passes are broken out below in
971/// add%Stage routines.
972///
973/// Any TargetPassConfig::addXX routine may be overriden by the Target. The
974/// addPre/Post methods with empty header implementations allow injecting
975/// target-specific fixups just before or after major stages. Additionally,
976/// targets have the flexibility to change pass order within a stage by
977/// overriding default implementation of add%Stage routines below. Each
978/// technique has maintainability tradeoffs because alternate pass orders are
979/// not well supported. addPre/Post works better if the target pass is easily
980/// tied to a common pass. But if it has subtle dependencies on multiple passes,
981/// the target should override the stage instead.
982///
983/// TODO: We could use a single addPre/Post(ID) hook to allow pass injection
984/// before/after any target-independent pass. But it's currently overkill.
985void TargetPassConfig::addMachinePasses() {
986 AddingMachinePasses = true;
987
988 // Add passes that optimize machine instructions in SSA form.
989 if (getOptLevel() != CodeGenOptLevel::None) {
990 addMachineSSAOptimization();
991 } else {
992 // If the target requests it, assign local variables to stack slots relative
993 // to one another and simplify frame index references where possible.
994 addPass(PassID: &LocalStackSlotAllocationID);
995 }
996
997 if (TM->Options.EnableIPRA)
998 addPass(P: createRegUsageInfoPropPass());
999
1000 if (getOptLevel() != CodeGenOptLevel::None &&
1001 valueOr(X: getOpts(TM: *TM).enable_pipeliner, Default: isMachinePipelinerEnabled()))
1002 addPass(PassID: &MachinePipelinerID);
1003
1004 // Run pre-ra passes.
1005 addPreRegAlloc();
1006
1007 // Debugifying the register allocator passes seems to provoke some
1008 // non-determinism that affects CodeGen and there doesn't seem to be a point
1009 // where it becomes safe again so stop debugifying here.
1010 DebugifyIsSafe = false;
1011
1012 // Add a FSDiscriminator pass right before RA, so that we could get
1013 // more precise SampleFDO profile for RA.
1014 if (EnableFSDiscriminator) {
1015 addPass(P: createMIRAddFSDiscriminatorsPass(
1016 P: sampleprof::FSDiscriminatorPass::Pass1));
1017 const std::string ProfileFile = getFSProfileFile(Opts, TM);
1018 if (!ProfileFile.empty() && !Opts.disable_ra_fsprofile_loader)
1019 addPass(P: createMIRProfileLoaderPass(
1020 File: ProfileFile, RemappingFile: getFSRemappingFile(Opts, TM),
1021 P: sampleprof::FSDiscriminatorPass::Pass1, FS: nullptr));
1022 }
1023
1024 // Run register allocation and passes that are tightly coupled with it,
1025 // including phi elimination and scheduling.
1026 if (getOptimizeRegAlloc())
1027 addOptimizedRegAlloc();
1028 else
1029 addFastRegAlloc();
1030
1031 // Run post-ra passes.
1032 addPostRegAlloc();
1033
1034 addPass(PassID: &RemoveRedundantDebugValuesID);
1035
1036 addPass(PassID: &FixupStatepointCallerSavedID);
1037
1038 // Insert prolog/epilog code. Eliminate abstract frame index references...
1039 if (getOptLevel() != CodeGenOptLevel::None) {
1040 addPass(PassID: &PostRAMachineSinkingID);
1041 addPass(PassID: &ShrinkWrapID);
1042 }
1043
1044 // Prolog/Epilog inserter needs a TargetMachine to instantiate. But only
1045 // do so if it hasn't been disabled, substituted, or overridden.
1046 if (!isPassSubstitutedOrOverridden(ID: &PrologEpilogCodeInserterID))
1047 addPass(P: createPrologEpilogInserterPass());
1048
1049 /// Add passes that optimize machine instructions after register allocation.
1050 if (getOptLevel() != CodeGenOptLevel::None)
1051 addMachineLateOptimization();
1052
1053 // Expand pseudo instructions before second scheduling pass.
1054 addPass(PassID: &ExpandPostRAPseudosID);
1055
1056 // Run pre-sched2 passes.
1057 addPreSched2();
1058
1059 if (Opts.enable_implicit_null_checks)
1060 addPass(PassID: &ImplicitNullChecksID);
1061
1062 // Second pass scheduler.
1063 // Let Target optionally insert this pass by itself at some other
1064 // point.
1065 if (getOptLevel() != CodeGenOptLevel::None &&
1066 !TM->targetSchedulesPostRAScheduling()) {
1067 if (Opts.misched_postra)
1068 addPass(PassID: &PostMachineSchedulerID);
1069 else
1070 addPass(PassID: &PostRASchedulerID);
1071 }
1072
1073 // GC
1074 addGCPasses();
1075
1076 // Basic block placement.
1077 if (getOptLevel() != CodeGenOptLevel::None)
1078 addBlockPlacement();
1079
1080 // Insert before XRay Instrumentation.
1081 addPass(PassID: &FEntryInserterID);
1082
1083 addPass(PassID: &XRayInstrumentationID);
1084 addPass(PassID: &PatchableFunctionID);
1085
1086 addPreEmitPass();
1087
1088 if (TM->Options.EnableIPRA)
1089 // Collect register usage information and produce a register mask of
1090 // clobbered registers, to be used to optimize call sites.
1091 addPass(P: createRegUsageInfoCollector());
1092
1093 // FIXME: Some backends are incompatible with running the verifier after
1094 // addPreEmitPass. Maybe only pass "false" here for those targets?
1095 addPass(PassID: &FuncletLayoutID);
1096
1097 addPass(PassID: &RemoveLoadsIntoFakeUsesID);
1098 addPass(PassID: &StackMapLivenessID);
1099 addPass(PassID: &LiveDebugValuesID);
1100 addPass(PassID: &MachineSanitizerBinaryMetadataID);
1101
1102 if (TM->Options.EnableMachineOutliner &&
1103 getOptLevel() != CodeGenOptLevel::None &&
1104 EnableMachineOutliner != RunOutliner::NeverOutline) {
1105 if (EnableMachineOutliner != RunOutliner::TargetDefault ||
1106 TM->supportsDefaultOutlining())
1107 addPass(P: createMachineOutlinerPass(RunOutlinerMode: EnableMachineOutliner));
1108 }
1109
1110 if (Opts.enable_gc_empty_basic_blocks)
1111 addPass(P: llvm::createGCEmptyBasicBlocksLegacyPass());
1112
1113 if (EnableFSDiscriminator)
1114 addPass(P: createMIRAddFSDiscriminatorsPass(
1115 P: sampleprof::FSDiscriminatorPass::PassLast));
1116
1117 const bool SplitFunctionsWithPGO =
1118 TM->Options.FunctionSplitting == FunctionSplittingMode::All;
1119
1120 if (SplitFunctionsWithPGO || Opts.split_static_data ||
1121 TM->Options.EnableStaticDataPartitioning) {
1122 const std::string ProfileFile = getFSProfileFile(Opts, TM);
1123 if (!ProfileFile.empty()) {
1124 if (EnableFSDiscriminator) {
1125 addPass(P: createMIRProfileLoaderPass(
1126 File: ProfileFile, RemappingFile: getFSRemappingFile(Opts, TM),
1127 P: sampleprof::FSDiscriminatorPass::PassLast, FS: nullptr));
1128 } else {
1129 // Sample profile is given, but FSDiscriminator is not
1130 // enabled, this may result in performance regression.
1131 WithColor::warning()
1132 << "Using AutoFDO without FSDiscriminator for MFS may regress "
1133 "performance.\n";
1134 }
1135 }
1136 }
1137
1138 if (Opts.split_static_data || TM->Options.EnableStaticDataPartitioning) {
1139 // The static data splitter pass is a machine function pass. and
1140 // static data annotator pass is a module-wide pass. See the file comment
1141 // in StaticDataAnnotator.cpp for the motivation.
1142 addPass(P: createStaticDataSplitterLegacyPass());
1143 addPass(P: createStaticDataAnnotatorLegacyPass());
1144 }
1145 // We run the BasicBlockSections pass if we need BB sections, BB address map,
1146 // or function splitting. With function splitting, functions which are
1147 // eligible for basic-block-sections optimizations (`=all`, or `=list=` with
1148 // function included in the list profile) get that optimization, and the other
1149 // functions are split using their PGO/AutoFDO profile.
1150 if (TM->getBBSectionsType() != llvm::BasicBlockSection::None ||
1151 TM->Options.BBAddrMap || SplitFunctionsWithPGO) {
1152 if (shouldEmitBBHash() || Opts.basic_block_section_match_infer)
1153 addPass(P: llvm::createMachineBlockHashInfoPass());
1154 if (TM->getBBSectionsType() == llvm::BasicBlockSection::List) {
1155 addPass(P: llvm::createBasicBlockSectionsProfileReaderWrapperPass(
1156 Buf: TM->getBBSectionsFuncListBuf()));
1157 if (!Opts.basic_block_section_match_infer || Opts.propeller_match_bbid)
1158 addPass(P: llvm::createBasicBlockPathCloningPass());
1159 if (Opts.basic_block_section_match_infer)
1160 addPass(P: llvm::createBasicBlockMatchingAndInferencePass());
1161 else
1162 addPass(P: llvm::createInsertCodePrefetchPass());
1163 }
1164 addPass(P: llvm::createBasicBlockSectionsPass(SplitWithPGO: SplitFunctionsWithPGO));
1165 }
1166
1167 addPostBBSections();
1168
1169 if (!Opts.disable_cfi_fixup && TM->Options.EnableCFIFixup)
1170 addPass(P: createCFIFixupLegacy());
1171
1172 PM->add(P: createStackFrameLayoutAnalysisPass());
1173
1174 // Add passes that directly emit MI after all other MI passes.
1175 addPreEmitPass2();
1176
1177 AddingMachinePasses = false;
1178}
1179
1180/// Add passes that optimize machine instructions in SSA form.
1181void TargetPassConfig::addMachineSSAOptimization() {
1182 // Pre-ra tail duplication.
1183 addPass(PassID: &EarlyTailDuplicateLegacyID);
1184
1185 // Optimize PHIs before DCE: removing dead PHI cycles may make more
1186 // instructions dead.
1187 addPass(PassID: &OptimizePHIsLegacyID);
1188
1189 // This pass merges large allocas. StackSlotColoring is a different pass
1190 // which merges spill slots.
1191 addPass(PassID: &StackColoringLegacyID);
1192
1193 // If the target requests it, assign local variables to stack slots relative
1194 // to one another and simplify frame index references where possible.
1195 addPass(PassID: &LocalStackSlotAllocationID);
1196
1197 // With optimization, dead code should already be eliminated. However
1198 // there is one known exception: lowered code for arguments that are only
1199 // used by tail calls, where the tail calls reuse the incoming stack
1200 // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
1201 addPass(PassID: &DeadMachineInstructionElimID);
1202
1203 // Allow targets to insert passes that improve instruction level parallelism,
1204 // like if-conversion. Such passes will typically need dominator trees and
1205 // loop info, just like LICM and CSE below.
1206 addILPOpts();
1207
1208 addPass(PassID: &EarlyMachineLICMID);
1209 addPass(PassID: &MachineCSELegacyID);
1210
1211 addPass(PassID: &MachineSinkingLegacyID);
1212
1213 addPass(PassID: &PeepholeOptimizerLegacyID);
1214 // Clean-up the dead code that may have been generated by peephole
1215 // rewriting.
1216 addPass(PassID: &DeadMachineInstructionElimID);
1217}
1218
1219//===---------------------------------------------------------------------===//
1220/// Register Allocation Pass Configuration
1221//===---------------------------------------------------------------------===//
1222
1223/// A dummy default pass factory indicates whether the register allocator is
1224/// overridden on the command line.
1225static llvm::once_flag InitializeDefaultRegisterAllocatorFlag;
1226
1227static RegisterRegAlloc
1228defaultRegAlloc("default",
1229 "pick register allocator based on -O option",
1230 useDefaultRegisterAllocator);
1231
1232static void initializeDefaultRegisterAllocatorOnce() {
1233 if (!RegisterRegAlloc::getDefault())
1234 RegisterRegAlloc::setDefault(RegAlloc);
1235}
1236
1237bool TargetPassConfig::getOptimizeRegAlloc() const {
1238 // An explicit -regalloc choice implies its pipeline: only the fast
1239 // allocator uses the unoptimized one.
1240 llvm::call_once(flag&: InitializeDefaultRegisterAllocatorFlag,
1241 F&: initializeDefaultRegisterAllocatorOnce);
1242 RegisterRegAlloc::FunctionPassCtor Ctor = RegisterRegAlloc::getDefault();
1243 if (Ctor != (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator)
1244 return Ctor !=
1245 (RegisterRegAlloc::FunctionPassCtor)&createFastRegisterAllocator;
1246 return getOptLevel() != CodeGenOptLevel::None;
1247}
1248
1249/// Instantiate the default register allocator pass for this target for either
1250/// the optimized or unoptimized allocation path. This will be added to the pass
1251/// manager by addFastRegAlloc in the unoptimized case or addOptimizedRegAlloc
1252/// in the optimized case.
1253///
1254/// A target that uses the standard regalloc pass order for fast or optimized
1255/// allocation may still override this for per-target regalloc
1256/// selection. But -regalloc=... always takes precedence.
1257FunctionPass *TargetPassConfig::createTargetRegisterAllocator(bool Optimized) {
1258 if (Optimized)
1259 return createGreedyRegisterAllocator();
1260 else
1261 return createFastRegisterAllocator();
1262}
1263
1264/// Find and instantiate the register allocation pass requested by this target
1265/// at the current optimization level. Different register allocators are
1266/// defined as separate passes because they may require different analysis.
1267///
1268/// This helper ensures that the regalloc= option is always available,
1269/// even for targets that override the default allocator.
1270///
1271/// FIXME: When MachinePassRegistry register pass IDs instead of function ptrs,
1272/// this can be folded into addPass.
1273FunctionPass *TargetPassConfig::createRegAllocPass(bool Optimized) {
1274 // getOptimizeRegAlloc, called before the pipeline branches, has initialized
1275 // the global default.
1276 RegisterRegAlloc::FunctionPassCtor Ctor = RegisterRegAlloc::getDefault();
1277 if (Ctor != useDefaultRegisterAllocator)
1278 return Ctor();
1279
1280 // With no -regalloc= override, ask the target for a regalloc pass.
1281 return createTargetRegisterAllocator(Optimized);
1282}
1283
1284bool TargetPassConfig::isCustomizedRegAlloc() {
1285 return RegAlloc !=
1286 (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator;
1287}
1288
1289bool TargetPassConfig::addRegAssignAndRewriteFast() {
1290 if (RegAlloc != (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator &&
1291 RegAlloc != (RegisterRegAlloc::FunctionPassCtor)&createFastRegisterAllocator)
1292 reportFatalUsageError(
1293 reason: "Must use fast (default) register allocator for unoptimized regalloc.");
1294
1295 addPass(P: createRegAllocPass(Optimized: false));
1296
1297 return true;
1298}
1299
1300bool TargetPassConfig::addRegAssignAndRewriteOptimized() {
1301 // Add the selected register allocation pass.
1302 addPass(P: createRegAllocPass(Optimized: true));
1303
1304 // Allow targets to change the register assignments before rewriting.
1305 addPreRewrite();
1306
1307 // Finally rewrite virtual registers.
1308 addPass(PassID: &VirtRegRewriterID);
1309
1310 // Regalloc scoring for ML-driven eviction - noop except when learning a new
1311 // eviction policy.
1312 addPass(P: createRegAllocScoringPass());
1313 return true;
1314}
1315
1316/// Return true if the default global register allocator is in use and
1317/// has not be overriden on the command line with '-regalloc=...'
1318bool TargetPassConfig::usingDefaultRegAlloc() const {
1319 return RegAlloc.getNumOccurrences() == 0;
1320}
1321
1322/// Add the minimum set of target-independent passes that are required for
1323/// register allocation. No coalescing or scheduling.
1324void TargetPassConfig::addFastRegAlloc() {
1325 addPass(PassID: &PHIEliminationID);
1326 if (!TM->enableTiedFastRegAlloc())
1327 addPass(PassID: &TwoAddressInstructionPassID);
1328
1329 addRegAssignAndRewriteFast();
1330}
1331
1332/// Add standard target-independent passes that are tightly coupled with
1333/// optimized register allocation, including coalescing, machine instruction
1334/// scheduling, and register allocation itself.
1335void TargetPassConfig::addOptimizedRegAlloc() {
1336 addPass(PassID: &DetectDeadLanesID);
1337
1338 addPass(PassID: &InitUndefID);
1339
1340 addPass(PassID: &ProcessImplicitDefsID);
1341
1342 // LiveIntervals requires pure SSA form and no unreachable blocks.
1343 // Explicit addition of UnreachableMachineBlockElim allows stopping before or
1344 // after it with -stop-before/-stop-after.
1345 addPass(PassID: &UnreachableMachineBlockElimID);
1346
1347 // Run SSA machine scheduler runs just before PHI elimination.
1348 if (EnableSSAMachineScheduler) {
1349 addPass(PassID: &LiveIntervalsID);
1350 addPass(PassID: &SSAMachineSchedulerID);
1351 }
1352
1353 addPass(PassID: &LiveIntervalsID);
1354
1355 // Edge splitting is smarter with machine loop info.
1356 addPass(PassID: &MachineLoopInfoID);
1357 addPass(PassID: &PHIEliminationID);
1358
1359 addPass(PassID: &TwoAddressInstructionPassID);
1360 addPass(PassID: &RegisterCoalescerID);
1361
1362 // The machine scheduler may accidentally create disconnected components
1363 // when moving subregister definitions around, avoid this by splitting them to
1364 // separate vregs before. Splitting can also improve reg. allocation quality.
1365 addPass(PassID: &RenameIndependentSubregsID);
1366
1367 // PreRA instruction scheduling.
1368 addPass(PassID: &MachineSchedulerID);
1369
1370 if (addRegAssignAndRewriteOptimized()) {
1371 // Perform stack slot coloring and post-ra machine LICM.
1372 addPass(PassID: &StackSlotColoringID);
1373
1374 // Allow targets to expand pseudo instructions depending on the choice of
1375 // registers before MachineCopyPropagation.
1376 addPostRewrite();
1377
1378 // Copy propagate to forward register uses and try to eliminate COPYs that
1379 // were not coalesced.
1380 addPass(PassID: &MachineCopyPropagationID);
1381
1382 // Run post-ra machine LICM to hoist reloads / remats.
1383 //
1384 // FIXME: can this move into MachineLateOptimization?
1385 addPass(PassID: &MachineLICMID);
1386 }
1387}
1388
1389//===---------------------------------------------------------------------===//
1390/// Post RegAlloc Pass Configuration
1391//===---------------------------------------------------------------------===//
1392
1393/// Add passes that optimize machine instructions after register allocation.
1394void TargetPassConfig::addMachineLateOptimization() {
1395 // Cleanup of redundant immediate/address loads.
1396 addPass(PassID: &MachineLateInstrsCleanupID);
1397
1398 // Branch folding must be run after regalloc and prolog/epilog insertion.
1399 addPass(PassID: &BranchFolderPassID);
1400
1401 // Tail duplication.
1402 // Note that duplicating tail just increases code size and degrades
1403 // performance for targets that require Structured Control Flow.
1404 // In addition it can also make CFG irreducible. Thus we disable it.
1405 if (!TM->requiresStructuredCFG())
1406 addPass(PassID: &TailDuplicateLegacyID);
1407
1408 // Copy propagation.
1409 addPass(PassID: &MachineCopyPropagationID);
1410}
1411
1412/// Add standard GC passes.
1413bool TargetPassConfig::addGCPasses() {
1414 addPass(PassID: &GCMachineCodeAnalysisID);
1415 return true;
1416}
1417
1418/// Add standard basic block placement passes.
1419void TargetPassConfig::addBlockPlacement() {
1420 if (EnableFSDiscriminator) {
1421 addPass(P: createMIRAddFSDiscriminatorsPass(
1422 P: sampleprof::FSDiscriminatorPass::Pass2));
1423 const std::string ProfileFile = getFSProfileFile(Opts, TM);
1424 if (!ProfileFile.empty() && !Opts.disable_layout_fsprofile_loader)
1425 addPass(P: createMIRProfileLoaderPass(
1426 File: ProfileFile, RemappingFile: getFSRemappingFile(Opts, TM),
1427 P: sampleprof::FSDiscriminatorPass::Pass2, FS: nullptr));
1428 }
1429 if (addPass(PassID: &MachineBlockPlacementID)) {
1430 // Run a separate pass to collect block placement statistics.
1431 if (Opts.enable_block_placement_stats)
1432 addPass(PassID: &MachineBlockPlacementStatsID);
1433 }
1434}
1435
1436//===---------------------------------------------------------------------===//
1437/// GlobalISel Configuration
1438//===---------------------------------------------------------------------===//
1439bool TargetPassConfig::isGlobalISelAbortEnabled() const {
1440 return TM->Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
1441}
1442
1443bool TargetPassConfig::reportDiagnosticWhenGlobalISelFallback() const {
1444 return TM->Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag;
1445}
1446
1447std::unique_ptr<CSEConfigBase> TargetPassConfig::getCSEConfig() const {
1448 return std::make_unique<CSEConfigBase>();
1449}
1450