| 1 | //===- TargetPassConfig.cpp - Target independent code generation passes ---===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file defines interfaces to access the target independent code |
| 10 | // generation passes provided by the LLVM backend. |
| 11 | // |
| 12 | //===---------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "llvm/CodeGen/TargetPassConfig.h" |
| 15 | #include "CodeGenOptions.h" |
| 16 | #include "llvm/ADT/DenseMap.h" |
| 17 | #include "llvm/ADT/SmallVector.h" |
| 18 | #include "llvm/ADT/StringRef.h" |
| 19 | #include "llvm/Analysis/BasicAliasAnalysis.h" |
| 20 | #include "llvm/Analysis/CallGraphSCCPass.h" |
| 21 | #include "llvm/Analysis/ScopedNoAliasAA.h" |
| 22 | #include "llvm/Analysis/TargetTransformInfo.h" |
| 23 | #include "llvm/Analysis/TypeBasedAliasAnalysis.h" |
| 24 | #include "llvm/CodeGen/BasicBlockSectionsProfileReader.h" |
| 25 | #include "llvm/CodeGen/CSEConfigBase.h" |
| 26 | #include "llvm/CodeGen/CodeGenTargetMachineImpl.h" |
| 27 | #include "llvm/CodeGen/MachineBlockHashInfo.h" |
| 28 | #include "llvm/CodeGen/MachineFunctionPass.h" |
| 29 | #include "llvm/CodeGen/MachinePassRegistry.h" |
| 30 | #include "llvm/CodeGen/Passes.h" |
| 31 | #include "llvm/CodeGen/RegAllocRegistry.h" |
| 32 | #include "llvm/IR/IRPrintingPasses.h" |
| 33 | #include "llvm/IR/LegacyPassManager.h" |
| 34 | #include "llvm/IR/PassInstrumentation.h" |
| 35 | #include "llvm/IR/Verifier.h" |
| 36 | #include "llvm/InitializePasses.h" |
| 37 | #include "llvm/MC/MCAsmInfo.h" |
| 38 | #include "llvm/MC/MCTargetOptions.h" |
| 39 | #include "llvm/Pass.h" |
| 40 | #include "llvm/Support/CodeGen.h" |
| 41 | #include "llvm/Support/CommandLine.h" |
| 42 | #include "llvm/Support/Compiler.h" |
| 43 | #include "llvm/Support/Debug.h" |
| 44 | #include "llvm/Support/Discriminator.h" |
| 45 | #include "llvm/Support/ErrorHandling.h" |
| 46 | #include "llvm/Support/SaveAndRestore.h" |
| 47 | #include "llvm/Support/Threading.h" |
| 48 | #include "llvm/Support/VirtualFileSystem.h" |
| 49 | #include "llvm/Support/WithColor.h" |
| 50 | #include "llvm/Target/CGPassBuilderOption.h" |
| 51 | #include "llvm/Target/TargetMachine.h" |
| 52 | #include "llvm/Transforms/ObjCARC.h" |
| 53 | #include "llvm/Transforms/Scalar.h" |
| 54 | #include "llvm/Transforms/Utils.h" |
| 55 | #include <cassert> |
| 56 | #include <optional> |
| 57 | #include <string> |
| 58 | |
| 59 | using namespace llvm; |
| 60 | |
| 61 | static const CodeGenOptions &getOpts(const TargetMachine &TM) { |
| 62 | return cl::LibraryOptionsContext::get<CodeGenOptions>( |
| 63 | C: TM.Options.MCOptions.LibraryOptions); |
| 64 | } |
| 65 | |
| 66 | namespace { |
| 67 | class TriggerCrashFunctionLegacyPass : public FunctionPass { |
| 68 | public: |
| 69 | static char ID; |
| 70 | TriggerCrashFunctionLegacyPass() : FunctionPass(ID) {} |
| 71 | bool runOnFunction(Function &F) override { |
| 72 | abort(); |
| 73 | return false; |
| 74 | } |
| 75 | StringRef getPassName() const override { return "TriggerCrashFunctionPass" ; } |
| 76 | }; |
| 77 | } // namespace |
| 78 | |
| 79 | char TriggerCrashFunctionLegacyPass::ID = 0; |
| 80 | |
| 81 | // Enable or disable the MachineOutliner. |
| 82 | static cl::opt<RunOutliner> EnableMachineOutliner( |
| 83 | "enable-machine-outliner" , cl::desc("Enable the machine outliner" ), |
| 84 | cl::Hidden, cl::ValueOptional, cl::init(Val: RunOutliner::TargetDefault), |
| 85 | cl::values( |
| 86 | clEnumValN(RunOutliner::AlwaysOutline, "always" , |
| 87 | "Run on all functions guaranteed to be beneficial" ), |
| 88 | clEnumValN(RunOutliner::OptimisticPGO, "optimistic-pgo" , |
| 89 | "Outline cold code only. If a code block does not have " |
| 90 | "profile data, optimistically assume it is cold." ), |
| 91 | clEnumValN(RunOutliner::ConservativePGO, "conservative-pgo" , |
| 92 | "Outline cold code only. If a code block does not have " |
| 93 | "profile, data, conservatively assume it is hot." ), |
| 94 | clEnumValN(RunOutliner::NeverOutline, "never" , "Disable all outlining" ), |
| 95 | // Sentinel value for unspecified option. |
| 96 | clEnumValN(RunOutliner::AlwaysOutline, "" , "" ))); |
| 97 | |
| 98 | /// Option names for limiting the codegen pipeline. |
| 99 | /// Those are used in error reporting and we didn't want |
| 100 | /// to duplicate their names all over the place. |
| 101 | static const char StartAfterOptName[] = "start-after" ; |
| 102 | static const char StartBeforeOptName[] = "start-before" ; |
| 103 | static const char StopAfterOptName[] = "stop-after" ; |
| 104 | static const char StopBeforeOptName[] = "stop-before" ; |
| 105 | |
| 106 | /// Allow standard passes to be disabled by command line options. This supports |
| 107 | /// simple binary flags that either suppress the pass or do nothing. |
| 108 | /// i.e. -disable-mypass=false has no effect. |
| 109 | /// These should be converted to boolOrDefault in order to use applyOverride. |
| 110 | static IdentifyingPassPtr applyDisable(IdentifyingPassPtr PassID, |
| 111 | bool Override) { |
| 112 | if (Override) |
| 113 | return IdentifyingPassPtr(); |
| 114 | return PassID; |
| 115 | } |
| 116 | |
| 117 | /// Allow standard passes to be disabled by the command line, regardless of who |
| 118 | /// is adding the pass. |
| 119 | /// |
| 120 | /// StandardID is the pass identified in the standard pass pipeline and provided |
| 121 | /// to addPass(). It may be a target-specific ID in the case that the target |
| 122 | /// directly adds its own pass, but in that case we harmlessly fall through. |
| 123 | /// |
| 124 | /// TargetID is the pass that the target has configured to override StandardID. |
| 125 | /// |
| 126 | /// StandardID may be a pseudo ID. In that case TargetID is the name of the real |
| 127 | /// pass to run. This allows multiple options to control a single pass depending |
| 128 | /// on where in the pipeline that pass is added. |
| 129 | static IdentifyingPassPtr overridePass(const CodeGenOptions &Opts, |
| 130 | AnalysisID StandardID, |
| 131 | IdentifyingPassPtr TargetID) { |
| 132 | if (StandardID == &PostRASchedulerID) |
| 133 | return applyDisable(PassID: TargetID, Override: Opts.disable_post_ra); |
| 134 | |
| 135 | if (StandardID == &BranchFolderPassID) |
| 136 | return applyDisable(PassID: TargetID, Override: Opts.disable_branch_fold); |
| 137 | |
| 138 | if (StandardID == &TailDuplicateLegacyID) |
| 139 | return applyDisable(PassID: TargetID, Override: Opts.disable_tail_duplicate); |
| 140 | |
| 141 | if (StandardID == &EarlyTailDuplicateLegacyID) |
| 142 | return applyDisable(PassID: TargetID, Override: Opts.disable_early_taildup); |
| 143 | |
| 144 | if (StandardID == &MachineBlockPlacementID) |
| 145 | return applyDisable(PassID: TargetID, Override: Opts.disable_block_placement); |
| 146 | |
| 147 | if (StandardID == &StackSlotColoringID) |
| 148 | return applyDisable(PassID: TargetID, Override: Opts.disable_ssc); |
| 149 | |
| 150 | if (StandardID == &DeadMachineInstructionElimID) |
| 151 | return applyDisable(PassID: TargetID, Override: Opts.disable_machine_dce); |
| 152 | |
| 153 | if (StandardID == &EarlyIfConverterLegacyID) |
| 154 | return applyDisable(PassID: TargetID, Override: Opts.disable_early_ifcvt); |
| 155 | |
| 156 | if (StandardID == &EarlyMachineLICMID) |
| 157 | return applyDisable(PassID: TargetID, Override: Opts.disable_machine_licm); |
| 158 | |
| 159 | if (StandardID == &MachineCSELegacyID) |
| 160 | return applyDisable(PassID: TargetID, Override: Opts.disable_machine_cse); |
| 161 | |
| 162 | if (StandardID == &MachineLICMID) |
| 163 | return applyDisable(PassID: TargetID, Override: Opts.disable_postra_machine_licm); |
| 164 | |
| 165 | if (StandardID == &MachineSinkingLegacyID) |
| 166 | return applyDisable(PassID: TargetID, Override: Opts.disable_machine_sink); |
| 167 | |
| 168 | if (StandardID == &PostRAMachineSinkingID) |
| 169 | return applyDisable(PassID: TargetID, Override: Opts.disable_postra_machine_sink); |
| 170 | |
| 171 | if (StandardID == &MachineCopyPropagationID) |
| 172 | return applyDisable(PassID: TargetID, Override: Opts.disable_copyprop); |
| 173 | |
| 174 | return TargetID; |
| 175 | } |
| 176 | |
| 177 | // Find the FSProfile file name. The internal option takes the precedence |
| 178 | // before getting from TargetMachine. |
| 179 | static std::string getFSProfileFile(const CodeGenOptions &Opts, |
| 180 | const TargetMachine *TM) { |
| 181 | if (!Opts.fs_profile_file.empty()) |
| 182 | return Opts.fs_profile_file.str(); |
| 183 | const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption(); |
| 184 | if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse) |
| 185 | return std::string(); |
| 186 | return PGOOpt->ProfileFile; |
| 187 | } |
| 188 | |
| 189 | // Find the Profile remapping file name. The internal option takes the |
| 190 | // precedence before getting from TargetMachine. |
| 191 | static std::string getFSRemappingFile(const CodeGenOptions &Opts, |
| 192 | const TargetMachine *TM) { |
| 193 | if (!Opts.fs_remapping_file.empty()) |
| 194 | return Opts.fs_remapping_file.str(); |
| 195 | const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption(); |
| 196 | if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse) |
| 197 | return std::string(); |
| 198 | return PGOOpt->ProfileRemappingFile; |
| 199 | } |
| 200 | |
| 201 | //===---------------------------------------------------------------------===// |
| 202 | /// TargetPassConfig |
| 203 | //===---------------------------------------------------------------------===// |
| 204 | |
| 205 | INITIALIZE_PASS(TargetPassConfig, "targetpassconfig" , |
| 206 | "Target Pass Configuration" , false, false) |
| 207 | char TargetPassConfig::ID = 0; |
| 208 | |
| 209 | namespace { |
| 210 | |
| 211 | struct InsertedPass { |
| 212 | AnalysisID TargetPassID; |
| 213 | IdentifyingPassPtr InsertedPassID; |
| 214 | |
| 215 | InsertedPass(AnalysisID TargetPassID, IdentifyingPassPtr InsertedPassID) |
| 216 | : TargetPassID(TargetPassID), InsertedPassID(InsertedPassID) {} |
| 217 | |
| 218 | Pass *getInsertedPass() const { |
| 219 | assert(InsertedPassID.isValid() && "Illegal Pass ID!" ); |
| 220 | if (InsertedPassID.isInstance()) |
| 221 | return InsertedPassID.getInstance(); |
| 222 | Pass *NP = Pass::createPass(ID: InsertedPassID.getID()); |
| 223 | assert(NP && "Pass ID not registered" ); |
| 224 | return NP; |
| 225 | } |
| 226 | }; |
| 227 | |
| 228 | } // end anonymous namespace |
| 229 | |
| 230 | namespace llvm { |
| 231 | |
| 232 | class PassConfigImpl { |
| 233 | public: |
| 234 | // List of passes explicitly substituted by this target. Normally this is |
| 235 | // empty, but it is a convenient way to suppress or replace specific passes |
| 236 | // that are part of a standard pass pipeline without overridding the entire |
| 237 | // pipeline. This mechanism allows target options to inherit a standard pass's |
| 238 | // user interface. For example, a target may disable a standard pass by |
| 239 | // default by substituting a pass ID of zero, and the user may still enable |
| 240 | // that standard pass with an explicit command line option. |
| 241 | DenseMap<AnalysisID,IdentifyingPassPtr> TargetPasses; |
| 242 | |
| 243 | /// Store the pairs of <AnalysisID, AnalysisID> of which the second pass |
| 244 | /// is inserted after each instance of the first one. |
| 245 | SmallVector<InsertedPass, 4> InsertedPasses; |
| 246 | }; |
| 247 | |
| 248 | } // end namespace llvm |
| 249 | |
| 250 | // Out of line virtual method. |
| 251 | TargetPassConfig::~TargetPassConfig() { |
| 252 | delete Impl; |
| 253 | } |
| 254 | |
| 255 | static const PassInfo *getPassInfo(StringRef PassName) { |
| 256 | if (PassName.empty()) |
| 257 | return nullptr; |
| 258 | |
| 259 | const PassRegistry &PR = *PassRegistry::getPassRegistry(); |
| 260 | const PassInfo *PI = PR.getPassInfo(Arg: PassName); |
| 261 | if (!PI) |
| 262 | reportFatalUsageError(reason: Twine('\"') + Twine(PassName) + |
| 263 | Twine("\" pass is not registered." )); |
| 264 | return PI; |
| 265 | } |
| 266 | |
| 267 | static AnalysisID getPassIDFromName(StringRef PassName) { |
| 268 | const PassInfo *PI = getPassInfo(PassName); |
| 269 | return PI ? PI->getTypeInfo() : nullptr; |
| 270 | } |
| 271 | |
| 272 | static std::pair<StringRef, unsigned> |
| 273 | getPassNameAndInstanceNum(StringRef PassName) { |
| 274 | StringRef Name, InstanceNumStr; |
| 275 | std::tie(args&: Name, args&: InstanceNumStr) = PassName.split(Separator: ','); |
| 276 | |
| 277 | unsigned InstanceNum = 0; |
| 278 | if (!InstanceNumStr.empty() && InstanceNumStr.getAsInteger(Radix: 10, Result&: InstanceNum)) |
| 279 | reportFatalUsageError(reason: "invalid pass instance specifier " + PassName); |
| 280 | |
| 281 | return std::make_pair(x&: Name, y&: InstanceNum); |
| 282 | } |
| 283 | |
| 284 | void TargetPassConfig::setStartStopPasses() { |
| 285 | StringRef StartBeforeName; |
| 286 | std::tie(args&: StartBeforeName, args&: StartBeforeInstanceNum) = |
| 287 | getPassNameAndInstanceNum(PassName: Opts.start_before); |
| 288 | |
| 289 | StringRef StartAfterName; |
| 290 | std::tie(args&: StartAfterName, args&: StartAfterInstanceNum) = |
| 291 | getPassNameAndInstanceNum(PassName: Opts.start_after); |
| 292 | |
| 293 | StringRef StopBeforeName; |
| 294 | std::tie(args&: StopBeforeName, args&: StopBeforeInstanceNum) = |
| 295 | getPassNameAndInstanceNum(PassName: Opts.stop_before); |
| 296 | |
| 297 | StringRef StopAfterName; |
| 298 | std::tie(args&: StopAfterName, args&: StopAfterInstanceNum) = |
| 299 | getPassNameAndInstanceNum(PassName: Opts.stop_after); |
| 300 | |
| 301 | StartBefore = getPassIDFromName(PassName: StartBeforeName); |
| 302 | StartAfter = getPassIDFromName(PassName: StartAfterName); |
| 303 | StopBefore = getPassIDFromName(PassName: StopBeforeName); |
| 304 | StopAfter = getPassIDFromName(PassName: StopAfterName); |
| 305 | if (StartBefore && StartAfter) |
| 306 | reportFatalUsageError(reason: Twine(StartBeforeOptName) + Twine(" and " ) + |
| 307 | Twine(StartAfterOptName) + Twine(" specified!" )); |
| 308 | if (StopBefore && StopAfter) |
| 309 | reportFatalUsageError(reason: Twine(StopBeforeOptName) + Twine(" and " ) + |
| 310 | Twine(StopAfterOptName) + Twine(" specified!" )); |
| 311 | Started = (StartAfter == nullptr) && (StartBefore == nullptr); |
| 312 | } |
| 313 | |
| 314 | static cl::boolOrDefault toBoolOrDefault(BoolOrDefault B) { |
| 315 | if (B == BoolOrDefault::Default) |
| 316 | return cl::boolOrDefault::BOU_UNSET; |
| 317 | return B == BoolOrDefault::True ? cl::boolOrDefault::BOU_TRUE |
| 318 | : cl::boolOrDefault::BOU_FALSE; |
| 319 | } |
| 320 | |
| 321 | static MachineVerifierMode |
| 322 | computeMachineVerifierMode(const CodeGenOptions &Opts, bool VerifyEach) { |
| 323 | if (VerifyEach) |
| 324 | return MachineVerifierMode::Each; |
| 325 | return Opts.disable_mir_output_verify ? MachineVerifierMode::None |
| 326 | : MachineVerifierMode::End; |
| 327 | } |
| 328 | |
| 329 | CGPassBuilderOption llvm::getCGPassBuilderOption(const TargetMachine &TM) { |
| 330 | const CodeGenOptions &Opts = getOpts(TM); |
| 331 | CGPassBuilderOption Opt; |
| 332 | Opt.EnableGlobalISelAbort = Opts.global_isel_abort; |
| 333 | if (Opts.enable_ipra != BoolOrDefault::Default) |
| 334 | Opt.EnableIPRA = Opts.enable_ipra == BoolOrDefault::True; |
| 335 | Opt.EnableFastISelOption = toBoolOrDefault(B: Opts.fast_isel); |
| 336 | Opt.EnableRegAllocFastTied = toBoolOrDefault(B: Opts.regalloc_fast_tied); |
| 337 | Opt.EnableMachinePipeliner = toBoolOrDefault(B: Opts.enable_pipeliner); |
| 338 | Opt.EnableGlobalISelOption = toBoolOrDefault(B: Opts.global_isel); |
| 339 | Opt.VerifyMachineCode = computeMachineVerifierMode( |
| 340 | Opts, VerifyEach: valueOr(X: Opts.verify_machineinstrs, Default: false)); |
| 341 | Opt.DisableAtExitBasedGlobalDtorLowering = |
| 342 | Opts.disable_atexit_based_global_dtor_lowering; |
| 343 | Opt.DisableExpandReductions = Opts.disable_expand_reductions; |
| 344 | Opt.PrintAfterISel = Opts.print_after_isel; |
| 345 | Opt.FSProfileFile = Opts.fs_profile_file.str(); |
| 346 | Opt.EnableGCEmptyBlocks = Opts.enable_gc_empty_basic_blocks; |
| 347 | Opt.EnableBlockPlacementStats = Opts.enable_block_placement_stats; |
| 348 | Opt.EnableGlobalMergeFunc = Opts.enable_global_merge_func; |
| 349 | Opt.EnableImplicitNullChecks = Opts.enable_implicit_null_checks; |
| 350 | Opt.EnableMachineOutliner = EnableMachineOutliner; |
| 351 | Opt.MISchedPostRA = Opts.misched_postra; |
| 352 | Opt.DisableLSR = Opts.disable_lsr; |
| 353 | Opt.DisableConstantHoisting = Opts.disable_constant_hoisting; |
| 354 | Opt.DisableCGP = Opts.disable_cgp; |
| 355 | Opt.DisablePartialLibcallInlining = Opts.disable_partial_libcall_inlining; |
| 356 | Opt.DisableSelectOptimize = Opts.disable_select_optimize; |
| 357 | Opt.PrintISelInput = Opts.print_isel_input; |
| 358 | Opt.PrintRegUsage = Opts.print_regusage; |
| 359 | Opt.DebugifyAndStripAll = toBoolOrDefault(B: Opts.debugify_and_strip_all_safe); |
| 360 | Opt.DebugifyCheckAndStripAll = |
| 361 | toBoolOrDefault(B: Opts.debugify_check_and_strip_all_safe); |
| 362 | Opt.DisableRAFSProfileLoader = Opts.disable_ra_fsprofile_loader; |
| 363 | Opt.DisableCFIFixup = Opts.disable_cfi_fixup; |
| 364 | return Opt; |
| 365 | } |
| 366 | |
| 367 | void llvm::registerCodeGenCallback(PassInstrumentationCallbacks &PIC, |
| 368 | TargetMachine &TM) { |
| 369 | const CodeGenOptions &Opts = getOpts(TM); |
| 370 | |
| 371 | // Register a callback for disabling passes. |
| 372 | PIC.registerShouldRunOptionalPassCallback(C: [&Opts](StringRef P, IRUnitRef) { |
| 373 | |
| 374 | #define DISABLE_PASS(Option, Name) \ |
| 375 | if (Option && P.contains(#Name)) \ |
| 376 | return false; |
| 377 | DISABLE_PASS(Opts.disable_block_placement, MachineBlockPlacementPass) |
| 378 | DISABLE_PASS(Opts.disable_branch_fold, BranchFolderPass) |
| 379 | DISABLE_PASS(Opts.disable_copyprop, MachineCopyPropagationPass) |
| 380 | DISABLE_PASS(Opts.disable_early_ifcvt, EarlyIfConverterLegacyPass) |
| 381 | DISABLE_PASS(Opts.disable_early_taildup, EarlyTailDuplicatePass) |
| 382 | DISABLE_PASS(Opts.disable_machine_cse, MachineCSELegacyPass) |
| 383 | DISABLE_PASS(Opts.disable_machine_dce, DeadMachineInstructionElimPass) |
| 384 | DISABLE_PASS(Opts.disable_machine_licm, EarlyMachineLICMPass) |
| 385 | DISABLE_PASS(Opts.disable_machine_sink, MachineSinkingPass) |
| 386 | DISABLE_PASS(Opts.disable_postra_machine_licm, MachineLICMPass) |
| 387 | DISABLE_PASS(Opts.disable_postra_machine_sink, PostRAMachineSinkingPass) |
| 388 | DISABLE_PASS(Opts.disable_post_ra, PostRASchedulerPass) |
| 389 | DISABLE_PASS(Opts.disable_ssc, StackSlotColoringPass) |
| 390 | DISABLE_PASS(Opts.disable_tail_duplicate, TailDuplicatePass) |
| 391 | |
| 392 | return true; |
| 393 | }); |
| 394 | } |
| 395 | |
| 396 | Expected<TargetPassConfig::StartStopInfo> |
| 397 | TargetPassConfig::getStartStopInfo(const TargetMachine &TM, |
| 398 | PassInstrumentationCallbacks &PIC) { |
| 399 | const CodeGenOptions &Opts = getOpts(TM); |
| 400 | auto [StartBefore, StartBeforeInstanceNum] = |
| 401 | getPassNameAndInstanceNum(PassName: Opts.start_before); |
| 402 | auto [StartAfter, StartAfterInstanceNum] = |
| 403 | getPassNameAndInstanceNum(PassName: Opts.start_after); |
| 404 | auto [StopBefore, StopBeforeInstanceNum] = |
| 405 | getPassNameAndInstanceNum(PassName: Opts.stop_before); |
| 406 | auto [StopAfter, StopAfterInstanceNum] = |
| 407 | getPassNameAndInstanceNum(PassName: Opts.stop_after); |
| 408 | |
| 409 | if (!StartBefore.empty() && !StartAfter.empty()) |
| 410 | return make_error<StringError>( |
| 411 | Args: Twine(StartBeforeOptName) + " and " + StartAfterOptName + " specified!" , |
| 412 | Args: std::make_error_code(e: std::errc::invalid_argument)); |
| 413 | if (!StopBefore.empty() && !StopAfter.empty()) |
| 414 | return make_error<StringError>( |
| 415 | Args: Twine(StopBeforeOptName) + " and " + StopAfterOptName + " specified!" , |
| 416 | Args: std::make_error_code(e: std::errc::invalid_argument)); |
| 417 | |
| 418 | StartStopInfo Result; |
| 419 | Result.StartPass = StartBefore.empty() ? StartAfter : StartBefore; |
| 420 | Result.StopPass = StopBefore.empty() ? StopAfter : StopBefore; |
| 421 | Result.StartInstanceNum = |
| 422 | StartBefore.empty() ? StartAfterInstanceNum : StartBeforeInstanceNum; |
| 423 | Result.StopInstanceNum = |
| 424 | StopBefore.empty() ? StopAfterInstanceNum : StopBeforeInstanceNum; |
| 425 | Result.StartAfter = !StartAfter.empty(); |
| 426 | Result.StopAfter = !StopAfter.empty(); |
| 427 | Result.StartInstanceNum += Result.StartInstanceNum == 0; |
| 428 | Result.StopInstanceNum += Result.StopInstanceNum == 0; |
| 429 | return Result; |
| 430 | } |
| 431 | |
| 432 | // Out of line constructor provides default values for pass options and |
| 433 | // registers all common codegen passes. |
| 434 | TargetPassConfig::TargetPassConfig(TargetMachine &TM, PassManagerBase &PM) |
| 435 | : ImmutablePass(ID), PM(&PM), Opts(getOpts(TM)), TM(&TM) { |
| 436 | Impl = new PassConfigImpl(); |
| 437 | |
| 438 | PassRegistry &PR = *PassRegistry::getPassRegistry(); |
| 439 | // Register all target independent codegen passes to activate their PassIDs, |
| 440 | // including this pass itself. |
| 441 | initializeCodeGen(PR); |
| 442 | |
| 443 | initializeLibcallLoweringInfoWrapperPass(PR); |
| 444 | |
| 445 | // Also register alias analysis passes required by codegen passes. |
| 446 | initializeBasicAAWrapperPassPass(PR); |
| 447 | initializeAAResultsWrapperPassPass(PR); |
| 448 | |
| 449 | if (Opts.enable_ipra != BoolOrDefault::Default) { |
| 450 | TM.Options.EnableIPRA = Opts.enable_ipra == BoolOrDefault::True; |
| 451 | } else { |
| 452 | // If not explicitly specified, use target default. |
| 453 | TM.Options.EnableIPRA |= TM.useIPRA(); |
| 454 | } |
| 455 | |
| 456 | if (TM.Options.EnableIPRA) |
| 457 | setRequiresCodeGenSCCOrder(); |
| 458 | |
| 459 | if (Opts.regalloc_fast_tied != BoolOrDefault::Default) |
| 460 | TM.setEnableTiedFastRegAlloc(Opts.regalloc_fast_tied == |
| 461 | BoolOrDefault::True); |
| 462 | |
| 463 | if (Opts.global_isel_abort) |
| 464 | TM.Options.GlobalISelAbort = *Opts.global_isel_abort; |
| 465 | |
| 466 | bool VerifyEach = valueOr(X: Opts.verify_machineinstrs, Default: false); |
| 467 | #ifdef EXPENSIVE_CHECKS |
| 468 | if (Opts.verify_machineinstrs == BoolOrDefault::Default) |
| 469 | VerifyEach = TM.isMachineVerifierClean(); |
| 470 | #endif |
| 471 | VerifyMode = computeMachineVerifierMode(Opts, VerifyEach); |
| 472 | |
| 473 | if (Opts.function_splitting) |
| 474 | TM.Options.FunctionSplitting = *Opts.function_splitting; |
| 475 | |
| 476 | setStartStopPasses(); |
| 477 | } |
| 478 | |
| 479 | CodeGenOptLevel TargetPassConfig::getOptLevel() const { |
| 480 | return TM->getOptLevel(); |
| 481 | } |
| 482 | |
| 483 | /// Insert InsertedPassID pass after TargetPassID. |
| 484 | void TargetPassConfig::insertPass(AnalysisID TargetPassID, |
| 485 | IdentifyingPassPtr InsertedPassID) { |
| 486 | assert(((!InsertedPassID.isInstance() && |
| 487 | TargetPassID != InsertedPassID.getID()) || |
| 488 | (InsertedPassID.isInstance() && |
| 489 | TargetPassID != InsertedPassID.getInstance()->getPassID())) && |
| 490 | "Insert a pass after itself!" ); |
| 491 | Impl->InsertedPasses.emplace_back(Args&: TargetPassID, Args&: InsertedPassID); |
| 492 | } |
| 493 | |
| 494 | /// createPassConfig - Create a pass configuration object to be used by |
| 495 | /// addPassToEmitX methods for generating a pipeline of CodeGen passes. |
| 496 | /// |
| 497 | /// Targets may override this to extend TargetPassConfig. |
| 498 | TargetPassConfig * |
| 499 | CodeGenTargetMachineImpl::createPassConfig(PassManagerBase &PM) { |
| 500 | return new TargetPassConfig(*this, PM); |
| 501 | } |
| 502 | |
| 503 | template <> Pass *llvm::callDefaultCtor<TargetPassConfig>() { |
| 504 | reportFatalUsageError(reason: "trying to construct TargetPassConfig without a target " |
| 505 | "machine. Scheduling a CodeGen pass without a target " |
| 506 | "triple set?" ); |
| 507 | } |
| 508 | |
| 509 | bool TargetPassConfig::willCompleteCodeGenPipeline() const { |
| 510 | return Opts.stop_before.empty() && Opts.stop_after.empty(); |
| 511 | } |
| 512 | |
| 513 | bool TargetPassConfig::hasLimitedCodeGenPipeline() const { |
| 514 | return !Opts.start_before.empty() || !Opts.start_after.empty() || |
| 515 | !willCompleteCodeGenPipeline(); |
| 516 | } |
| 517 | |
| 518 | static std::string formatLimitReason(StringRef StartAfter, |
| 519 | StringRef StartBefore, StringRef StopAfter, |
| 520 | StringRef StopBefore) { |
| 521 | std::string Res; |
| 522 | StringRef PassNames[] = {StartAfter, StartBefore, StopAfter, StopBefore}; |
| 523 | static const char *OptNames[] = {StartAfterOptName, StartBeforeOptName, |
| 524 | StopAfterOptName, StopBeforeOptName}; |
| 525 | bool IsFirst = true; |
| 526 | for (int Idx = 0; Idx < 4; ++Idx) |
| 527 | if (!PassNames[Idx].empty()) { |
| 528 | if (!IsFirst) |
| 529 | Res += " and " ; |
| 530 | IsFirst = false; |
| 531 | Res += OptNames[Idx]; |
| 532 | } |
| 533 | return Res; |
| 534 | } |
| 535 | |
| 536 | std::string TargetPassConfig::getLimitedCodeGenPipelineReason() const { |
| 537 | return formatLimitReason(StartAfter: Opts.start_after, StartBefore: Opts.start_before, StopAfter: Opts.stop_after, |
| 538 | StopBefore: Opts.stop_before); |
| 539 | } |
| 540 | |
| 541 | std::string TargetPassConfig::StartStopInfo::getLimitReason() const { |
| 542 | return formatLimitReason( |
| 543 | StartAfter: StartAfter ? StartPass : "" , StartBefore: StartAfter ? "" : StartPass, |
| 544 | StopAfter: StopAfter ? StopPass : "" , StopBefore: StopAfter ? "" : StopPass); |
| 545 | } |
| 546 | |
| 547 | // Helper to verify the analysis is really immutable. |
| 548 | void TargetPassConfig::setOpt(bool &Opt, bool Val) { |
| 549 | assert(!Initialized && "PassConfig is immutable" ); |
| 550 | Opt = Val; |
| 551 | } |
| 552 | |
| 553 | void TargetPassConfig::substitutePass(AnalysisID StandardID, |
| 554 | IdentifyingPassPtr TargetID) { |
| 555 | Impl->TargetPasses[StandardID] = TargetID; |
| 556 | } |
| 557 | |
| 558 | IdentifyingPassPtr TargetPassConfig::getPassSubstitution(AnalysisID ID) const { |
| 559 | DenseMap<AnalysisID, IdentifyingPassPtr>::const_iterator |
| 560 | I = Impl->TargetPasses.find(Val: ID); |
| 561 | if (I == Impl->TargetPasses.end()) |
| 562 | return ID; |
| 563 | return I->second; |
| 564 | } |
| 565 | |
| 566 | bool TargetPassConfig::isPassSubstitutedOrOverridden(AnalysisID ID) const { |
| 567 | IdentifyingPassPtr TargetID = getPassSubstitution(ID); |
| 568 | IdentifyingPassPtr FinalPtr = overridePass(Opts, StandardID: ID, TargetID); |
| 569 | return !FinalPtr.isValid() || FinalPtr.isInstance() || |
| 570 | FinalPtr.getID() != ID; |
| 571 | } |
| 572 | |
| 573 | /// Add a pass to the PassManager if that pass is supposed to be run. If the |
| 574 | /// Started/Stopped flags indicate either that the compilation should start at |
| 575 | /// a later pass or that it should stop after an earlier pass, then do not add |
| 576 | /// the pass. Finally, compare the current pass against the StartAfter |
| 577 | /// and StopAfter options and change the Started/Stopped flags accordingly. |
| 578 | void TargetPassConfig::addPass(Pass *P) { |
| 579 | assert(!Initialized && "PassConfig is immutable" ); |
| 580 | |
| 581 | // Cache the Pass ID here in case the pass manager finds this pass is |
| 582 | // redundant with ones already scheduled / available, and deletes it. |
| 583 | // Fundamentally, once we add the pass to the manager, we no longer own it |
| 584 | // and shouldn't reference it. |
| 585 | AnalysisID PassID = P->getPassID(); |
| 586 | |
| 587 | if (StartBefore == PassID && StartBeforeCount++ == StartBeforeInstanceNum) |
| 588 | Started = true; |
| 589 | if (StopBefore == PassID && StopBeforeCount++ == StopBeforeInstanceNum) |
| 590 | Stopped = true; |
| 591 | if (Started && !Stopped) { |
| 592 | if (AddingMachinePasses) { |
| 593 | // Construct banner message before PM->add() as that may delete the pass. |
| 594 | std::string Banner = |
| 595 | std::string("After " ) + std::string(P->getPassName()); |
| 596 | addMachinePrePasses(); |
| 597 | PM->add(P); |
| 598 | addMachinePostPasses(Banner); |
| 599 | } else { |
| 600 | PM->add(P); |
| 601 | } |
| 602 | |
| 603 | // Add the passes after the pass P if there is any. |
| 604 | for (const auto &IP : Impl->InsertedPasses) |
| 605 | if (IP.TargetPassID == PassID) |
| 606 | addPass(P: IP.getInsertedPass()); |
| 607 | } else { |
| 608 | delete P; |
| 609 | } |
| 610 | |
| 611 | if (StopAfter == PassID && StopAfterCount++ == StopAfterInstanceNum) |
| 612 | Stopped = true; |
| 613 | |
| 614 | if (StartAfter == PassID && StartAfterCount++ == StartAfterInstanceNum) |
| 615 | Started = true; |
| 616 | if (Stopped && !Started) |
| 617 | reportFatalUsageError(reason: "Cannot stop compilation after pass that is not run" ); |
| 618 | } |
| 619 | |
| 620 | /// Add a CodeGen pass at this point in the pipeline after checking for target |
| 621 | /// and command line overrides. |
| 622 | /// |
| 623 | /// addPass cannot return a pointer to the pass instance because is internal the |
| 624 | /// PassManager and the instance we create here may already be freed. |
| 625 | AnalysisID TargetPassConfig::addPass(AnalysisID PassID) { |
| 626 | IdentifyingPassPtr TargetID = getPassSubstitution(ID: PassID); |
| 627 | IdentifyingPassPtr FinalPtr = overridePass(Opts, StandardID: PassID, TargetID); |
| 628 | if (!FinalPtr.isValid()) |
| 629 | return nullptr; |
| 630 | |
| 631 | Pass *P; |
| 632 | if (FinalPtr.isInstance()) |
| 633 | P = FinalPtr.getInstance(); |
| 634 | else { |
| 635 | P = Pass::createPass(ID: FinalPtr.getID()); |
| 636 | if (!P) |
| 637 | llvm_unreachable("Pass ID not registered" ); |
| 638 | } |
| 639 | AnalysisID FinalID = P->getPassID(); |
| 640 | addPass(P); // Ends the lifetime of P. |
| 641 | |
| 642 | return FinalID; |
| 643 | } |
| 644 | |
| 645 | void TargetPassConfig::printAndVerify(const std::string &Banner) { |
| 646 | addPrintPass(Banner); |
| 647 | addVerifyPass(Banner); |
| 648 | } |
| 649 | |
| 650 | void TargetPassConfig::addPrintPass(const std::string &Banner) { |
| 651 | if (Opts.print_after_isel) |
| 652 | PM->add(P: createMachineFunctionPrinterPass(OS&: dbgs(), Banner)); |
| 653 | } |
| 654 | |
| 655 | void TargetPassConfig::addVerifyPass(const std::string &Banner) { |
| 656 | if (VerifyMode == MachineVerifierMode::Each) |
| 657 | PM->add(P: createMachineVerifierPass(Banner)); |
| 658 | } |
| 659 | |
| 660 | void TargetPassConfig::addDebugifyPass() { |
| 661 | PM->add(P: createDebugifyMachineModulePass()); |
| 662 | } |
| 663 | |
| 664 | void TargetPassConfig::addStripDebugPass() { |
| 665 | PM->add(P: createStripDebugMachineModuleLegacyPass(/*OnlyDebugified=*/true)); |
| 666 | } |
| 667 | |
| 668 | void TargetPassConfig::addCheckDebugPass() { |
| 669 | PM->add(P: createCheckDebugMachineModuleLegacyPass()); |
| 670 | } |
| 671 | |
| 672 | void TargetPassConfig::addMachinePrePasses(bool AllowDebugify) { |
| 673 | if (AllowDebugify && DebugifyIsSafe && |
| 674 | (valueOr(X: Opts.debugify_and_strip_all_safe, Default: false) || |
| 675 | valueOr(X: Opts.debugify_check_and_strip_all_safe, Default: false))) |
| 676 | addDebugifyPass(); |
| 677 | } |
| 678 | |
| 679 | void TargetPassConfig::addMachinePostPasses(const std::string &Banner) { |
| 680 | if (DebugifyIsSafe) { |
| 681 | if (valueOr(X: Opts.debugify_check_and_strip_all_safe, Default: false)) { |
| 682 | addCheckDebugPass(); |
| 683 | addStripDebugPass(); |
| 684 | } else if (valueOr(X: Opts.debugify_and_strip_all_safe, Default: false)) |
| 685 | addStripDebugPass(); |
| 686 | } |
| 687 | addVerifyPass(Banner); |
| 688 | } |
| 689 | |
| 690 | /// Add common target configurable passes that perform LLVM IR to IR transforms |
| 691 | /// following machine independent optimization. |
| 692 | void TargetPassConfig::addIRPasses() { |
| 693 | // Before running any passes, run the verifier to determine if the input |
| 694 | // coming from the front-end and/or optimizer is valid. |
| 695 | if (!DisableVerify) |
| 696 | addPass(P: createVerifierPass()); |
| 697 | |
| 698 | if (getOptLevel() != CodeGenOptLevel::None) { |
| 699 | // Basic AliasAnalysis support. |
| 700 | // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that |
| 701 | // BasicAliasAnalysis wins if they disagree. This is intended to help |
| 702 | // support "obvious" type-punning idioms. |
| 703 | addPass(P: createTypeBasedAAWrapperPass()); |
| 704 | addPass(P: createScopedNoAliasAAWrapperPass()); |
| 705 | addPass(P: createBasicAAWrapperPass()); |
| 706 | |
| 707 | // Run loop strength reduction before anything else. |
| 708 | if (EnableLoopStrengthReduce && !Opts.disable_lsr) { |
| 709 | addPass(P: createCanonicalizeFreezeInLoopsPass()); |
| 710 | addPass(P: createLoopStrengthReducePass()); |
| 711 | if (EnableLoopTermFold) |
| 712 | addPass(P: createLoopTermFoldPass()); |
| 713 | } |
| 714 | } |
| 715 | |
| 716 | // Run GC lowering passes for builtin collectors |
| 717 | // TODO: add a pass insertion point here |
| 718 | addPass(PassID: &GCLoweringID); |
| 719 | addPass(PassID: &ShadowStackGCLoweringID); |
| 720 | |
| 721 | // For MachO, lower @llvm.global_dtors into @llvm.global_ctors with |
| 722 | // __cxa_atexit() calls to avoid emitting the deprecated __mod_term_func. |
| 723 | if (TM->getTargetTriple().isOSBinFormatMachO() && |
| 724 | !Opts.disable_atexit_based_global_dtor_lowering) |
| 725 | addPass(P: createLowerGlobalDtorsLegacyPass()); |
| 726 | |
| 727 | // Make sure that no unreachable blocks are instruction selected. |
| 728 | addPass(P: createUnreachableBlockEliminationPass()); |
| 729 | |
| 730 | // Prepare expensive constants for SelectionDAG. |
| 731 | if (getOptLevel() != CodeGenOptLevel::None && !Opts.disable_constant_hoisting) |
| 732 | addPass(P: createConstantHoistingPass()); |
| 733 | |
| 734 | if (getOptLevel() != CodeGenOptLevel::None && |
| 735 | !Opts.disable_replace_with_vec_lib) |
| 736 | addPass(P: createReplaceWithVeclibLegacyPass()); |
| 737 | |
| 738 | if (getOptLevel() != CodeGenOptLevel::None && |
| 739 | !Opts.disable_partial_libcall_inlining) |
| 740 | addPass(P: createPartiallyInlineLibCallsPass()); |
| 741 | |
| 742 | // Instrument function entry after all inlining. |
| 743 | addPass(P: createPostInlineEntryExitInstrumenterPass()); |
| 744 | |
| 745 | // Add scalarization of target's unsupported masked memory intrinsics pass. |
| 746 | // the unsupported intrinsic will be replaced with a chain of basic blocks, |
| 747 | // that stores/loads element one-by-one if the appropriate mask bit is set. |
| 748 | addPass(P: createScalarizeMaskedMemIntrinLegacyPass()); |
| 749 | |
| 750 | // Expand reduction intrinsics into shuffle sequences if the target wants to. |
| 751 | // Allow disabling it for testing purposes. |
| 752 | if (!Opts.disable_expand_reductions) |
| 753 | addPass(P: createExpandReductionsPass()); |
| 754 | |
| 755 | // Convert conditional moves to conditional jumps when profitable. |
| 756 | if (getOptLevel() != CodeGenOptLevel::None && !Opts.disable_select_optimize) |
| 757 | addPass(P: createSelectOptimizePass()); |
| 758 | |
| 759 | if (Opts.enable_global_merge_func) |
| 760 | addPass(P: createGlobalMergeFuncPass()); |
| 761 | |
| 762 | if (TM->getTargetTriple().isOSWindows()) |
| 763 | addPass(P: createWindowsSecureHotPatchingPass()); |
| 764 | } |
| 765 | |
| 766 | /// Turn exception handling constructs into something the code generators can |
| 767 | /// handle. |
| 768 | void TargetPassConfig::addPassesToHandleExceptions() { |
| 769 | // The module's exception model is not known when the pipeline is built, so |
| 770 | // schedule the preparation pass for every model the target supports. Each |
| 771 | // pass does nothing for other models. |
| 772 | bool HasEHPrep = false; |
| 773 | |
| 774 | // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both |
| 775 | // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise, |
| 776 | // catch info can get misplaced when a selector ends up more than one block |
| 777 | // removed from the parent invoke(s). This could happen when a landing |
| 778 | // pad is shared by multiple invokes and is also a target of a normal |
| 779 | // edge from elsewhere. |
| 780 | if (TM->supportsExceptionModel(Model: ExceptionHandling::SjLj)) { |
| 781 | addPass(P: createSjLjEHPreparePass()); |
| 782 | HasEHPrep = true; |
| 783 | } |
| 784 | |
| 785 | // We support using both GCC-style and MSVC-style exceptions on Windows, so |
| 786 | // WinEHPrepare runs in addition to DwarfEHPrepare below. Each pass will only |
| 787 | // actually run if it recognizes the personality function. |
| 788 | // |
| 789 | // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs |
| 790 | // on catchpads and cleanuppads because it does not outline them into |
| 791 | // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we |
| 792 | // should remove PHIs there. WinEHPrepare derives this from the Wasm |
| 793 | // personality, so no explicit flag is needed here. |
| 794 | if (TM->supportsExceptionModel(Model: ExceptionHandling::WinEH) || |
| 795 | TM->supportsExceptionModel(Model: ExceptionHandling::Wasm)) { |
| 796 | addPass(P: createWinEHPass()); |
| 797 | HasEHPrep = true; |
| 798 | } |
| 799 | |
| 800 | if (TM->supportsExceptionModel(Model: ExceptionHandling::Wasm)) |
| 801 | addPass(P: createWasmEHPass()); |
| 802 | |
| 803 | if (TM->supportsExceptionModel(Model: ExceptionHandling::DwarfCFI)) { |
| 804 | addPass(P: createDwarfEHPass(OptLevel: getOptLevel())); |
| 805 | HasEHPrep = true; |
| 806 | } |
| 807 | |
| 808 | // If the target supports no exception model, nothing else lowers invokes. |
| 809 | if (!HasEHPrep) { |
| 810 | addPass(P: createLowerInvokePass()); |
| 811 | |
| 812 | // The lower invoke pass may create unreachable code. Remove it. |
| 813 | addPass(P: createUnreachableBlockEliminationPass()); |
| 814 | } |
| 815 | } |
| 816 | |
| 817 | /// Add pass to prepare the LLVM IR for code generation. This should be done |
| 818 | /// before exception handling preparation passes. |
| 819 | void TargetPassConfig::addCodeGenPrepare() { |
| 820 | if (getOptLevel() != CodeGenOptLevel::None && !Opts.disable_cgp) |
| 821 | addPass(P: createCodeGenPrepareLegacyPass()); |
| 822 | } |
| 823 | |
| 824 | /// Add common passes that perform LLVM IR to IR transforms in preparation for |
| 825 | /// instruction selection. |
| 826 | void TargetPassConfig::addISelPrepare() { |
| 827 | addPreISel(); |
| 828 | |
| 829 | // Force codegen to run according to the callgraph. |
| 830 | if (requiresCodeGenSCCOrder()) |
| 831 | addPass(P: new DummyCGSCCPass); |
| 832 | |
| 833 | addPass(P: createInlineAsmPreparePass()); |
| 834 | |
| 835 | // Add both the safe stack and the stack protection passes: each of them will |
| 836 | // only protect functions that have corresponding attributes. |
| 837 | addPass(P: createSafeStackPass()); |
| 838 | addPass(P: createStackProtectorPass()); |
| 839 | |
| 840 | if (Opts.print_isel_input) |
| 841 | addPass(P: createPrintFunctionPass( |
| 842 | OS&: dbgs(), Banner: "\n\n*** Final LLVM Code input to ISel ***\n" )); |
| 843 | |
| 844 | // All passes which modify the LLVM IR are now complete; run the verifier |
| 845 | // to ensure that the IR is valid. |
| 846 | if (!DisableVerify) |
| 847 | addPass(P: createVerifierPass()); |
| 848 | } |
| 849 | |
| 850 | bool TargetPassConfig::addCoreISelPasses() { |
| 851 | // Enable FastISel with -fast-isel, but allow that to be overridden. |
| 852 | TM->setO0WantsFastISel(valueOr(X: Opts.fast_isel, Default: true)); |
| 853 | |
| 854 | // Determine an instruction selector. |
| 855 | enum class SelectorType { SelectionDAG, FastISel, GlobalISel }; |
| 856 | SelectorType Selector; |
| 857 | |
| 858 | if (valueOr(X: Opts.fast_isel, Default: false)) |
| 859 | Selector = SelectorType::FastISel; |
| 860 | else if (valueOr(X: Opts.global_isel, Default: false) || |
| 861 | (TM->Options.EnableGlobalISel && valueOr(X: Opts.global_isel, Default: true))) |
| 862 | Selector = SelectorType::GlobalISel; |
| 863 | else if (TM->getOptLevel() == CodeGenOptLevel::None && |
| 864 | TM->getO0WantsFastISel()) |
| 865 | Selector = SelectorType::FastISel; |
| 866 | else |
| 867 | Selector = SelectorType::SelectionDAG; |
| 868 | |
| 869 | // Set consistently TM->Options.EnableFastISel and EnableGlobalISel. |
| 870 | if (Selector == SelectorType::FastISel) { |
| 871 | TM->setFastISel(true); |
| 872 | TM->setGlobalISel(false); |
| 873 | } else if (Selector == SelectorType::GlobalISel) { |
| 874 | TM->setFastISel(false); |
| 875 | TM->setGlobalISel(true); |
| 876 | } |
| 877 | |
| 878 | // FIXME: Injecting into the DAGISel pipeline seems to cause issues with |
| 879 | // analyses needing to be re-run. This can result in being unable to |
| 880 | // schedule passes (particularly with 'Function Alias Analysis |
| 881 | // Results'). It's not entirely clear why but AFAICT this seems to be |
| 882 | // due to one FunctionPassManager not being able to use analyses from a |
| 883 | // previous one. As we're injecting a ModulePass we break the usual |
| 884 | // pass manager into two. GlobalISel with the fallback path disabled |
| 885 | // and -run-pass seem to be unaffected. The majority of GlobalISel |
| 886 | // testing uses -run-pass so this probably isn't too bad. |
| 887 | SaveAndRestore SavedDebugifyIsSafe(DebugifyIsSafe); |
| 888 | if (Selector != SelectorType::GlobalISel || !isGlobalISelAbortEnabled()) |
| 889 | DebugifyIsSafe = false; |
| 890 | |
| 891 | // Add instruction selector passes for global isel if enabled. |
| 892 | if (Selector == SelectorType::GlobalISel) { |
| 893 | SaveAndRestore SavedAddingMachinePasses(AddingMachinePasses, true); |
| 894 | if (addIRTranslator()) |
| 895 | return true; |
| 896 | |
| 897 | addPreLegalizeMachineIR(); |
| 898 | |
| 899 | if (addLegalizeMachineIR()) |
| 900 | return true; |
| 901 | |
| 902 | // Before running the register bank selector, ask the target if it |
| 903 | // wants to run some passes. |
| 904 | addPreRegBankSelect(); |
| 905 | |
| 906 | if (addRegBankSelect()) |
| 907 | return true; |
| 908 | |
| 909 | addPreGlobalInstructionSelect(); |
| 910 | |
| 911 | if (addGlobalInstructionSelect()) |
| 912 | return true; |
| 913 | } |
| 914 | |
| 915 | // Pass to reset the MachineFunction if the ISel failed. Outside of the above |
| 916 | // if so that the verifier is not added to it. |
| 917 | if (Selector == SelectorType::GlobalISel) |
| 918 | addPass(P: createResetMachineFunctionLegacyPass( |
| 919 | EmitFallbackDiag: reportDiagnosticWhenGlobalISelFallback(), AbortOnFailedISel: isGlobalISelAbortEnabled())); |
| 920 | |
| 921 | // Run the SDAG InstSelector, providing a fallback path when we do not want to |
| 922 | // abort on not-yet-supported input. |
| 923 | if (Selector != SelectorType::GlobalISel || !isGlobalISelAbortEnabled()) |
| 924 | if (addInstSelector()) |
| 925 | return true; |
| 926 | |
| 927 | // Expand pseudo-instructions emitted by ISel. Don't run the verifier before |
| 928 | // FinalizeISel. |
| 929 | addPass(PassID: &FinalizeISelID); |
| 930 | |
| 931 | // Print the instruction selected machine code... |
| 932 | printAndVerify(Banner: "After Instruction Selection" ); |
| 933 | |
| 934 | return false; |
| 935 | } |
| 936 | |
| 937 | bool TargetPassConfig::addISelPasses() { |
| 938 | if (TM->useEmulatedTLS()) |
| 939 | addPass(P: createLowerEmuTLSPass()); |
| 940 | |
| 941 | PM->add(P: createTargetTransformInfoWrapperPass(TIRA: TM->getTargetIRAnalysis())); |
| 942 | // ObjCARCContract operates on ObjC intrinsics and must run before |
| 943 | // PreISelIntrinsicLowering. |
| 944 | if (getOptLevel() != CodeGenOptLevel::None) |
| 945 | addPass(P: createObjCARCContractPass()); |
| 946 | addPass(P: createPreISelIntrinsicLoweringPass()); |
| 947 | addPass(P: createExpandIRInstsPass(getOptLevel())); |
| 948 | addIRPasses(); |
| 949 | |
| 950 | if (Opts.codegen_pipeline_trigger_crash) |
| 951 | addPass(P: new TriggerCrashFunctionLegacyPass()); |
| 952 | |
| 953 | addCodeGenPrepare(); |
| 954 | addPassesToHandleExceptions(); |
| 955 | addISelPrepare(); |
| 956 | |
| 957 | return addCoreISelPasses(); |
| 958 | } |
| 959 | |
| 960 | /// -regalloc=... command line option. |
| 961 | static FunctionPass *useDefaultRegisterAllocator() { return nullptr; } |
| 962 | static cl::opt<RegisterRegAlloc::FunctionPassCtor, false, |
| 963 | RegisterPassParser<RegisterRegAlloc>> |
| 964 | RegAlloc("regalloc" , cl::Hidden, cl::init(Val: &useDefaultRegisterAllocator), |
| 965 | cl::desc("Register allocator to use" )); |
| 966 | |
| 967 | /// Add the complete set of target-independent postISel code generator passes. |
| 968 | /// |
| 969 | /// This can be read as the standard order of major LLVM CodeGen stages. Stages |
| 970 | /// with nontrivial configuration or multiple passes are broken out below in |
| 971 | /// add%Stage routines. |
| 972 | /// |
| 973 | /// Any TargetPassConfig::addXX routine may be overriden by the Target. The |
| 974 | /// addPre/Post methods with empty header implementations allow injecting |
| 975 | /// target-specific fixups just before or after major stages. Additionally, |
| 976 | /// targets have the flexibility to change pass order within a stage by |
| 977 | /// overriding default implementation of add%Stage routines below. Each |
| 978 | /// technique has maintainability tradeoffs because alternate pass orders are |
| 979 | /// not well supported. addPre/Post works better if the target pass is easily |
| 980 | /// tied to a common pass. But if it has subtle dependencies on multiple passes, |
| 981 | /// the target should override the stage instead. |
| 982 | /// |
| 983 | /// TODO: We could use a single addPre/Post(ID) hook to allow pass injection |
| 984 | /// before/after any target-independent pass. But it's currently overkill. |
| 985 | void TargetPassConfig::addMachinePasses() { |
| 986 | AddingMachinePasses = true; |
| 987 | |
| 988 | // Add passes that optimize machine instructions in SSA form. |
| 989 | if (getOptLevel() != CodeGenOptLevel::None) { |
| 990 | addMachineSSAOptimization(); |
| 991 | } else { |
| 992 | // If the target requests it, assign local variables to stack slots relative |
| 993 | // to one another and simplify frame index references where possible. |
| 994 | addPass(PassID: &LocalStackSlotAllocationID); |
| 995 | } |
| 996 | |
| 997 | if (TM->Options.EnableIPRA) |
| 998 | addPass(P: createRegUsageInfoPropPass()); |
| 999 | |
| 1000 | if (getOptLevel() != CodeGenOptLevel::None && |
| 1001 | valueOr(X: getOpts(TM: *TM).enable_pipeliner, Default: isMachinePipelinerEnabled())) |
| 1002 | addPass(PassID: &MachinePipelinerID); |
| 1003 | |
| 1004 | // Run pre-ra passes. |
| 1005 | addPreRegAlloc(); |
| 1006 | |
| 1007 | // Debugifying the register allocator passes seems to provoke some |
| 1008 | // non-determinism that affects CodeGen and there doesn't seem to be a point |
| 1009 | // where it becomes safe again so stop debugifying here. |
| 1010 | DebugifyIsSafe = false; |
| 1011 | |
| 1012 | // Add a FSDiscriminator pass right before RA, so that we could get |
| 1013 | // more precise SampleFDO profile for RA. |
| 1014 | if (EnableFSDiscriminator) { |
| 1015 | addPass(P: createMIRAddFSDiscriminatorsPass( |
| 1016 | P: sampleprof::FSDiscriminatorPass::Pass1)); |
| 1017 | const std::string ProfileFile = getFSProfileFile(Opts, TM); |
| 1018 | if (!ProfileFile.empty() && !Opts.disable_ra_fsprofile_loader) |
| 1019 | addPass(P: createMIRProfileLoaderPass( |
| 1020 | File: ProfileFile, RemappingFile: getFSRemappingFile(Opts, TM), |
| 1021 | P: sampleprof::FSDiscriminatorPass::Pass1, FS: nullptr)); |
| 1022 | } |
| 1023 | |
| 1024 | // Run register allocation and passes that are tightly coupled with it, |
| 1025 | // including phi elimination and scheduling. |
| 1026 | if (getOptimizeRegAlloc()) |
| 1027 | addOptimizedRegAlloc(); |
| 1028 | else |
| 1029 | addFastRegAlloc(); |
| 1030 | |
| 1031 | // Run post-ra passes. |
| 1032 | addPostRegAlloc(); |
| 1033 | |
| 1034 | addPass(PassID: &RemoveRedundantDebugValuesID); |
| 1035 | |
| 1036 | addPass(PassID: &FixupStatepointCallerSavedID); |
| 1037 | |
| 1038 | // Insert prolog/epilog code. Eliminate abstract frame index references... |
| 1039 | if (getOptLevel() != CodeGenOptLevel::None) { |
| 1040 | addPass(PassID: &PostRAMachineSinkingID); |
| 1041 | addPass(PassID: &ShrinkWrapID); |
| 1042 | } |
| 1043 | |
| 1044 | // Prolog/Epilog inserter needs a TargetMachine to instantiate. But only |
| 1045 | // do so if it hasn't been disabled, substituted, or overridden. |
| 1046 | if (!isPassSubstitutedOrOverridden(ID: &PrologEpilogCodeInserterID)) |
| 1047 | addPass(P: createPrologEpilogInserterPass()); |
| 1048 | |
| 1049 | /// Add passes that optimize machine instructions after register allocation. |
| 1050 | if (getOptLevel() != CodeGenOptLevel::None) |
| 1051 | addMachineLateOptimization(); |
| 1052 | |
| 1053 | // Expand pseudo instructions before second scheduling pass. |
| 1054 | addPass(PassID: &ExpandPostRAPseudosID); |
| 1055 | |
| 1056 | // Run pre-sched2 passes. |
| 1057 | addPreSched2(); |
| 1058 | |
| 1059 | if (Opts.enable_implicit_null_checks) |
| 1060 | addPass(PassID: &ImplicitNullChecksID); |
| 1061 | |
| 1062 | // Second pass scheduler. |
| 1063 | // Let Target optionally insert this pass by itself at some other |
| 1064 | // point. |
| 1065 | if (getOptLevel() != CodeGenOptLevel::None && |
| 1066 | !TM->targetSchedulesPostRAScheduling()) { |
| 1067 | if (Opts.misched_postra) |
| 1068 | addPass(PassID: &PostMachineSchedulerID); |
| 1069 | else |
| 1070 | addPass(PassID: &PostRASchedulerID); |
| 1071 | } |
| 1072 | |
| 1073 | // GC |
| 1074 | addGCPasses(); |
| 1075 | |
| 1076 | // Basic block placement. |
| 1077 | if (getOptLevel() != CodeGenOptLevel::None) |
| 1078 | addBlockPlacement(); |
| 1079 | |
| 1080 | // Insert before XRay Instrumentation. |
| 1081 | addPass(PassID: &FEntryInserterID); |
| 1082 | |
| 1083 | addPass(PassID: &XRayInstrumentationID); |
| 1084 | addPass(PassID: &PatchableFunctionID); |
| 1085 | |
| 1086 | addPreEmitPass(); |
| 1087 | |
| 1088 | if (TM->Options.EnableIPRA) |
| 1089 | // Collect register usage information and produce a register mask of |
| 1090 | // clobbered registers, to be used to optimize call sites. |
| 1091 | addPass(P: createRegUsageInfoCollector()); |
| 1092 | |
| 1093 | // FIXME: Some backends are incompatible with running the verifier after |
| 1094 | // addPreEmitPass. Maybe only pass "false" here for those targets? |
| 1095 | addPass(PassID: &FuncletLayoutID); |
| 1096 | |
| 1097 | addPass(PassID: &RemoveLoadsIntoFakeUsesID); |
| 1098 | addPass(PassID: &StackMapLivenessID); |
| 1099 | addPass(PassID: &LiveDebugValuesID); |
| 1100 | addPass(PassID: &MachineSanitizerBinaryMetadataID); |
| 1101 | |
| 1102 | if (TM->Options.EnableMachineOutliner && |
| 1103 | getOptLevel() != CodeGenOptLevel::None && |
| 1104 | EnableMachineOutliner != RunOutliner::NeverOutline) { |
| 1105 | if (EnableMachineOutliner != RunOutliner::TargetDefault || |
| 1106 | TM->supportsDefaultOutlining()) |
| 1107 | addPass(P: createMachineOutlinerPass(RunOutlinerMode: EnableMachineOutliner)); |
| 1108 | } |
| 1109 | |
| 1110 | if (Opts.enable_gc_empty_basic_blocks) |
| 1111 | addPass(P: llvm::createGCEmptyBasicBlocksLegacyPass()); |
| 1112 | |
| 1113 | if (EnableFSDiscriminator) |
| 1114 | addPass(P: createMIRAddFSDiscriminatorsPass( |
| 1115 | P: sampleprof::FSDiscriminatorPass::PassLast)); |
| 1116 | |
| 1117 | const bool SplitFunctionsWithPGO = |
| 1118 | TM->Options.FunctionSplitting == FunctionSplittingMode::All; |
| 1119 | |
| 1120 | if (SplitFunctionsWithPGO || Opts.split_static_data || |
| 1121 | TM->Options.EnableStaticDataPartitioning) { |
| 1122 | const std::string ProfileFile = getFSProfileFile(Opts, TM); |
| 1123 | if (!ProfileFile.empty()) { |
| 1124 | if (EnableFSDiscriminator) { |
| 1125 | addPass(P: createMIRProfileLoaderPass( |
| 1126 | File: ProfileFile, RemappingFile: getFSRemappingFile(Opts, TM), |
| 1127 | P: sampleprof::FSDiscriminatorPass::PassLast, FS: nullptr)); |
| 1128 | } else { |
| 1129 | // Sample profile is given, but FSDiscriminator is not |
| 1130 | // enabled, this may result in performance regression. |
| 1131 | WithColor::warning() |
| 1132 | << "Using AutoFDO without FSDiscriminator for MFS may regress " |
| 1133 | "performance.\n" ; |
| 1134 | } |
| 1135 | } |
| 1136 | } |
| 1137 | |
| 1138 | if (Opts.split_static_data || TM->Options.EnableStaticDataPartitioning) { |
| 1139 | // The static data splitter pass is a machine function pass. and |
| 1140 | // static data annotator pass is a module-wide pass. See the file comment |
| 1141 | // in StaticDataAnnotator.cpp for the motivation. |
| 1142 | addPass(P: createStaticDataSplitterLegacyPass()); |
| 1143 | addPass(P: createStaticDataAnnotatorLegacyPass()); |
| 1144 | } |
| 1145 | // We run the BasicBlockSections pass if we need BB sections, BB address map, |
| 1146 | // or function splitting. With function splitting, functions which are |
| 1147 | // eligible for basic-block-sections optimizations (`=all`, or `=list=` with |
| 1148 | // function included in the list profile) get that optimization, and the other |
| 1149 | // functions are split using their PGO/AutoFDO profile. |
| 1150 | if (TM->getBBSectionsType() != llvm::BasicBlockSection::None || |
| 1151 | TM->Options.BBAddrMap || SplitFunctionsWithPGO) { |
| 1152 | if (shouldEmitBBHash() || Opts.basic_block_section_match_infer) |
| 1153 | addPass(P: llvm::createMachineBlockHashInfoPass()); |
| 1154 | if (TM->getBBSectionsType() == llvm::BasicBlockSection::List) { |
| 1155 | addPass(P: llvm::createBasicBlockSectionsProfileReaderWrapperPass( |
| 1156 | Buf: TM->getBBSectionsFuncListBuf())); |
| 1157 | if (!Opts.basic_block_section_match_infer || Opts.propeller_match_bbid) |
| 1158 | addPass(P: llvm::createBasicBlockPathCloningPass()); |
| 1159 | if (Opts.basic_block_section_match_infer) |
| 1160 | addPass(P: llvm::createBasicBlockMatchingAndInferencePass()); |
| 1161 | else |
| 1162 | addPass(P: llvm::createInsertCodePrefetchPass()); |
| 1163 | } |
| 1164 | addPass(P: llvm::createBasicBlockSectionsPass(SplitWithPGO: SplitFunctionsWithPGO)); |
| 1165 | } |
| 1166 | |
| 1167 | addPostBBSections(); |
| 1168 | |
| 1169 | if (!Opts.disable_cfi_fixup && TM->Options.EnableCFIFixup) |
| 1170 | addPass(P: createCFIFixupLegacy()); |
| 1171 | |
| 1172 | PM->add(P: createStackFrameLayoutAnalysisPass()); |
| 1173 | |
| 1174 | // Add passes that directly emit MI after all other MI passes. |
| 1175 | addPreEmitPass2(); |
| 1176 | |
| 1177 | AddingMachinePasses = false; |
| 1178 | } |
| 1179 | |
| 1180 | /// Add passes that optimize machine instructions in SSA form. |
| 1181 | void TargetPassConfig::addMachineSSAOptimization() { |
| 1182 | // Pre-ra tail duplication. |
| 1183 | addPass(PassID: &EarlyTailDuplicateLegacyID); |
| 1184 | |
| 1185 | // Optimize PHIs before DCE: removing dead PHI cycles may make more |
| 1186 | // instructions dead. |
| 1187 | addPass(PassID: &OptimizePHIsLegacyID); |
| 1188 | |
| 1189 | // This pass merges large allocas. StackSlotColoring is a different pass |
| 1190 | // which merges spill slots. |
| 1191 | addPass(PassID: &StackColoringLegacyID); |
| 1192 | |
| 1193 | // If the target requests it, assign local variables to stack slots relative |
| 1194 | // to one another and simplify frame index references where possible. |
| 1195 | addPass(PassID: &LocalStackSlotAllocationID); |
| 1196 | |
| 1197 | // With optimization, dead code should already be eliminated. However |
| 1198 | // there is one known exception: lowered code for arguments that are only |
| 1199 | // used by tail calls, where the tail calls reuse the incoming stack |
| 1200 | // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll). |
| 1201 | addPass(PassID: &DeadMachineInstructionElimID); |
| 1202 | |
| 1203 | // Allow targets to insert passes that improve instruction level parallelism, |
| 1204 | // like if-conversion. Such passes will typically need dominator trees and |
| 1205 | // loop info, just like LICM and CSE below. |
| 1206 | addILPOpts(); |
| 1207 | |
| 1208 | addPass(PassID: &EarlyMachineLICMID); |
| 1209 | addPass(PassID: &MachineCSELegacyID); |
| 1210 | |
| 1211 | addPass(PassID: &MachineSinkingLegacyID); |
| 1212 | |
| 1213 | addPass(PassID: &PeepholeOptimizerLegacyID); |
| 1214 | // Clean-up the dead code that may have been generated by peephole |
| 1215 | // rewriting. |
| 1216 | addPass(PassID: &DeadMachineInstructionElimID); |
| 1217 | } |
| 1218 | |
| 1219 | //===---------------------------------------------------------------------===// |
| 1220 | /// Register Allocation Pass Configuration |
| 1221 | //===---------------------------------------------------------------------===// |
| 1222 | |
| 1223 | /// A dummy default pass factory indicates whether the register allocator is |
| 1224 | /// overridden on the command line. |
| 1225 | static llvm::once_flag InitializeDefaultRegisterAllocatorFlag; |
| 1226 | |
| 1227 | static RegisterRegAlloc |
| 1228 | defaultRegAlloc("default" , |
| 1229 | "pick register allocator based on -O option" , |
| 1230 | useDefaultRegisterAllocator); |
| 1231 | |
| 1232 | static void initializeDefaultRegisterAllocatorOnce() { |
| 1233 | if (!RegisterRegAlloc::getDefault()) |
| 1234 | RegisterRegAlloc::setDefault(RegAlloc); |
| 1235 | } |
| 1236 | |
| 1237 | bool TargetPassConfig::getOptimizeRegAlloc() const { |
| 1238 | // An explicit -regalloc choice implies its pipeline: only the fast |
| 1239 | // allocator uses the unoptimized one. |
| 1240 | llvm::call_once(flag&: InitializeDefaultRegisterAllocatorFlag, |
| 1241 | F&: initializeDefaultRegisterAllocatorOnce); |
| 1242 | RegisterRegAlloc::FunctionPassCtor Ctor = RegisterRegAlloc::getDefault(); |
| 1243 | if (Ctor != (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator) |
| 1244 | return Ctor != |
| 1245 | (RegisterRegAlloc::FunctionPassCtor)&createFastRegisterAllocator; |
| 1246 | return getOptLevel() != CodeGenOptLevel::None; |
| 1247 | } |
| 1248 | |
| 1249 | /// Instantiate the default register allocator pass for this target for either |
| 1250 | /// the optimized or unoptimized allocation path. This will be added to the pass |
| 1251 | /// manager by addFastRegAlloc in the unoptimized case or addOptimizedRegAlloc |
| 1252 | /// in the optimized case. |
| 1253 | /// |
| 1254 | /// A target that uses the standard regalloc pass order for fast or optimized |
| 1255 | /// allocation may still override this for per-target regalloc |
| 1256 | /// selection. But -regalloc=... always takes precedence. |
| 1257 | FunctionPass *TargetPassConfig::createTargetRegisterAllocator(bool Optimized) { |
| 1258 | if (Optimized) |
| 1259 | return createGreedyRegisterAllocator(); |
| 1260 | else |
| 1261 | return createFastRegisterAllocator(); |
| 1262 | } |
| 1263 | |
| 1264 | /// Find and instantiate the register allocation pass requested by this target |
| 1265 | /// at the current optimization level. Different register allocators are |
| 1266 | /// defined as separate passes because they may require different analysis. |
| 1267 | /// |
| 1268 | /// This helper ensures that the regalloc= option is always available, |
| 1269 | /// even for targets that override the default allocator. |
| 1270 | /// |
| 1271 | /// FIXME: When MachinePassRegistry register pass IDs instead of function ptrs, |
| 1272 | /// this can be folded into addPass. |
| 1273 | FunctionPass *TargetPassConfig::createRegAllocPass(bool Optimized) { |
| 1274 | // getOptimizeRegAlloc, called before the pipeline branches, has initialized |
| 1275 | // the global default. |
| 1276 | RegisterRegAlloc::FunctionPassCtor Ctor = RegisterRegAlloc::getDefault(); |
| 1277 | if (Ctor != useDefaultRegisterAllocator) |
| 1278 | return Ctor(); |
| 1279 | |
| 1280 | // With no -regalloc= override, ask the target for a regalloc pass. |
| 1281 | return createTargetRegisterAllocator(Optimized); |
| 1282 | } |
| 1283 | |
| 1284 | bool TargetPassConfig::isCustomizedRegAlloc() { |
| 1285 | return RegAlloc != |
| 1286 | (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator; |
| 1287 | } |
| 1288 | |
| 1289 | bool TargetPassConfig::addRegAssignAndRewriteFast() { |
| 1290 | if (RegAlloc != (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator && |
| 1291 | RegAlloc != (RegisterRegAlloc::FunctionPassCtor)&createFastRegisterAllocator) |
| 1292 | reportFatalUsageError( |
| 1293 | reason: "Must use fast (default) register allocator for unoptimized regalloc." ); |
| 1294 | |
| 1295 | addPass(P: createRegAllocPass(Optimized: false)); |
| 1296 | |
| 1297 | return true; |
| 1298 | } |
| 1299 | |
| 1300 | bool TargetPassConfig::addRegAssignAndRewriteOptimized() { |
| 1301 | // Add the selected register allocation pass. |
| 1302 | addPass(P: createRegAllocPass(Optimized: true)); |
| 1303 | |
| 1304 | // Allow targets to change the register assignments before rewriting. |
| 1305 | addPreRewrite(); |
| 1306 | |
| 1307 | // Finally rewrite virtual registers. |
| 1308 | addPass(PassID: &VirtRegRewriterID); |
| 1309 | |
| 1310 | // Regalloc scoring for ML-driven eviction - noop except when learning a new |
| 1311 | // eviction policy. |
| 1312 | addPass(P: createRegAllocScoringPass()); |
| 1313 | return true; |
| 1314 | } |
| 1315 | |
| 1316 | /// Return true if the default global register allocator is in use and |
| 1317 | /// has not be overriden on the command line with '-regalloc=...' |
| 1318 | bool TargetPassConfig::usingDefaultRegAlloc() const { |
| 1319 | return RegAlloc.getNumOccurrences() == 0; |
| 1320 | } |
| 1321 | |
| 1322 | /// Add the minimum set of target-independent passes that are required for |
| 1323 | /// register allocation. No coalescing or scheduling. |
| 1324 | void TargetPassConfig::addFastRegAlloc() { |
| 1325 | addPass(PassID: &PHIEliminationID); |
| 1326 | if (!TM->enableTiedFastRegAlloc()) |
| 1327 | addPass(PassID: &TwoAddressInstructionPassID); |
| 1328 | |
| 1329 | addRegAssignAndRewriteFast(); |
| 1330 | } |
| 1331 | |
| 1332 | /// Add standard target-independent passes that are tightly coupled with |
| 1333 | /// optimized register allocation, including coalescing, machine instruction |
| 1334 | /// scheduling, and register allocation itself. |
| 1335 | void TargetPassConfig::addOptimizedRegAlloc() { |
| 1336 | addPass(PassID: &DetectDeadLanesID); |
| 1337 | |
| 1338 | addPass(PassID: &InitUndefID); |
| 1339 | |
| 1340 | addPass(PassID: &ProcessImplicitDefsID); |
| 1341 | |
| 1342 | // LiveIntervals requires pure SSA form and no unreachable blocks. |
| 1343 | // Explicit addition of UnreachableMachineBlockElim allows stopping before or |
| 1344 | // after it with -stop-before/-stop-after. |
| 1345 | addPass(PassID: &UnreachableMachineBlockElimID); |
| 1346 | |
| 1347 | // Run SSA machine scheduler runs just before PHI elimination. |
| 1348 | if (EnableSSAMachineScheduler) { |
| 1349 | addPass(PassID: &LiveIntervalsID); |
| 1350 | addPass(PassID: &SSAMachineSchedulerID); |
| 1351 | } |
| 1352 | |
| 1353 | addPass(PassID: &LiveIntervalsID); |
| 1354 | |
| 1355 | // Edge splitting is smarter with machine loop info. |
| 1356 | addPass(PassID: &MachineLoopInfoID); |
| 1357 | addPass(PassID: &PHIEliminationID); |
| 1358 | |
| 1359 | addPass(PassID: &TwoAddressInstructionPassID); |
| 1360 | addPass(PassID: &RegisterCoalescerID); |
| 1361 | |
| 1362 | // The machine scheduler may accidentally create disconnected components |
| 1363 | // when moving subregister definitions around, avoid this by splitting them to |
| 1364 | // separate vregs before. Splitting can also improve reg. allocation quality. |
| 1365 | addPass(PassID: &RenameIndependentSubregsID); |
| 1366 | |
| 1367 | // PreRA instruction scheduling. |
| 1368 | addPass(PassID: &MachineSchedulerID); |
| 1369 | |
| 1370 | if (addRegAssignAndRewriteOptimized()) { |
| 1371 | // Perform stack slot coloring and post-ra machine LICM. |
| 1372 | addPass(PassID: &StackSlotColoringID); |
| 1373 | |
| 1374 | // Allow targets to expand pseudo instructions depending on the choice of |
| 1375 | // registers before MachineCopyPropagation. |
| 1376 | addPostRewrite(); |
| 1377 | |
| 1378 | // Copy propagate to forward register uses and try to eliminate COPYs that |
| 1379 | // were not coalesced. |
| 1380 | addPass(PassID: &MachineCopyPropagationID); |
| 1381 | |
| 1382 | // Run post-ra machine LICM to hoist reloads / remats. |
| 1383 | // |
| 1384 | // FIXME: can this move into MachineLateOptimization? |
| 1385 | addPass(PassID: &MachineLICMID); |
| 1386 | } |
| 1387 | } |
| 1388 | |
| 1389 | //===---------------------------------------------------------------------===// |
| 1390 | /// Post RegAlloc Pass Configuration |
| 1391 | //===---------------------------------------------------------------------===// |
| 1392 | |
| 1393 | /// Add passes that optimize machine instructions after register allocation. |
| 1394 | void TargetPassConfig::addMachineLateOptimization() { |
| 1395 | // Cleanup of redundant immediate/address loads. |
| 1396 | addPass(PassID: &MachineLateInstrsCleanupID); |
| 1397 | |
| 1398 | // Branch folding must be run after regalloc and prolog/epilog insertion. |
| 1399 | addPass(PassID: &BranchFolderPassID); |
| 1400 | |
| 1401 | // Tail duplication. |
| 1402 | // Note that duplicating tail just increases code size and degrades |
| 1403 | // performance for targets that require Structured Control Flow. |
| 1404 | // In addition it can also make CFG irreducible. Thus we disable it. |
| 1405 | if (!TM->requiresStructuredCFG()) |
| 1406 | addPass(PassID: &TailDuplicateLegacyID); |
| 1407 | |
| 1408 | // Copy propagation. |
| 1409 | addPass(PassID: &MachineCopyPropagationID); |
| 1410 | } |
| 1411 | |
| 1412 | /// Add standard GC passes. |
| 1413 | bool TargetPassConfig::addGCPasses() { |
| 1414 | addPass(PassID: &GCMachineCodeAnalysisID); |
| 1415 | return true; |
| 1416 | } |
| 1417 | |
| 1418 | /// Add standard basic block placement passes. |
| 1419 | void TargetPassConfig::addBlockPlacement() { |
| 1420 | if (EnableFSDiscriminator) { |
| 1421 | addPass(P: createMIRAddFSDiscriminatorsPass( |
| 1422 | P: sampleprof::FSDiscriminatorPass::Pass2)); |
| 1423 | const std::string ProfileFile = getFSProfileFile(Opts, TM); |
| 1424 | if (!ProfileFile.empty() && !Opts.disable_layout_fsprofile_loader) |
| 1425 | addPass(P: createMIRProfileLoaderPass( |
| 1426 | File: ProfileFile, RemappingFile: getFSRemappingFile(Opts, TM), |
| 1427 | P: sampleprof::FSDiscriminatorPass::Pass2, FS: nullptr)); |
| 1428 | } |
| 1429 | if (addPass(PassID: &MachineBlockPlacementID)) { |
| 1430 | // Run a separate pass to collect block placement statistics. |
| 1431 | if (Opts.enable_block_placement_stats) |
| 1432 | addPass(PassID: &MachineBlockPlacementStatsID); |
| 1433 | } |
| 1434 | } |
| 1435 | |
| 1436 | //===---------------------------------------------------------------------===// |
| 1437 | /// GlobalISel Configuration |
| 1438 | //===---------------------------------------------------------------------===// |
| 1439 | bool TargetPassConfig::isGlobalISelAbortEnabled() const { |
| 1440 | return TM->Options.GlobalISelAbort == GlobalISelAbortMode::Enable; |
| 1441 | } |
| 1442 | |
| 1443 | bool TargetPassConfig::reportDiagnosticWhenGlobalISelFallback() const { |
| 1444 | return TM->Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag; |
| 1445 | } |
| 1446 | |
| 1447 | std::unique_ptr<CSEConfigBase> TargetPassConfig::getCSEConfig() const { |
| 1448 | return std::make_unique<CSEConfigBase>(); |
| 1449 | } |
| 1450 | |