| 1 | //===-- GCNSchedStrategy.h - GCN Scheduler Strategy -*- C++ -*-------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | /// \file |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #ifndef LLVM_LIB_TARGET_AMDGPU_GCNSCHEDSTRATEGY_H |
| 14 | #define LLVM_LIB_TARGET_AMDGPU_GCNSCHEDSTRATEGY_H |
| 15 | |
| 16 | #include "GCNRegPressure.h" |
| 17 | #include "GCNRematStrategy.h" |
| 18 | #include "llvm/ADT/DenseMap.h" |
| 19 | #include "llvm/CodeGen/MachineBasicBlock.h" |
| 20 | #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" |
| 21 | #include "llvm/CodeGen/MachineInstr.h" |
| 22 | #include "llvm/CodeGen/MachineScheduler.h" |
| 23 | #include "llvm/Support/CommandLine.h" |
| 24 | |
| 25 | namespace llvm { |
| 26 | |
| 27 | struct VGPRThresholdParser : public cl::parser<unsigned> { |
| 28 | VGPRThresholdParser(cl::Option &O) : cl::parser<unsigned>(O) {} |
| 29 | bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value); |
| 30 | }; |
| 31 | |
| 32 | extern cl::opt<unsigned, false, VGPRThresholdParser> VGPRThresholdPercentOpt; |
| 33 | |
| 34 | class SIMachineFunctionInfo; |
| 35 | class SIRegisterInfo; |
| 36 | class GCNSubtarget; |
| 37 | class GCNSchedStage; |
| 38 | |
| 39 | enum class GCNSchedStageID : unsigned { |
| 40 | OccInitialSchedule = 0, |
| 41 | RewriteMFMAForm = 1, |
| 42 | UnclusteredHighRPReschedule = 2, |
| 43 | ClusteredLowOccupancyReschedule = 3, |
| 44 | PreRARematerialize = 4, |
| 45 | ILPInitialSchedule = 5, |
| 46 | MemoryClauseInitialSchedule = 6, |
| 47 | LiveIntervalRPReschedule = 7 |
| 48 | }; |
| 49 | |
| 50 | #ifndef NDEBUG |
| 51 | raw_ostream &operator<<(raw_ostream &OS, const GCNSchedStageID &StageID); |
| 52 | #endif |
| 53 | |
| 54 | /// This is a minimal scheduler strategy. The main difference between this |
| 55 | /// and the GenericScheduler is that GCNSchedStrategy uses different |
| 56 | /// heuristics to determine excess/critical pressure sets. |
| 57 | class GCNSchedStrategy : public GenericScheduler { |
| 58 | protected: |
| 59 | SUnit *pickNodeBidirectional(bool &IsTopNode, bool &PickedPending); |
| 60 | |
| 61 | void pickNodeFromQueue(SchedBoundary &Zone, const CandPolicy &ZonePolicy, |
| 62 | const RegPressureTracker &RPTracker, |
| 63 | SchedCandidate &Cand, bool &IsPending, |
| 64 | bool IsBottomUp); |
| 65 | |
| 66 | void initCandidate(SchedCandidate &Cand, SUnit *SU, bool AtTop, |
| 67 | const RegPressureTracker &RPTracker, |
| 68 | const SIRegisterInfo *SRI, unsigned SGPRPressure, |
| 69 | unsigned VGPRPressure, unsigned AGPRPressure, |
| 70 | bool IsBottomUp); |
| 71 | |
| 72 | /// Evaluates instructions in the pending queue using a subset of scheduling |
| 73 | /// heuristics. |
| 74 | /// |
| 75 | /// Instructions that cannot be issued due to hardware constraints are placed |
| 76 | /// in the pending queue rather than the available queue, making them normally |
| 77 | /// invisible to scheduling heuristics. However, in certain scenarios (such as |
| 78 | /// avoiding register spilling), it may be beneficial to consider scheduling |
| 79 | /// these not-yet-ready instructions. |
| 80 | bool tryPendingCandidate(SchedCandidate &Cand, SchedCandidate &TryCand, |
| 81 | SchedBoundary *Zone) const; |
| 82 | |
| 83 | void printCandidateDecision(const SchedCandidate &Current, |
| 84 | const SchedCandidate &Preferred); |
| 85 | |
| 86 | void getRegisterPressures(bool AtTop, const RegPressureTracker &RPTracker, |
| 87 | SUnit *SU, std::vector<unsigned> &Pressure, |
| 88 | std::vector<unsigned> &MaxPressure, |
| 89 | const GCNDownwardRPTrackerBase &DownwardTracker, |
| 90 | const GCNUpwardRPTracker &UpwardTracker, |
| 91 | ScheduleDAGMI *DAG, const SIRegisterInfo *SRI); |
| 92 | |
| 93 | /// \returns the ArchVGPR and AGPR pressure of \p RP after distributing |
| 94 | /// bank-agnostic AVGPR pressure across the two banks. |
| 95 | std::pair<unsigned, unsigned> |
| 96 | getBankedPressure(const GCNRegPressure &RP) const; |
| 97 | |
| 98 | const AMDGPUOptions &CLOpts; |
| 99 | |
| 100 | std::vector<unsigned> Pressure; |
| 101 | |
| 102 | std::vector<unsigned> MaxPressure; |
| 103 | |
| 104 | /// Capacity of the ArchVGPR bank, captured before VGPRExcessLimit is cut |
| 105 | /// down by ErrorMargin, VGPRLimitBias, or VGPRThresholdPercent. |
| 106 | unsigned ArchVGPRBankLimit = 0; |
| 107 | |
| 108 | /// Capacity of the AGPR bank, captured before AGPRExcessLimit is cut down by |
| 109 | /// ErrorMargin and VGPRLimitBias. Zero when no AGPRs are allocatable. |
| 110 | unsigned AGPRBankLimit = 0; |
| 111 | |
| 112 | /// Budget for the combined ArchVGPR + AGPR file, or zero on targets without |
| 113 | /// one. |
| 114 | unsigned UnifiedVGPRExcessLimit = 0; |
| 115 | |
| 116 | /// Combined-file pressure of the candidate most recently passed to |
| 117 | /// getRegisterPressures(). Only maintained when the GCN trackers are in use. |
| 118 | unsigned CandUnifiedVGPRPressure = 0; |
| 119 | |
| 120 | unsigned TargetOccupancy; |
| 121 | |
| 122 | MachineFunction *MF; |
| 123 | |
| 124 | // Scheduling stages for this strategy. |
| 125 | SmallVector<GCNSchedStageID, 4> SchedStages; |
| 126 | |
| 127 | // Pointer to the current SchedStageID. |
| 128 | SmallVectorImpl<GCNSchedStageID>::iterator CurrentStage = nullptr; |
| 129 | |
| 130 | // GCN RP Tracker for top-down scheduling |
| 131 | mutable GCNManualDownwardRPTracker DownwardTracker; |
| 132 | |
| 133 | // GCN RP Tracker for botttom-up scheduling |
| 134 | mutable GCNUpwardRPTracker UpwardTracker; |
| 135 | |
| 136 | bool UseGCNTrackers = false; |
| 137 | |
| 138 | std::optional<bool> GCNTrackersOverride; |
| 139 | |
| 140 | public: |
| 141 | // schedule() have seen register pressure over the critical limits and had to |
| 142 | // track register pressure for actual scheduling heuristics. |
| 143 | bool HasHighPressure; |
| 144 | |
| 145 | // Schedule known to have excess register pressure. Be more conservative in |
| 146 | // increasing ILP and preserving VGPRs. |
| 147 | bool KnownExcessRP = false; |
| 148 | |
| 149 | // An error margin is necessary because of poor performance of the generic RP |
| 150 | // tracker and can be adjusted up for tuning heuristics to try and more |
| 151 | // aggressively reduce register pressure. |
| 152 | unsigned ErrorMargin = 3; |
| 153 | |
| 154 | // Bias for SGPR limits under a high register pressure. |
| 155 | const unsigned HighRPSGPRBias = 7; |
| 156 | |
| 157 | // Bias for VGPR limits under a high register pressure. |
| 158 | const unsigned HighRPVGPRBias = 7; |
| 159 | |
| 160 | unsigned SGPRExcessLimit = 0; |
| 161 | |
| 162 | unsigned VGPRExcessLimit = 0; |
| 163 | |
| 164 | unsigned AGPRExcessLimit = 0; |
| 165 | |
| 166 | /// \returns the allocatable ArchVGPR and AGPR counts. |
| 167 | std::pair<unsigned, unsigned> getBankLimits() const { |
| 168 | return {ArchVGPRBankLimit, AGPRBankLimit}; |
| 169 | } |
| 170 | |
| 171 | unsigned SGPRCriticalLimit = 0; |
| 172 | |
| 173 | unsigned VGPRCriticalLimit = 0; |
| 174 | |
| 175 | // Occupancy budget for the combined VGPR/AGPR allocation on unified targets. |
| 176 | // Unlike the per-bank limit, this is not clamped to allocatable VGPRs. |
| 177 | unsigned UnifiedVGPRCriticalLimit = 0; |
| 178 | |
| 179 | unsigned AGPRCriticalLimit = 0; |
| 180 | |
| 181 | unsigned SGPRLimitBias = 0; |
| 182 | |
| 183 | unsigned VGPRLimitBias = 0; |
| 184 | |
| 185 | unsigned VGPRThresholdPercent = 0; |
| 186 | |
| 187 | GCNSchedStrategy(const MachineSchedContext *C); |
| 188 | |
| 189 | SUnit *pickNode(bool &IsTopNode) override; |
| 190 | |
| 191 | void schedNode(SUnit *SU, bool IsTopNode) override; |
| 192 | |
| 193 | void initialize(ScheduleDAGMI *DAG) override; |
| 194 | |
| 195 | unsigned getTargetOccupancy() { return TargetOccupancy; } |
| 196 | |
| 197 | void setTargetOccupancy(unsigned Occ) { TargetOccupancy = Occ; } |
| 198 | |
| 199 | GCNSchedStageID getCurrentStage(); |
| 200 | |
| 201 | // Advances stage. Returns true if there are remaining stages. |
| 202 | bool advanceStage(); |
| 203 | |
| 204 | bool hasNextStage() const; |
| 205 | |
| 206 | bool useGCNTrackers() const { |
| 207 | return GCNTrackersOverride.value_or(u: UseGCNTrackers); |
| 208 | } |
| 209 | |
| 210 | GCNDownwardRPTrackerBase &getDownwardTracker() { return DownwardTracker; } |
| 211 | |
| 212 | GCNUpwardRPTracker &getUpwardTracker() { return UpwardTracker; } |
| 213 | }; |
| 214 | |
| 215 | /// The goal of this scheduling strategy is to maximize kernel occupancy (i.e. |
| 216 | /// maximum number of waves per simd). |
| 217 | class GCNMaxOccupancySchedStrategy final : public GCNSchedStrategy { |
| 218 | public: |
| 219 | GCNMaxOccupancySchedStrategy(const MachineSchedContext *C, |
| 220 | bool IsLegacyScheduler = false); |
| 221 | }; |
| 222 | |
| 223 | /// The goal of this scheduling strategy is to maximize ILP for a single wave |
| 224 | /// (i.e. latency hiding). |
| 225 | class GCNMaxILPSchedStrategy final : public GCNSchedStrategy { |
| 226 | protected: |
| 227 | bool tryCandidate(SchedCandidate &Cand, SchedCandidate &TryCand, |
| 228 | SchedBoundary *Zone) const override; |
| 229 | |
| 230 | public: |
| 231 | GCNMaxILPSchedStrategy(const MachineSchedContext *C); |
| 232 | }; |
| 233 | |
| 234 | /// The goal of this scheduling strategy is to maximize memory clause for a |
| 235 | /// single wave. |
| 236 | class GCNMaxMemoryClauseSchedStrategy final : public GCNSchedStrategy { |
| 237 | protected: |
| 238 | bool tryCandidate(SchedCandidate &Cand, SchedCandidate &TryCand, |
| 239 | SchedBoundary *Zone) const override; |
| 240 | |
| 241 | public: |
| 242 | GCNMaxMemoryClauseSchedStrategy(const MachineSchedContext *C); |
| 243 | }; |
| 244 | |
| 245 | class ScheduleMetrics { |
| 246 | unsigned ScheduleLength; |
| 247 | unsigned BubbleCycles; |
| 248 | |
| 249 | public: |
| 250 | ScheduleMetrics() = default; |
| 251 | ScheduleMetrics(unsigned L, unsigned BC) |
| 252 | : ScheduleLength(L), BubbleCycles(BC) {} |
| 253 | unsigned getLength() const { return ScheduleLength; } |
| 254 | unsigned getBubbles() const { return BubbleCycles; } |
| 255 | unsigned getMetric() const { |
| 256 | unsigned Metric = (BubbleCycles * ScaleFactor) / ScheduleLength; |
| 257 | // Metric is zero if the amount of bubbles is less than 1% which is too |
| 258 | // small. So, return 1. |
| 259 | return Metric ? Metric : 1; |
| 260 | } |
| 261 | static const unsigned ScaleFactor; |
| 262 | }; |
| 263 | |
| 264 | inline raw_ostream &operator<<(raw_ostream &OS, const ScheduleMetrics &Sm) { |
| 265 | dbgs() << "\n Schedule Metric (scaled by " << ScheduleMetrics::ScaleFactor |
| 266 | << " ) is: " << Sm.getMetric() << " [ " << Sm.getBubbles() << "/" |
| 267 | << Sm.getLength() << " ]\n" ; |
| 268 | return OS; |
| 269 | } |
| 270 | |
| 271 | class GCNScheduleDAGMILive; |
| 272 | class RegionPressureMap { |
| 273 | GCNScheduleDAGMILive *DAG; |
| 274 | // The live in/out pressure as indexed by the first or last MI in the region |
| 275 | // before scheduling. |
| 276 | DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> RegionLiveRegMap; |
| 277 | // The mapping of RegionIDx to key instruction |
| 278 | DenseMap<unsigned, MachineInstr *> IdxToInstruction; |
| 279 | // Whether we are calculating LiveOuts or LiveIns |
| 280 | bool IsLiveOut; |
| 281 | |
| 282 | public: |
| 283 | RegionPressureMap() = default; |
| 284 | RegionPressureMap(GCNScheduleDAGMILive *GCNDAG, bool LiveOut) |
| 285 | : DAG(GCNDAG), IsLiveOut(LiveOut) {} |
| 286 | // Build the Instr->LiveReg and RegionIdx->Instr maps |
| 287 | void buildLiveRegMap(); |
| 288 | |
| 289 | // Retrieve the LiveReg for a given RegionIdx |
| 290 | GCNRPTracker::LiveRegSet &getLiveRegsForRegionIdx(unsigned RegionIdx) { |
| 291 | assert(IdxToInstruction.contains(RegionIdx)); |
| 292 | MachineInstr *Key = IdxToInstruction[RegionIdx]; |
| 293 | return RegionLiveRegMap[Key]; |
| 294 | } |
| 295 | }; |
| 296 | |
| 297 | /// A region's boundaries i.e. a pair of instruction bundle iterators. The lower |
| 298 | /// boundary is inclusive, the upper boundary is exclusive. |
| 299 | using RegionBoundaries = |
| 300 | std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>; |
| 301 | |
| 302 | class GCNScheduleDAGMILive final : public ScheduleDAGMILive { |
| 303 | friend class GCNSchedStage; |
| 304 | friend class OccInitialScheduleStage; |
| 305 | friend class RewriteMFMAFormStage; |
| 306 | friend class UnclusteredHighRPStage; |
| 307 | friend class ClusteredLowOccStage; |
| 308 | friend class PreRARematStage; |
| 309 | friend class ILPInitialScheduleStage; |
| 310 | friend class LiveIntervalRPStage; |
| 311 | friend class RegionPressureMap; |
| 312 | |
| 313 | const GCNSubtarget &ST; |
| 314 | |
| 315 | SIMachineFunctionInfo &MFI; |
| 316 | |
| 317 | // Occupancy target at the beginning of function scheduling cycle. |
| 318 | unsigned StartingOccupancy; |
| 319 | |
| 320 | // Minimal real occupancy recorder for the function. |
| 321 | unsigned MinOccupancy; |
| 322 | |
| 323 | // Vector of regions recorder for later rescheduling |
| 324 | SmallVector<RegionBoundaries, 32> Regions; |
| 325 | |
| 326 | // Record regions with high register pressure. |
| 327 | BitVector RegionsWithHighRP; |
| 328 | |
| 329 | // Record regions with excess register pressure over the physical register |
| 330 | // limit. Register pressure in these regions usually will result in spilling. |
| 331 | BitVector RegionsWithExcessRP; |
| 332 | |
| 333 | // Regions that have IGLP instructions (SCHED_GROUP_BARRIER or IGLP_OPT). |
| 334 | BitVector RegionsWithIGLPInstrs; |
| 335 | |
| 336 | // Region live-in cache. |
| 337 | SmallVector<GCNRPTracker::LiveRegSet, 32> LiveIns; |
| 338 | |
| 339 | // Region pressure cache. |
| 340 | SmallVector<GCNRegPressure, 32> Pressure; |
| 341 | |
| 342 | // Temporary basic block live-in cache. |
| 343 | DenseMap<const MachineBasicBlock *, GCNRPTracker::LiveRegSet> MBBLiveIns; |
| 344 | |
| 345 | // The map of the initial first region instruction to region live in registers |
| 346 | DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> BBLiveInMap; |
| 347 | |
| 348 | // Calculate the map of the initial first region instruction to region live in |
| 349 | // registers |
| 350 | DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> getRegionLiveInMap() const; |
| 351 | |
| 352 | // Calculate the map of the initial last region instruction to region live out |
| 353 | // registers |
| 354 | DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> |
| 355 | getRegionLiveOutMap() const; |
| 356 | |
| 357 | // The live out registers per region. These are internally stored as a map of |
| 358 | // the initial last region instruction to region live out registers, but can |
| 359 | // be retreived with the regionIdx by calls to getLiveRegsForRegionIdx. |
| 360 | RegionPressureMap RegionLiveOuts; |
| 361 | |
| 362 | // Return current region pressure. |
| 363 | /// \returns the maximum pressure of region \p RegionIdx. If set, \p |
| 364 | /// PressureCallback is called with the pressure of every program point of |
| 365 | /// the region. |
| 366 | GCNRegPressure getRealRegPressure(unsigned RegionIdx, |
| 367 | std::function<void(const GCNRegPressure &)> |
| 368 | PressureCallback = nullptr) const; |
| 369 | |
| 370 | // Compute and cache live-ins and pressure for all regions in block. |
| 371 | void computeBlockPressure(unsigned RegionIdx, const MachineBasicBlock *MBB); |
| 372 | |
| 373 | /// Makes the scheduler try to achieve an occupancy of \p TargetOccupancy. |
| 374 | void setTargetOccupancy(unsigned TargetOccupancy); |
| 375 | |
| 376 | void runSchedStages(); |
| 377 | |
| 378 | std::unique_ptr<GCNSchedStage> createSchedStage(GCNSchedStageID SchedStageID); |
| 379 | |
| 380 | public: |
| 381 | GCNScheduleDAGMILive(MachineSchedContext *C, |
| 382 | std::unique_ptr<MachineSchedStrategy> S); |
| 383 | |
| 384 | void schedule() override; |
| 385 | |
| 386 | void finalizeSchedule() override; |
| 387 | }; |
| 388 | |
| 389 | // GCNSchedStrategy applies multiple scheduling stages to a function. |
| 390 | class GCNSchedStage { |
| 391 | protected: |
| 392 | GCNScheduleDAGMILive &DAG; |
| 393 | |
| 394 | GCNSchedStrategy &S; |
| 395 | |
| 396 | MachineFunction &MF; |
| 397 | |
| 398 | SIMachineFunctionInfo &MFI; |
| 399 | |
| 400 | const GCNSubtarget &ST; |
| 401 | |
| 402 | const GCNSchedStageID StageID; |
| 403 | |
| 404 | // The current block being scheduled. |
| 405 | MachineBasicBlock *CurrentMBB = nullptr; |
| 406 | |
| 407 | // Current region index. |
| 408 | unsigned RegionIdx = 0; |
| 409 | |
| 410 | // Record the original order of instructions before scheduling. |
| 411 | std::vector<MachineInstr *> Unsched; |
| 412 | |
| 413 | // RP before scheduling the current region. |
| 414 | GCNRegPressure PressureBefore; |
| 415 | |
| 416 | // RP after scheduling the current region. |
| 417 | GCNRegPressure PressureAfter; |
| 418 | |
| 419 | std::vector<std::unique_ptr<ScheduleDAGMutation>> SavedMutations; |
| 420 | |
| 421 | GCNSchedStage(GCNSchedStageID StageID, GCNScheduleDAGMILive &DAG); |
| 422 | |
| 423 | public: |
| 424 | // Initialize state for a scheduling stage. Returns false if the current stage |
| 425 | // should be skipped. |
| 426 | virtual bool initGCNSchedStage(); |
| 427 | |
| 428 | // Finalize state after finishing a scheduling pass on the function. |
| 429 | virtual void finalizeGCNSchedStage(); |
| 430 | |
| 431 | // Setup for scheduling a region. Returns false if the current region should |
| 432 | // be skipped. |
| 433 | virtual bool initGCNRegion(); |
| 434 | |
| 435 | // Finalize state after scheduling a region. |
| 436 | virtual void finalizeGCNRegion(); |
| 437 | |
| 438 | // Track whether a new region is also a new MBB. |
| 439 | void setupNewBlock(); |
| 440 | |
| 441 | // Check result of scheduling. |
| 442 | void checkScheduling(); |
| 443 | |
| 444 | // computes the given schedule virtual execution time in clocks |
| 445 | ScheduleMetrics getScheduleMetrics(const std::vector<SUnit> &InputSchedule); |
| 446 | ScheduleMetrics getScheduleMetrics(const GCNScheduleDAGMILive &DAG); |
| 447 | unsigned computeSUnitReadyCycle(const SUnit &SU, unsigned CurrCycle, |
| 448 | DenseMap<unsigned, unsigned> &ReadyCycles, |
| 449 | const TargetSchedModel &SM); |
| 450 | |
| 451 | // Returns true if scheduling should be reverted. |
| 452 | virtual bool shouldRevertScheduling(unsigned WavesAfter); |
| 453 | |
| 454 | // Returns true if current region has known excess pressure. |
| 455 | bool isRegionWithExcessRP() const { |
| 456 | return DAG.RegionsWithExcessRP[RegionIdx]; |
| 457 | } |
| 458 | |
| 459 | // The region number this stage is currently working on |
| 460 | unsigned getRegionIdx() { return RegionIdx; } |
| 461 | |
| 462 | // Returns true if the new schedule may result in more spilling. |
| 463 | bool mayCauseSpilling(unsigned WavesAfter); |
| 464 | |
| 465 | /// Sets the schedule of region \p RegionIdx to \p MIOrder. The MIs in \p |
| 466 | /// MIOrder must be exactly the same as the ones currently existing inside the |
| 467 | /// region, only in a different order that honors def-use chains. |
| 468 | void modifyRegionSchedule(unsigned RegionIdx, |
| 469 | ArrayRef<MachineInstr *> MIOrder); |
| 470 | |
| 471 | void advanceRegion() { RegionIdx++; } |
| 472 | |
| 473 | virtual ~GCNSchedStage() = default; |
| 474 | }; |
| 475 | |
| 476 | class OccInitialScheduleStage : public GCNSchedStage { |
| 477 | public: |
| 478 | bool shouldRevertScheduling(unsigned WavesAfter) override; |
| 479 | |
| 480 | OccInitialScheduleStage(GCNSchedStageID StageID, GCNScheduleDAGMILive &DAG) |
| 481 | : GCNSchedStage(StageID, DAG) {} |
| 482 | }; |
| 483 | |
| 484 | class RewriteMFMAFormStage : public GCNSchedStage { |
| 485 | private: |
| 486 | // Record regions with excess archvgpr register pressure over the physical |
| 487 | // register limit. Register pressure in these regions usually will result in |
| 488 | // spilling. |
| 489 | BitVector RegionsWithExcessArchVGPR; |
| 490 | |
| 491 | const SIInstrInfo *TII; |
| 492 | const SIRegisterInfo *SRI; |
| 493 | |
| 494 | /// Per-candidate cache of the src2 "needs VGPR" decision, computed once |
| 495 | /// and reused on-demand. |
| 496 | DenseMap<const MachineInstr *, bool> Src2NeedsVGPRCache; |
| 497 | |
| 498 | /// Do a speculative rewrite and collect copy locations. The speculative |
| 499 | /// rewrite allows us to calculate the RP of the code after the rewrite, and |
| 500 | /// the copy locations allow us to calculate the total cost of copies required |
| 501 | /// for the rewrite. Stores the rewritten instructions in \p RewriteCands , |
| 502 | /// the copy locations for uses (of the MFMA result) in \p CopyForUse and the |
| 503 | /// copy locations for defs (of the MFMA operands) in \p CopyForDef |
| 504 | bool |
| 505 | initHeuristics(std::vector<std::pair<MachineInstr *, unsigned>> &RewriteCands, |
| 506 | DenseMap<MachineBasicBlock *, std::set<Register>> &CopyForUse, |
| 507 | SmallPtrSetImpl<MachineInstr *> &CopyForDef); |
| 508 | |
| 509 | /// Calculate the rewrite cost and undo the state change (e.g. rewriting) done |
| 510 | /// in initHeuristics. Uses \p CopyForUse and \p CopyForDef to calculate copy |
| 511 | /// costs, and \p RewriteCands to undo rewriting. |
| 512 | int64_t getRewriteCost( |
| 513 | ArrayRef<std::pair<MachineInstr *, unsigned>> RewriteCands, |
| 514 | const DenseMap<MachineBasicBlock *, std::set<Register>> &CopyForUse, |
| 515 | const SmallPtrSetImpl<MachineInstr *> &CopyForDef); |
| 516 | |
| 517 | /// Do the final rewrite on \p RewriteCands and insert any needed copies. |
| 518 | bool rewrite(ArrayRef<std::pair<MachineInstr *, unsigned>> RewriteCands); |
| 519 | |
| 520 | /// \returns true if this MI is a rewrite candidate. |
| 521 | bool isRewriteCandidate(MachineInstr *MI) const; |
| 522 | |
| 523 | /// Resets all candidates in \p RewriteCands back to VGPR form. |
| 524 | void resetRewriteCandsToVGPR( |
| 525 | ArrayRef<std::pair<MachineInstr *, unsigned>> RewriteCands); |
| 526 | |
| 527 | /// Finds all the reaching defs of \p UseMO and stores the SlotIndexes into \p |
| 528 | /// DefIdxs |
| 529 | void findReachingDefs(MachineOperand &UseMO, LiveIntervals *LIS, |
| 530 | SmallVectorImpl<SlotIndex> &DefIdxs); |
| 531 | |
| 532 | /// Finds all the reaching uses of \p DefMI and stores the use operands in \p |
| 533 | /// ReachingUses |
| 534 | void findReachingUses(const MachineInstr *DefMI, LiveIntervals *LIS, |
| 535 | SmallVectorImpl<MachineOperand *> &ReachingUses); |
| 536 | |
| 537 | /// Returns true if the src2 register with reaching defs \p Src2ReachingDefs |
| 538 | /// has a use other than a group MFMA (in \p RewriteSet) or a copy, which |
| 539 | /// would keep it in VGPR form rather than let it be reclassified to AGPR. |
| 540 | bool hasUseRequiringVGPR(ArrayRef<SlotIndex> Src2ReachingDefs, |
| 541 | const SmallPtrSetImpl<MachineInstr *> &RewriteSet); |
| 542 | |
| 543 | public: |
| 544 | bool initGCNSchedStage() override; |
| 545 | |
| 546 | RewriteMFMAFormStage(GCNSchedStageID StageID, GCNScheduleDAGMILive &DAG) |
| 547 | : GCNSchedStage(StageID, DAG) {} |
| 548 | }; |
| 549 | |
| 550 | class UnclusteredHighRPStage : public GCNSchedStage { |
| 551 | private: |
| 552 | // Save the initial occupancy before starting this stage. |
| 553 | unsigned InitialOccupancy; |
| 554 | // Save the temporary target occupancy before starting this stage. |
| 555 | unsigned TempTargetOccupancy; |
| 556 | // Track whether any region was scheduled by this stage. |
| 557 | bool IsAnyRegionScheduled; |
| 558 | |
| 559 | public: |
| 560 | bool initGCNSchedStage() override; |
| 561 | |
| 562 | void finalizeGCNSchedStage() override; |
| 563 | |
| 564 | bool initGCNRegion() override; |
| 565 | |
| 566 | bool shouldRevertScheduling(unsigned WavesAfter) override; |
| 567 | |
| 568 | UnclusteredHighRPStage(GCNSchedStageID StageID, GCNScheduleDAGMILive &DAG) |
| 569 | : GCNSchedStage(StageID, DAG) {} |
| 570 | }; |
| 571 | |
| 572 | // Retry function scheduling if we found resulting occupancy and it is |
| 573 | // lower than used for other scheduling passes. This will give more freedom |
| 574 | // to schedule low register pressure blocks. |
| 575 | class ClusteredLowOccStage : public GCNSchedStage { |
| 576 | public: |
| 577 | bool initGCNSchedStage() override; |
| 578 | |
| 579 | bool initGCNRegion() override; |
| 580 | |
| 581 | bool shouldRevertScheduling(unsigned WavesAfter) override; |
| 582 | |
| 583 | ClusteredLowOccStage(GCNSchedStageID StageID, GCNScheduleDAGMILive &DAG) |
| 584 | : GCNSchedStage(StageID, DAG) {} |
| 585 | }; |
| 586 | |
| 587 | /// Attempts to reduce function spilling or, if there is no spilling, to |
| 588 | /// increase function occupancy by one with respect to register usage by sinking |
| 589 | /// rematerializable instructions to their use. When the stage estimates that |
| 590 | /// reducing spilling or increasing occupancy is possible, it tries to |
| 591 | /// rematerialize as few registers as possible to reduce potential negative |
| 592 | /// effects on function latency. Uses `GCNRematStrategy` to drive |
| 593 | /// rematerializations. |
| 594 | class PreRARematStage : public GCNSchedStage { |
| 595 | private: |
| 596 | /// Live-outs are not maintained in a contiguous array so we have to have one |
| 597 | /// here to give to the strategy. |
| 598 | SmallVector<GCNRPTracker::LiveRegSet> LiveOuts; |
| 599 | /// Rematerialization strategy. |
| 600 | std::unique_ptr<GCNRematStrategy> Strategy; |
| 601 | /// The target occupancy the stage is tring to achieve. Empty when the |
| 602 | /// objective is spill reduction. |
| 603 | std::optional<unsigned> TargetOcc; |
| 604 | /// Achieved occupancy *only* through rematerializations (pre-rescheduling). |
| 605 | unsigned AchievedOcc; |
| 606 | /// After successful stage initialization, indicates which regions should be |
| 607 | /// rescheduled. |
| 608 | BitVector RescheduleRegions; |
| 609 | |
| 610 | /// State of a region pre-re-scheduling but post-rematerializations that we |
| 611 | /// must keep to be able to revert re-scheduling effects. |
| 612 | struct RegionSchedRevert { |
| 613 | /// Region number. |
| 614 | unsigned RegionIdx; |
| 615 | /// Original instruction order (both debug and non-debug MIs). |
| 616 | std::vector<MachineInstr *> OrigMIOrder; |
| 617 | /// Maximum pressure recorded in the region. |
| 618 | GCNRegPressure MaxPressure; |
| 619 | |
| 620 | RegionSchedRevert(unsigned RegionIdx, ArrayRef<MachineInstr *> OrigMIOrder, |
| 621 | const GCNRegPressure &MaxPressure) |
| 622 | : RegionIdx(RegionIdx), OrigMIOrder(OrigMIOrder), |
| 623 | MaxPressure(MaxPressure) {} |
| 624 | }; |
| 625 | /// After re-scheduling, contains pre-re-scheduling data for all re-scheduled |
| 626 | /// regions. |
| 627 | SmallVector<RegionSchedRevert> RegionReverts; |
| 628 | /// Whether we should revert all re-scheduled regions. |
| 629 | bool RevertAllRegions = false; |
| 630 | |
| 631 | /// Returns the occupancy the stage is trying to achieve. |
| 632 | unsigned getStageTargetOccupancy() const; |
| 633 | |
| 634 | /// If remat alone did not increase occupancy to the target one, rollbacks all |
| 635 | /// rematerializations and resets live-ins/RP in all regions impacted by the |
| 636 | /// stage to their pre-stage values. |
| 637 | void finalizeGCNSchedStage() override; |
| 638 | |
| 639 | public: |
| 640 | bool initGCNSchedStage() override; |
| 641 | |
| 642 | bool initGCNRegion() override; |
| 643 | |
| 644 | void finalizeGCNRegion() override; |
| 645 | |
| 646 | bool shouldRevertScheduling(unsigned WavesAfter) override; |
| 647 | |
| 648 | PreRARematStage(GCNSchedStageID StageID, GCNScheduleDAGMILive &DAG) |
| 649 | : GCNSchedStage(StageID, DAG) {} |
| 650 | }; |
| 651 | |
| 652 | class ILPInitialScheduleStage : public GCNSchedStage { |
| 653 | public: |
| 654 | bool shouldRevertScheduling(unsigned WavesAfter) override; |
| 655 | |
| 656 | ILPInitialScheduleStage(GCNSchedStageID StageID, GCNScheduleDAGMILive &DAG) |
| 657 | : GCNSchedStage(StageID, DAG) {} |
| 658 | }; |
| 659 | |
| 660 | class MemoryClauseInitialScheduleStage : public GCNSchedStage { |
| 661 | public: |
| 662 | bool shouldRevertScheduling(unsigned WavesAfter) override; |
| 663 | |
| 664 | MemoryClauseInitialScheduleStage(GCNSchedStageID StageID, |
| 665 | GCNScheduleDAGMILive &DAG) |
| 666 | : GCNSchedStage(StageID, DAG) {} |
| 667 | }; |
| 668 | |
| 669 | // Live interval register pressure stage: |
| 670 | // estimates register pressure accounting for live interval interference, |
| 671 | // optionally reschedules the region with tigher register bounds. |
| 672 | class LiveIntervalRPStage : public GCNSchedStage { |
| 673 | public: |
| 674 | bool initGCNSchedStage() override; |
| 675 | bool initGCNRegion() override; |
| 676 | void finalizeGCNRegion() override; |
| 677 | |
| 678 | LiveIntervalRPStage(GCNSchedStageID StageID, GCNScheduleDAGMILive &DAG) |
| 679 | : GCNSchedStage(StageID, DAG) {} |
| 680 | |
| 681 | private: |
| 682 | unsigned SavedVGPRThresholdPercent = 0; |
| 683 | unsigned SavedVGPRExcessLimit = 0; |
| 684 | unsigned SavedVGPRCriticalLimit = 0; |
| 685 | unsigned SavedUnifiedVGPRCriticalLimit = 0; |
| 686 | }; |
| 687 | |
| 688 | class GCNPostScheduleDAGMILive final : public ScheduleDAGMI { |
| 689 | private: |
| 690 | std::vector<std::unique_ptr<ScheduleDAGMutation>> SavedMutations; |
| 691 | |
| 692 | bool HasIGLPInstrs = false; |
| 693 | |
| 694 | public: |
| 695 | void schedule() override; |
| 696 | |
| 697 | void finalizeSchedule() override; |
| 698 | |
| 699 | GCNPostScheduleDAGMILive(MachineSchedContext *C, |
| 700 | std::unique_ptr<MachineSchedStrategy> S, |
| 701 | bool RemoveKillFlags); |
| 702 | }; |
| 703 | |
| 704 | } // End namespace llvm |
| 705 | |
| 706 | #endif // LLVM_LIB_TARGET_AMDGPU_GCNSCHEDSTRATEGY_H |
| 707 | |