1//===-- GCNSchedStrategy.h - GCN Scheduler Strategy -*- C++ -*-------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_LIB_TARGET_AMDGPU_GCNSCHEDSTRATEGY_H
14#define LLVM_LIB_TARGET_AMDGPU_GCNSCHEDSTRATEGY_H
15
16#include "GCNRegPressure.h"
17#include "GCNRematStrategy.h"
18#include "llvm/ADT/DenseMap.h"
19#include "llvm/CodeGen/MachineBasicBlock.h"
20#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
21#include "llvm/CodeGen/MachineInstr.h"
22#include "llvm/CodeGen/MachineScheduler.h"
23#include "llvm/Support/CommandLine.h"
24
25namespace llvm {
26
27struct VGPRThresholdParser : public cl::parser<unsigned> {
28 VGPRThresholdParser(cl::Option &O) : cl::parser<unsigned>(O) {}
29 bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value);
30};
31
32extern cl::opt<unsigned, false, VGPRThresholdParser> VGPRThresholdPercentOpt;
33
34class SIMachineFunctionInfo;
35class SIRegisterInfo;
36class GCNSubtarget;
37class GCNSchedStage;
38
39enum class GCNSchedStageID : unsigned {
40 OccInitialSchedule = 0,
41 RewriteMFMAForm = 1,
42 UnclusteredHighRPReschedule = 2,
43 ClusteredLowOccupancyReschedule = 3,
44 PreRARematerialize = 4,
45 ILPInitialSchedule = 5,
46 MemoryClauseInitialSchedule = 6,
47 LiveIntervalRPReschedule = 7
48};
49
50#ifndef NDEBUG
51raw_ostream &operator<<(raw_ostream &OS, const GCNSchedStageID &StageID);
52#endif
53
54/// This is a minimal scheduler strategy. The main difference between this
55/// and the GenericScheduler is that GCNSchedStrategy uses different
56/// heuristics to determine excess/critical pressure sets.
57class GCNSchedStrategy : public GenericScheduler {
58protected:
59 SUnit *pickNodeBidirectional(bool &IsTopNode, bool &PickedPending);
60
61 void pickNodeFromQueue(SchedBoundary &Zone, const CandPolicy &ZonePolicy,
62 const RegPressureTracker &RPTracker,
63 SchedCandidate &Cand, bool &IsPending,
64 bool IsBottomUp);
65
66 void initCandidate(SchedCandidate &Cand, SUnit *SU, bool AtTop,
67 const RegPressureTracker &RPTracker,
68 const SIRegisterInfo *SRI, unsigned SGPRPressure,
69 unsigned VGPRPressure, unsigned AGPRPressure,
70 bool IsBottomUp);
71
72 /// Evaluates instructions in the pending queue using a subset of scheduling
73 /// heuristics.
74 ///
75 /// Instructions that cannot be issued due to hardware constraints are placed
76 /// in the pending queue rather than the available queue, making them normally
77 /// invisible to scheduling heuristics. However, in certain scenarios (such as
78 /// avoiding register spilling), it may be beneficial to consider scheduling
79 /// these not-yet-ready instructions.
80 bool tryPendingCandidate(SchedCandidate &Cand, SchedCandidate &TryCand,
81 SchedBoundary *Zone) const;
82
83 void printCandidateDecision(const SchedCandidate &Current,
84 const SchedCandidate &Preferred);
85
86 void getRegisterPressures(bool AtTop, const RegPressureTracker &RPTracker,
87 SUnit *SU, std::vector<unsigned> &Pressure,
88 std::vector<unsigned> &MaxPressure,
89 const GCNDownwardRPTrackerBase &DownwardTracker,
90 const GCNUpwardRPTracker &UpwardTracker,
91 ScheduleDAGMI *DAG, const SIRegisterInfo *SRI);
92
93 /// \returns the ArchVGPR and AGPR pressure of \p RP after distributing
94 /// bank-agnostic AVGPR pressure across the two banks.
95 std::pair<unsigned, unsigned>
96 getBankedPressure(const GCNRegPressure &RP) const;
97
98 const AMDGPUOptions &CLOpts;
99
100 std::vector<unsigned> Pressure;
101
102 std::vector<unsigned> MaxPressure;
103
104 /// Capacity of the ArchVGPR bank, captured before VGPRExcessLimit is cut
105 /// down by ErrorMargin, VGPRLimitBias, or VGPRThresholdPercent.
106 unsigned ArchVGPRBankLimit = 0;
107
108 /// Capacity of the AGPR bank, captured before AGPRExcessLimit is cut down by
109 /// ErrorMargin and VGPRLimitBias. Zero when no AGPRs are allocatable.
110 unsigned AGPRBankLimit = 0;
111
112 /// Budget for the combined ArchVGPR + AGPR file, or zero on targets without
113 /// one.
114 unsigned UnifiedVGPRExcessLimit = 0;
115
116 /// Combined-file pressure of the candidate most recently passed to
117 /// getRegisterPressures(). Only maintained when the GCN trackers are in use.
118 unsigned CandUnifiedVGPRPressure = 0;
119
120 unsigned TargetOccupancy;
121
122 MachineFunction *MF;
123
124 // Scheduling stages for this strategy.
125 SmallVector<GCNSchedStageID, 4> SchedStages;
126
127 // Pointer to the current SchedStageID.
128 SmallVectorImpl<GCNSchedStageID>::iterator CurrentStage = nullptr;
129
130 // GCN RP Tracker for top-down scheduling
131 mutable GCNManualDownwardRPTracker DownwardTracker;
132
133 // GCN RP Tracker for botttom-up scheduling
134 mutable GCNUpwardRPTracker UpwardTracker;
135
136 bool UseGCNTrackers = false;
137
138 std::optional<bool> GCNTrackersOverride;
139
140public:
141 // schedule() have seen register pressure over the critical limits and had to
142 // track register pressure for actual scheduling heuristics.
143 bool HasHighPressure;
144
145 // Schedule known to have excess register pressure. Be more conservative in
146 // increasing ILP and preserving VGPRs.
147 bool KnownExcessRP = false;
148
149 // An error margin is necessary because of poor performance of the generic RP
150 // tracker and can be adjusted up for tuning heuristics to try and more
151 // aggressively reduce register pressure.
152 unsigned ErrorMargin = 3;
153
154 // Bias for SGPR limits under a high register pressure.
155 const unsigned HighRPSGPRBias = 7;
156
157 // Bias for VGPR limits under a high register pressure.
158 const unsigned HighRPVGPRBias = 7;
159
160 unsigned SGPRExcessLimit = 0;
161
162 unsigned VGPRExcessLimit = 0;
163
164 unsigned AGPRExcessLimit = 0;
165
166 /// \returns the allocatable ArchVGPR and AGPR counts.
167 std::pair<unsigned, unsigned> getBankLimits() const {
168 return {ArchVGPRBankLimit, AGPRBankLimit};
169 }
170
171 unsigned SGPRCriticalLimit = 0;
172
173 unsigned VGPRCriticalLimit = 0;
174
175 // Occupancy budget for the combined VGPR/AGPR allocation on unified targets.
176 // Unlike the per-bank limit, this is not clamped to allocatable VGPRs.
177 unsigned UnifiedVGPRCriticalLimit = 0;
178
179 unsigned AGPRCriticalLimit = 0;
180
181 unsigned SGPRLimitBias = 0;
182
183 unsigned VGPRLimitBias = 0;
184
185 unsigned VGPRThresholdPercent = 0;
186
187 GCNSchedStrategy(const MachineSchedContext *C);
188
189 SUnit *pickNode(bool &IsTopNode) override;
190
191 void schedNode(SUnit *SU, bool IsTopNode) override;
192
193 void initialize(ScheduleDAGMI *DAG) override;
194
195 unsigned getTargetOccupancy() { return TargetOccupancy; }
196
197 void setTargetOccupancy(unsigned Occ) { TargetOccupancy = Occ; }
198
199 GCNSchedStageID getCurrentStage();
200
201 // Advances stage. Returns true if there are remaining stages.
202 bool advanceStage();
203
204 bool hasNextStage() const;
205
206 bool useGCNTrackers() const {
207 return GCNTrackersOverride.value_or(u: UseGCNTrackers);
208 }
209
210 GCNDownwardRPTrackerBase &getDownwardTracker() { return DownwardTracker; }
211
212 GCNUpwardRPTracker &getUpwardTracker() { return UpwardTracker; }
213};
214
215/// The goal of this scheduling strategy is to maximize kernel occupancy (i.e.
216/// maximum number of waves per simd).
217class GCNMaxOccupancySchedStrategy final : public GCNSchedStrategy {
218public:
219 GCNMaxOccupancySchedStrategy(const MachineSchedContext *C,
220 bool IsLegacyScheduler = false);
221};
222
223/// The goal of this scheduling strategy is to maximize ILP for a single wave
224/// (i.e. latency hiding).
225class GCNMaxILPSchedStrategy final : public GCNSchedStrategy {
226protected:
227 bool tryCandidate(SchedCandidate &Cand, SchedCandidate &TryCand,
228 SchedBoundary *Zone) const override;
229
230public:
231 GCNMaxILPSchedStrategy(const MachineSchedContext *C);
232};
233
234/// The goal of this scheduling strategy is to maximize memory clause for a
235/// single wave.
236class GCNMaxMemoryClauseSchedStrategy final : public GCNSchedStrategy {
237protected:
238 bool tryCandidate(SchedCandidate &Cand, SchedCandidate &TryCand,
239 SchedBoundary *Zone) const override;
240
241public:
242 GCNMaxMemoryClauseSchedStrategy(const MachineSchedContext *C);
243};
244
245class ScheduleMetrics {
246 unsigned ScheduleLength;
247 unsigned BubbleCycles;
248
249public:
250 ScheduleMetrics() = default;
251 ScheduleMetrics(unsigned L, unsigned BC)
252 : ScheduleLength(L), BubbleCycles(BC) {}
253 unsigned getLength() const { return ScheduleLength; }
254 unsigned getBubbles() const { return BubbleCycles; }
255 unsigned getMetric() const {
256 unsigned Metric = (BubbleCycles * ScaleFactor) / ScheduleLength;
257 // Metric is zero if the amount of bubbles is less than 1% which is too
258 // small. So, return 1.
259 return Metric ? Metric : 1;
260 }
261 static const unsigned ScaleFactor;
262};
263
264inline raw_ostream &operator<<(raw_ostream &OS, const ScheduleMetrics &Sm) {
265 dbgs() << "\n Schedule Metric (scaled by " << ScheduleMetrics::ScaleFactor
266 << " ) is: " << Sm.getMetric() << " [ " << Sm.getBubbles() << "/"
267 << Sm.getLength() << " ]\n";
268 return OS;
269}
270
271class GCNScheduleDAGMILive;
272class RegionPressureMap {
273 GCNScheduleDAGMILive *DAG;
274 // The live in/out pressure as indexed by the first or last MI in the region
275 // before scheduling.
276 DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> RegionLiveRegMap;
277 // The mapping of RegionIDx to key instruction
278 DenseMap<unsigned, MachineInstr *> IdxToInstruction;
279 // Whether we are calculating LiveOuts or LiveIns
280 bool IsLiveOut;
281
282public:
283 RegionPressureMap() = default;
284 RegionPressureMap(GCNScheduleDAGMILive *GCNDAG, bool LiveOut)
285 : DAG(GCNDAG), IsLiveOut(LiveOut) {}
286 // Build the Instr->LiveReg and RegionIdx->Instr maps
287 void buildLiveRegMap();
288
289 // Retrieve the LiveReg for a given RegionIdx
290 GCNRPTracker::LiveRegSet &getLiveRegsForRegionIdx(unsigned RegionIdx) {
291 assert(IdxToInstruction.contains(RegionIdx));
292 MachineInstr *Key = IdxToInstruction[RegionIdx];
293 return RegionLiveRegMap[Key];
294 }
295};
296
297/// A region's boundaries i.e. a pair of instruction bundle iterators. The lower
298/// boundary is inclusive, the upper boundary is exclusive.
299using RegionBoundaries =
300 std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>;
301
302class GCNScheduleDAGMILive final : public ScheduleDAGMILive {
303 friend class GCNSchedStage;
304 friend class OccInitialScheduleStage;
305 friend class RewriteMFMAFormStage;
306 friend class UnclusteredHighRPStage;
307 friend class ClusteredLowOccStage;
308 friend class PreRARematStage;
309 friend class ILPInitialScheduleStage;
310 friend class LiveIntervalRPStage;
311 friend class RegionPressureMap;
312
313 const GCNSubtarget &ST;
314
315 SIMachineFunctionInfo &MFI;
316
317 // Occupancy target at the beginning of function scheduling cycle.
318 unsigned StartingOccupancy;
319
320 // Minimal real occupancy recorder for the function.
321 unsigned MinOccupancy;
322
323 // Vector of regions recorder for later rescheduling
324 SmallVector<RegionBoundaries, 32> Regions;
325
326 // Record regions with high register pressure.
327 BitVector RegionsWithHighRP;
328
329 // Record regions with excess register pressure over the physical register
330 // limit. Register pressure in these regions usually will result in spilling.
331 BitVector RegionsWithExcessRP;
332
333 // Regions that have IGLP instructions (SCHED_GROUP_BARRIER or IGLP_OPT).
334 BitVector RegionsWithIGLPInstrs;
335
336 // Region live-in cache.
337 SmallVector<GCNRPTracker::LiveRegSet, 32> LiveIns;
338
339 // Region pressure cache.
340 SmallVector<GCNRegPressure, 32> Pressure;
341
342 // Temporary basic block live-in cache.
343 DenseMap<const MachineBasicBlock *, GCNRPTracker::LiveRegSet> MBBLiveIns;
344
345 // The map of the initial first region instruction to region live in registers
346 DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> BBLiveInMap;
347
348 // Calculate the map of the initial first region instruction to region live in
349 // registers
350 DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> getRegionLiveInMap() const;
351
352 // Calculate the map of the initial last region instruction to region live out
353 // registers
354 DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet>
355 getRegionLiveOutMap() const;
356
357 // The live out registers per region. These are internally stored as a map of
358 // the initial last region instruction to region live out registers, but can
359 // be retreived with the regionIdx by calls to getLiveRegsForRegionIdx.
360 RegionPressureMap RegionLiveOuts;
361
362 // Return current region pressure.
363 /// \returns the maximum pressure of region \p RegionIdx. If set, \p
364 /// PressureCallback is called with the pressure of every program point of
365 /// the region.
366 GCNRegPressure getRealRegPressure(unsigned RegionIdx,
367 std::function<void(const GCNRegPressure &)>
368 PressureCallback = nullptr) const;
369
370 // Compute and cache live-ins and pressure for all regions in block.
371 void computeBlockPressure(unsigned RegionIdx, const MachineBasicBlock *MBB);
372
373 /// Makes the scheduler try to achieve an occupancy of \p TargetOccupancy.
374 void setTargetOccupancy(unsigned TargetOccupancy);
375
376 void runSchedStages();
377
378 std::unique_ptr<GCNSchedStage> createSchedStage(GCNSchedStageID SchedStageID);
379
380public:
381 GCNScheduleDAGMILive(MachineSchedContext *C,
382 std::unique_ptr<MachineSchedStrategy> S);
383
384 void schedule() override;
385
386 void finalizeSchedule() override;
387};
388
389// GCNSchedStrategy applies multiple scheduling stages to a function.
390class GCNSchedStage {
391protected:
392 GCNScheduleDAGMILive &DAG;
393
394 GCNSchedStrategy &S;
395
396 MachineFunction &MF;
397
398 SIMachineFunctionInfo &MFI;
399
400 const GCNSubtarget &ST;
401
402 const GCNSchedStageID StageID;
403
404 // The current block being scheduled.
405 MachineBasicBlock *CurrentMBB = nullptr;
406
407 // Current region index.
408 unsigned RegionIdx = 0;
409
410 // Record the original order of instructions before scheduling.
411 std::vector<MachineInstr *> Unsched;
412
413 // RP before scheduling the current region.
414 GCNRegPressure PressureBefore;
415
416 // RP after scheduling the current region.
417 GCNRegPressure PressureAfter;
418
419 std::vector<std::unique_ptr<ScheduleDAGMutation>> SavedMutations;
420
421 GCNSchedStage(GCNSchedStageID StageID, GCNScheduleDAGMILive &DAG);
422
423public:
424 // Initialize state for a scheduling stage. Returns false if the current stage
425 // should be skipped.
426 virtual bool initGCNSchedStage();
427
428 // Finalize state after finishing a scheduling pass on the function.
429 virtual void finalizeGCNSchedStage();
430
431 // Setup for scheduling a region. Returns false if the current region should
432 // be skipped.
433 virtual bool initGCNRegion();
434
435 // Finalize state after scheduling a region.
436 virtual void finalizeGCNRegion();
437
438 // Track whether a new region is also a new MBB.
439 void setupNewBlock();
440
441 // Check result of scheduling.
442 void checkScheduling();
443
444 // computes the given schedule virtual execution time in clocks
445 ScheduleMetrics getScheduleMetrics(const std::vector<SUnit> &InputSchedule);
446 ScheduleMetrics getScheduleMetrics(const GCNScheduleDAGMILive &DAG);
447 unsigned computeSUnitReadyCycle(const SUnit &SU, unsigned CurrCycle,
448 DenseMap<unsigned, unsigned> &ReadyCycles,
449 const TargetSchedModel &SM);
450
451 // Returns true if scheduling should be reverted.
452 virtual bool shouldRevertScheduling(unsigned WavesAfter);
453
454 // Returns true if current region has known excess pressure.
455 bool isRegionWithExcessRP() const {
456 return DAG.RegionsWithExcessRP[RegionIdx];
457 }
458
459 // The region number this stage is currently working on
460 unsigned getRegionIdx() { return RegionIdx; }
461
462 // Returns true if the new schedule may result in more spilling.
463 bool mayCauseSpilling(unsigned WavesAfter);
464
465 /// Sets the schedule of region \p RegionIdx to \p MIOrder. The MIs in \p
466 /// MIOrder must be exactly the same as the ones currently existing inside the
467 /// region, only in a different order that honors def-use chains.
468 void modifyRegionSchedule(unsigned RegionIdx,
469 ArrayRef<MachineInstr *> MIOrder);
470
471 void advanceRegion() { RegionIdx++; }
472
473 virtual ~GCNSchedStage() = default;
474};
475
476class OccInitialScheduleStage : public GCNSchedStage {
477public:
478 bool shouldRevertScheduling(unsigned WavesAfter) override;
479
480 OccInitialScheduleStage(GCNSchedStageID StageID, GCNScheduleDAGMILive &DAG)
481 : GCNSchedStage(StageID, DAG) {}
482};
483
484class RewriteMFMAFormStage : public GCNSchedStage {
485private:
486 // Record regions with excess archvgpr register pressure over the physical
487 // register limit. Register pressure in these regions usually will result in
488 // spilling.
489 BitVector RegionsWithExcessArchVGPR;
490
491 const SIInstrInfo *TII;
492 const SIRegisterInfo *SRI;
493
494 /// Per-candidate cache of the src2 "needs VGPR" decision, computed once
495 /// and reused on-demand.
496 DenseMap<const MachineInstr *, bool> Src2NeedsVGPRCache;
497
498 /// Do a speculative rewrite and collect copy locations. The speculative
499 /// rewrite allows us to calculate the RP of the code after the rewrite, and
500 /// the copy locations allow us to calculate the total cost of copies required
501 /// for the rewrite. Stores the rewritten instructions in \p RewriteCands ,
502 /// the copy locations for uses (of the MFMA result) in \p CopyForUse and the
503 /// copy locations for defs (of the MFMA operands) in \p CopyForDef
504 bool
505 initHeuristics(std::vector<std::pair<MachineInstr *, unsigned>> &RewriteCands,
506 DenseMap<MachineBasicBlock *, std::set<Register>> &CopyForUse,
507 SmallPtrSetImpl<MachineInstr *> &CopyForDef);
508
509 /// Calculate the rewrite cost and undo the state change (e.g. rewriting) done
510 /// in initHeuristics. Uses \p CopyForUse and \p CopyForDef to calculate copy
511 /// costs, and \p RewriteCands to undo rewriting.
512 int64_t getRewriteCost(
513 ArrayRef<std::pair<MachineInstr *, unsigned>> RewriteCands,
514 const DenseMap<MachineBasicBlock *, std::set<Register>> &CopyForUse,
515 const SmallPtrSetImpl<MachineInstr *> &CopyForDef);
516
517 /// Do the final rewrite on \p RewriteCands and insert any needed copies.
518 bool rewrite(ArrayRef<std::pair<MachineInstr *, unsigned>> RewriteCands);
519
520 /// \returns true if this MI is a rewrite candidate.
521 bool isRewriteCandidate(MachineInstr *MI) const;
522
523 /// Resets all candidates in \p RewriteCands back to VGPR form.
524 void resetRewriteCandsToVGPR(
525 ArrayRef<std::pair<MachineInstr *, unsigned>> RewriteCands);
526
527 /// Finds all the reaching defs of \p UseMO and stores the SlotIndexes into \p
528 /// DefIdxs
529 void findReachingDefs(MachineOperand &UseMO, LiveIntervals *LIS,
530 SmallVectorImpl<SlotIndex> &DefIdxs);
531
532 /// Finds all the reaching uses of \p DefMI and stores the use operands in \p
533 /// ReachingUses
534 void findReachingUses(const MachineInstr *DefMI, LiveIntervals *LIS,
535 SmallVectorImpl<MachineOperand *> &ReachingUses);
536
537 /// Returns true if the src2 register with reaching defs \p Src2ReachingDefs
538 /// has a use other than a group MFMA (in \p RewriteSet) or a copy, which
539 /// would keep it in VGPR form rather than let it be reclassified to AGPR.
540 bool hasUseRequiringVGPR(ArrayRef<SlotIndex> Src2ReachingDefs,
541 const SmallPtrSetImpl<MachineInstr *> &RewriteSet);
542
543public:
544 bool initGCNSchedStage() override;
545
546 RewriteMFMAFormStage(GCNSchedStageID StageID, GCNScheduleDAGMILive &DAG)
547 : GCNSchedStage(StageID, DAG) {}
548};
549
550class UnclusteredHighRPStage : public GCNSchedStage {
551private:
552 // Save the initial occupancy before starting this stage.
553 unsigned InitialOccupancy;
554 // Save the temporary target occupancy before starting this stage.
555 unsigned TempTargetOccupancy;
556 // Track whether any region was scheduled by this stage.
557 bool IsAnyRegionScheduled;
558
559public:
560 bool initGCNSchedStage() override;
561
562 void finalizeGCNSchedStage() override;
563
564 bool initGCNRegion() override;
565
566 bool shouldRevertScheduling(unsigned WavesAfter) override;
567
568 UnclusteredHighRPStage(GCNSchedStageID StageID, GCNScheduleDAGMILive &DAG)
569 : GCNSchedStage(StageID, DAG) {}
570};
571
572// Retry function scheduling if we found resulting occupancy and it is
573// lower than used for other scheduling passes. This will give more freedom
574// to schedule low register pressure blocks.
575class ClusteredLowOccStage : public GCNSchedStage {
576public:
577 bool initGCNSchedStage() override;
578
579 bool initGCNRegion() override;
580
581 bool shouldRevertScheduling(unsigned WavesAfter) override;
582
583 ClusteredLowOccStage(GCNSchedStageID StageID, GCNScheduleDAGMILive &DAG)
584 : GCNSchedStage(StageID, DAG) {}
585};
586
587/// Attempts to reduce function spilling or, if there is no spilling, to
588/// increase function occupancy by one with respect to register usage by sinking
589/// rematerializable instructions to their use. When the stage estimates that
590/// reducing spilling or increasing occupancy is possible, it tries to
591/// rematerialize as few registers as possible to reduce potential negative
592/// effects on function latency. Uses `GCNRematStrategy` to drive
593/// rematerializations.
594class PreRARematStage : public GCNSchedStage {
595private:
596 /// Live-outs are not maintained in a contiguous array so we have to have one
597 /// here to give to the strategy.
598 SmallVector<GCNRPTracker::LiveRegSet> LiveOuts;
599 /// Rematerialization strategy.
600 std::unique_ptr<GCNRematStrategy> Strategy;
601 /// The target occupancy the stage is tring to achieve. Empty when the
602 /// objective is spill reduction.
603 std::optional<unsigned> TargetOcc;
604 /// Achieved occupancy *only* through rematerializations (pre-rescheduling).
605 unsigned AchievedOcc;
606 /// After successful stage initialization, indicates which regions should be
607 /// rescheduled.
608 BitVector RescheduleRegions;
609
610 /// State of a region pre-re-scheduling but post-rematerializations that we
611 /// must keep to be able to revert re-scheduling effects.
612 struct RegionSchedRevert {
613 /// Region number.
614 unsigned RegionIdx;
615 /// Original instruction order (both debug and non-debug MIs).
616 std::vector<MachineInstr *> OrigMIOrder;
617 /// Maximum pressure recorded in the region.
618 GCNRegPressure MaxPressure;
619
620 RegionSchedRevert(unsigned RegionIdx, ArrayRef<MachineInstr *> OrigMIOrder,
621 const GCNRegPressure &MaxPressure)
622 : RegionIdx(RegionIdx), OrigMIOrder(OrigMIOrder),
623 MaxPressure(MaxPressure) {}
624 };
625 /// After re-scheduling, contains pre-re-scheduling data for all re-scheduled
626 /// regions.
627 SmallVector<RegionSchedRevert> RegionReverts;
628 /// Whether we should revert all re-scheduled regions.
629 bool RevertAllRegions = false;
630
631 /// Returns the occupancy the stage is trying to achieve.
632 unsigned getStageTargetOccupancy() const;
633
634 /// If remat alone did not increase occupancy to the target one, rollbacks all
635 /// rematerializations and resets live-ins/RP in all regions impacted by the
636 /// stage to their pre-stage values.
637 void finalizeGCNSchedStage() override;
638
639public:
640 bool initGCNSchedStage() override;
641
642 bool initGCNRegion() override;
643
644 void finalizeGCNRegion() override;
645
646 bool shouldRevertScheduling(unsigned WavesAfter) override;
647
648 PreRARematStage(GCNSchedStageID StageID, GCNScheduleDAGMILive &DAG)
649 : GCNSchedStage(StageID, DAG) {}
650};
651
652class ILPInitialScheduleStage : public GCNSchedStage {
653public:
654 bool shouldRevertScheduling(unsigned WavesAfter) override;
655
656 ILPInitialScheduleStage(GCNSchedStageID StageID, GCNScheduleDAGMILive &DAG)
657 : GCNSchedStage(StageID, DAG) {}
658};
659
660class MemoryClauseInitialScheduleStage : public GCNSchedStage {
661public:
662 bool shouldRevertScheduling(unsigned WavesAfter) override;
663
664 MemoryClauseInitialScheduleStage(GCNSchedStageID StageID,
665 GCNScheduleDAGMILive &DAG)
666 : GCNSchedStage(StageID, DAG) {}
667};
668
669// Live interval register pressure stage:
670// estimates register pressure accounting for live interval interference,
671// optionally reschedules the region with tigher register bounds.
672class LiveIntervalRPStage : public GCNSchedStage {
673public:
674 bool initGCNSchedStage() override;
675 bool initGCNRegion() override;
676 void finalizeGCNRegion() override;
677
678 LiveIntervalRPStage(GCNSchedStageID StageID, GCNScheduleDAGMILive &DAG)
679 : GCNSchedStage(StageID, DAG) {}
680
681private:
682 unsigned SavedVGPRThresholdPercent = 0;
683 unsigned SavedVGPRExcessLimit = 0;
684 unsigned SavedVGPRCriticalLimit = 0;
685 unsigned SavedUnifiedVGPRCriticalLimit = 0;
686};
687
688class GCNPostScheduleDAGMILive final : public ScheduleDAGMI {
689private:
690 std::vector<std::unique_ptr<ScheduleDAGMutation>> SavedMutations;
691
692 bool HasIGLPInstrs = false;
693
694public:
695 void schedule() override;
696
697 void finalizeSchedule() override;
698
699 GCNPostScheduleDAGMILive(MachineSchedContext *C,
700 std::unique_ptr<MachineSchedStrategy> S,
701 bool RemoveKillFlags);
702};
703
704} // End namespace llvm
705
706#endif // LLVM_LIB_TARGET_AMDGPU_GCNSCHEDSTRATEGY_H
707