1//===- GCNIterativeScheduler.cpp ------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the class GCNIterativeScheduler.
11///
12//===----------------------------------------------------------------------===//
13
14#include "GCNIterativeScheduler.h"
15#include "AMDGPUIGroupLP.h"
16#include "GCNSchedStrategy.h"
17#include "SIMachineFunctionInfo.h"
18#include "llvm/CodeGen/RegisterPressure.h"
19
20using namespace llvm;
21
22#define DEBUG_TYPE "machine-scheduler"
23
24namespace llvm {
25
26std::vector<const SUnit *> makeMinRegSchedule(ArrayRef<const SUnit *> TopRoots,
27 const ScheduleDAG &DAG);
28
29std::vector<const SUnit *> makeGCNILPScheduler(ArrayRef<const SUnit *> BotRoots,
30 const ScheduleDAG &DAG);
31} // namespace llvm
32
33// shim accessors for different order containers
34static inline MachineInstr *getMachineInstr(MachineInstr *MI) {
35 return MI;
36}
37static inline MachineInstr *getMachineInstr(const SUnit *SU) {
38 return SU->getInstr();
39}
40static inline MachineInstr *getMachineInstr(const SUnit &SU) {
41 return SU.getInstr();
42}
43
44#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
45LLVM_DUMP_METHOD
46static void printRegion(raw_ostream &OS,
47 MachineBasicBlock::iterator Begin,
48 MachineBasicBlock::iterator End,
49 const LiveIntervals *LIS,
50 unsigned MaxInstNum =
51 std::numeric_limits<unsigned>::max()) {
52 auto *BB = Begin->getParent();
53 OS << BB->getParent()->getName() << ":" << printMBBReference(*BB) << ' '
54 << BB->getName() << ":\n";
55 auto I = Begin;
56 MaxInstNum = std::max(MaxInstNum, 1u);
57 for (; I != End && MaxInstNum; ++I, --MaxInstNum) {
58 if (!I->isDebugInstr() && LIS)
59 OS << LIS->getInstructionIndex(*I);
60 OS << '\t' << *I;
61 }
62 if (I != End) {
63 OS << "\t...\n";
64 I = std::prev(End);
65 if (!I->isDebugInstr() && LIS)
66 OS << LIS->getInstructionIndex(*I);
67 OS << '\t' << *I;
68 }
69 if (End != BB->end()) { // print boundary inst if present
70 OS << "----\n";
71 if (LIS) OS << LIS->getInstructionIndex(*End) << '\t';
72 OS << *End;
73 }
74}
75
76LLVM_DUMP_METHOD
77static void printLivenessInfo(raw_ostream &OS,
78 MachineBasicBlock::iterator Begin,
79 MachineBasicBlock::iterator End,
80 const LiveIntervals *LIS) {
81 auto *const BB = Begin->getParent();
82 const auto &MRI = BB->getParent()->getRegInfo();
83
84 const auto LiveIns = getLiveRegsBefore(*Begin, *LIS);
85 OS << "LIn RP: " << print(getRegPressure(MRI, LiveIns));
86
87 const auto BottomMI = End == BB->end() ? std::prev(End) : End;
88 const auto LiveOuts = getLiveRegsAfter(*BottomMI, *LIS);
89 OS << "LOt RP: " << print(getRegPressure(MRI, LiveOuts));
90}
91
92LLVM_DUMP_METHOD
93void GCNIterativeScheduler::printRegions(raw_ostream &OS) const {
94 const auto &ST = MF.getSubtarget<GCNSubtarget>();
95 for (auto *const R : Regions) {
96 OS << "Region to schedule ";
97 printRegion(OS, R->Begin, R->End, LIS, 1);
98 printLivenessInfo(OS, R->Begin, R->End, LIS);
99 OS << "Max RP: " << print(R->MaxPressure, &ST);
100 }
101}
102
103LLVM_DUMP_METHOD
104void GCNIterativeScheduler::printSchedResult(raw_ostream &OS,
105 const Region *R,
106 const GCNRegPressure &RP) const {
107 OS << "\nAfter scheduling ";
108 printRegion(OS, R->Begin, R->End, LIS);
109 printSchedRP(OS, R->MaxPressure, RP);
110 OS << '\n';
111}
112
113LLVM_DUMP_METHOD
114void GCNIterativeScheduler::printSchedRP(raw_ostream &OS,
115 const GCNRegPressure &Before,
116 const GCNRegPressure &After) const {
117 const auto &ST = MF.getSubtarget<GCNSubtarget>();
118 OS << "RP before: " << print(Before, &ST)
119 << "RP after: " << print(After, &ST);
120}
121#endif
122
123void GCNIterativeScheduler::swapIGLPMutations(const Region &R, bool IsReentry) {
124 bool HasIGLPInstrs = false;
125 const SIInstrInfo *SII = static_cast<const SIInstrInfo *>(TII);
126 for (MachineBasicBlock::iterator I = R.Begin; I != R.End; I++) {
127 if (SII->isIGLPMutationOnly(Opcode: I->getOpcode())) {
128 HasIGLPInstrs = true;
129 break;
130 }
131 }
132
133 if (HasIGLPInstrs) {
134 SavedMutations.clear();
135 SavedMutations.swap(x&: Mutations);
136 auto SchedPhase = IsReentry ? AMDGPU::SchedulingPhase::PreRAReentry
137 : AMDGPU::SchedulingPhase::Initial;
138
139 addMutation(Mutation: createIGroupLPDAGMutation(Phase: SchedPhase));
140 }
141}
142
143// DAG builder helper
144class GCNIterativeScheduler::BuildDAG {
145 GCNIterativeScheduler &Sch;
146 SmallVector<SUnit *, 8> TopRoots;
147
148 SmallVector<SUnit*, 8> BotRoots;
149public:
150 BuildDAG(const Region &R, GCNIterativeScheduler &_Sch, bool IsReentry = false)
151 : Sch(_Sch) {
152 auto *BB = R.Begin->getParent();
153 Sch.BaseClass::startBlock(bb: BB);
154 Sch.BaseClass::enterRegion(bb: BB, begin: R.Begin, end: R.End, regioninstrs: R.NumRegionInstrs);
155 Sch.swapIGLPMutations(R, IsReentry);
156 Sch.buildSchedGraph(AA: Sch.AA, RPTracker: nullptr, PDiffs: nullptr, LIS: nullptr,
157 /*TrackLaneMask*/TrackLaneMasks: true);
158 Sch.postProcessDAG();
159 Sch.Topo.InitDAGTopologicalSorting();
160 Sch.findRootsAndBiasEdges(TopRoots, BotRoots);
161 }
162
163 ~BuildDAG() {
164 Sch.BaseClass::exitRegion();
165 Sch.BaseClass::finishBlock();
166 }
167
168 ArrayRef<const SUnit *> getTopRoots() const {
169 return TopRoots;
170 }
171 ArrayRef<SUnit*> getBottomRoots() const {
172 return BotRoots;
173 }
174};
175
176class GCNIterativeScheduler::OverrideLegacyStrategy {
177 GCNIterativeScheduler &Sch;
178 Region &Rgn;
179 std::unique_ptr<MachineSchedStrategy> SaveSchedImpl;
180 GCNRegPressure SaveMaxRP;
181
182public:
183 OverrideLegacyStrategy(Region &R,
184 MachineSchedStrategy &OverrideStrategy,
185 GCNIterativeScheduler &_Sch)
186 : Sch(_Sch)
187 , Rgn(R)
188 , SaveSchedImpl(std::move(_Sch.SchedImpl))
189 , SaveMaxRP(R.MaxPressure) {
190 Sch.SchedImpl.reset(p: &OverrideStrategy);
191 auto *BB = R.Begin->getParent();
192 Sch.BaseClass::startBlock(bb: BB);
193 Sch.BaseClass::enterRegion(bb: BB, begin: R.Begin, end: R.End, regioninstrs: R.NumRegionInstrs);
194 }
195
196 ~OverrideLegacyStrategy() {
197 Sch.BaseClass::exitRegion();
198 Sch.BaseClass::finishBlock();
199 Sch.SchedImpl.release();
200 Sch.SchedImpl = std::move(SaveSchedImpl);
201 }
202
203 void schedule() {
204 assert(Sch.RegionBegin == Rgn.Begin && Sch.RegionEnd == Rgn.End);
205 LLVM_DEBUG(dbgs() << "\nScheduling ";
206 printRegion(dbgs(), Rgn.Begin, Rgn.End, Sch.LIS, 2));
207 Sch.BaseClass::schedule();
208
209 // Unfortunately placeDebugValues incorrectly modifies RegionEnd, restore
210 Sch.RegionEnd = Rgn.End;
211 //assert(Rgn.End == Sch.RegionEnd);
212 Rgn.Begin = Sch.RegionBegin;
213 Rgn.MaxPressure.clear();
214 }
215
216 void restoreOrder() {
217 assert(Sch.RegionBegin == Rgn.Begin && Sch.RegionEnd == Rgn.End);
218 // DAG SUnits are stored using original region's order
219 // so just use SUnits as the restoring schedule
220 Sch.scheduleRegion(R&: Rgn, Schedule&: Sch.SUnits, MaxRP: SaveMaxRP);
221 }
222};
223
224namespace {
225
226// just a stub to make base class happy
227class SchedStrategyStub : public MachineSchedStrategy {
228public:
229 bool shouldTrackPressure() const override { return false; }
230 bool shouldTrackLaneMasks() const override { return false; }
231 void initialize(ScheduleDAGMI *DAG) override {}
232 SUnit *pickNode(bool &IsTopNode) override { return nullptr; }
233 void schedNode(SUnit *SU, bool IsTopNode) override {}
234 void releaseTopNode(SUnit *SU) override {}
235 void releaseBottomNode(SUnit *SU) override {}
236};
237
238} // end anonymous namespace
239
240GCNIterativeScheduler::GCNIterativeScheduler(MachineSchedContext *C,
241 StrategyKind S)
242 : BaseClass(C, std::make_unique<SchedStrategyStub>())
243 , Context(C)
244 , Strategy(S)
245 , UPTracker(*LIS) {
246}
247
248// returns max pressure for a region
249GCNRegPressure
250GCNIterativeScheduler::getRegionPressure(MachineBasicBlock::iterator Begin,
251 MachineBasicBlock::iterator End)
252 const {
253 // For the purpose of pressure tracking bottom inst of the region should
254 // be also processed. End is either BB end, BB terminator inst or sched
255 // boundary inst.
256 auto const BBEnd = Begin->getParent()->end();
257 auto const BottomMI = End == BBEnd ? std::prev(x: End) : End;
258
259 // scheduleRegions walks bottom to top, so its likely we just get next
260 // instruction to track
261 auto AfterBottomMI = std::next(x: BottomMI);
262 if (AfterBottomMI == BBEnd ||
263 &*AfterBottomMI != UPTracker.getLastTrackedMI()) {
264 UPTracker.reset(MI: *BottomMI);
265 } else {
266 assert(UPTracker.isValid());
267 }
268
269 for (auto I = BottomMI; I != Begin; --I)
270 UPTracker.recede(MI: *I);
271
272 UPTracker.recede(MI: *Begin);
273
274 assert(UPTracker.isValid() ||
275 (dbgs() << "Tracked region ",
276 printRegion(dbgs(), Begin, End, LIS), false));
277 return UPTracker.getMaxPressureAndReset();
278}
279
280// returns max pressure for a tentative schedule
281template <typename Range> GCNRegPressure
282GCNIterativeScheduler::getSchedulePressure(const Region &R,
283 Range &&Schedule) const {
284 auto const BBEnd = R.Begin->getParent()->end();
285 GCNUpwardRPTracker RPTracker(*LIS);
286 if (R.End != BBEnd) {
287 // R.End points to the boundary instruction but the
288 // schedule doesn't include it
289 RPTracker.reset(MI: *R.End);
290 RPTracker.recede(MI: *R.End);
291 } else {
292 // R.End doesn't point to the boundary instruction
293 RPTracker.reset(MI: *std::prev(x: BBEnd));
294 }
295 for (auto I = Schedule.end(), B = Schedule.begin(); I != B;) {
296 RPTracker.recede(MI: *getMachineInstr(*--I));
297 }
298 return RPTracker.getMaxPressureAndReset();
299}
300
301void GCNIterativeScheduler::enterRegion(MachineBasicBlock *BB, // overridden
302 MachineBasicBlock::iterator Begin,
303 MachineBasicBlock::iterator End,
304 unsigned NumRegionInstrs) {
305 BaseClass::enterRegion(bb: BB, begin: Begin, end: End, regioninstrs: NumRegionInstrs);
306 if (NumRegionInstrs > 2) {
307 Regions.push_back(
308 x: new (Alloc.Allocate())
309 Region { .Begin: Begin, .End: End, .NumRegionInstrs: NumRegionInstrs,
310 .MaxPressure: getRegionPressure(Begin, End), .BestSchedule: nullptr });
311 }
312}
313
314void GCNIterativeScheduler::schedule() { // overridden
315 // do nothing
316 LLVM_DEBUG(printLivenessInfo(dbgs(), RegionBegin, RegionEnd, LIS);
317 if (!Regions.empty() && Regions.back()->Begin == RegionBegin) {
318 dbgs() << "Max RP: "
319 << print(Regions.back()->MaxPressure,
320 &MF.getSubtarget<GCNSubtarget>());
321 } dbgs()
322 << '\n';);
323}
324
325void GCNIterativeScheduler::finalizeSchedule() { // overridden
326 if (Regions.empty())
327 return;
328 switch (Strategy) {
329 case SCHEDULE_MINREGONLY: scheduleMinReg(); break;
330 case SCHEDULE_MINREGFORCED: scheduleMinReg(force: true); break;
331 case SCHEDULE_LEGACYMAXOCCUPANCY: scheduleLegacyMaxOccupancy(); break;
332 case SCHEDULE_ILP: scheduleILP(TryMaximizeOccupancy: false); break;
333 }
334}
335
336// Detach schedule from SUnits and interleave it with debug values.
337// Returned schedule becomes independent of DAG state.
338std::vector<MachineInstr*>
339GCNIterativeScheduler::detachSchedule(ScheduleRef Schedule) const {
340 std::vector<MachineInstr*> Res;
341 Res.reserve(n: Schedule.size() * 2);
342
343 if (FirstDbgValue)
344 Res.push_back(x: FirstDbgValue);
345
346 const auto DbgB = DbgValues.begin(), DbgE = DbgValues.end();
347 for (const auto *SU : Schedule) {
348 Res.push_back(x: SU->getInstr());
349 const auto &D = std::find_if(first: DbgB, last: DbgE, pred: [SU](decltype(*DbgB) &P) {
350 return P.second == SU->getInstr();
351 });
352 if (D != DbgE)
353 Res.push_back(x: D->first);
354 }
355 return Res;
356}
357
358void GCNIterativeScheduler::setBestSchedule(Region &R,
359 ScheduleRef Schedule,
360 const GCNRegPressure &MaxRP) {
361 R.BestSchedule.reset(
362 p: new TentativeSchedule{ .Schedule: detachSchedule(Schedule), .MaxPressure: MaxRP });
363}
364
365void GCNIterativeScheduler::scheduleBest(Region &R) {
366 assert(R.BestSchedule.get() && "No schedule specified");
367 scheduleRegion(R, Schedule&: R.BestSchedule->Schedule, MaxRP: R.BestSchedule->MaxPressure);
368 R.BestSchedule.reset();
369}
370
371void GCNIterativeScheduler::restoreRegionLivenessFlags(const Region &R) {
372 for (MachineBasicBlock::iterator I = R.Begin; I != R.End; ++I) {
373 if (!I->isDebugInstr())
374 RegisterOperands::restoreLivenessFlags(MI&: *I, TRI: *TRI, MRI, LIS&: *LIS);
375 }
376}
377
378// minimal required region scheduler, works for ranges of SUnits*,
379// SUnits or MachineIntrs*
380template <typename Range>
381void GCNIterativeScheduler::scheduleRegion(Region &R, Range &&Schedule,
382 const GCNRegPressure &MaxRP) {
383 assert(RegionBegin == R.Begin && RegionEnd == R.End);
384 assert(LIS != nullptr);
385#ifndef NDEBUG
386 const auto SchedMaxRP = getSchedulePressure(R, Schedule);
387#endif
388 auto *BB = R.Begin->getParent();
389 auto Top = R.Begin;
390 for (const auto &I : Schedule) {
391 auto MI = getMachineInstr(I);
392
393 MachineBasicBlock::iterator MII = MI->getIterator();
394 if (MII != Top) {
395 bool NonDebugReordered =
396 !MI->isDebugInstr() && skipDebugInstructionsForward(It: Top, End: MII) != MII;
397 BB->remove(I: MI);
398 BB->insert(Top, MI);
399 if (NonDebugReordered)
400 LIS->handleMove(MI&: *MI, UpdateFlags: true);
401 }
402 if (!MI->isDebugInstr())
403 RegisterOperands::restoreLivenessFlags(MI&: *MI, TRI: *TRI, MRI, LIS&: *LIS);
404 Top = std::next(MI->getIterator());
405 }
406 RegionBegin = getMachineInstr(Schedule.front());
407
408 // Schedule consisting of MachineInstr* is considered 'detached'
409 // and already interleaved with debug values
410 if (!std::is_same_v<decltype(*Schedule.begin()), MachineInstr*>) {
411 placeDebugValues();
412 // Unfortunately placeDebugValues incorrectly modifies RegionEnd, restore
413 // assert(R.End == RegionEnd);
414 RegionEnd = R.End;
415 }
416
417 R.Begin = RegionBegin;
418 R.MaxPressure = MaxRP;
419
420#ifndef NDEBUG
421 const auto RegionMaxRP = getRegionPressure(R);
422 const auto &ST = MF.getSubtarget<GCNSubtarget>();
423#endif
424 assert(
425 (SchedMaxRP == RegionMaxRP && (MaxRP.empty() || SchedMaxRP == MaxRP)) ||
426 (dbgs() << "Max RP mismatch!!!\n"
427 "RP for schedule (calculated): "
428 << print(SchedMaxRP, &ST)
429 << "RP for schedule (reported): " << print(MaxRP, &ST)
430 << "RP after scheduling: " << print(RegionMaxRP, &ST),
431 false));
432}
433
434// Sort recorded regions by pressure - highest at the front
435void GCNIterativeScheduler::sortRegionsByPressure(unsigned TargetOcc) {
436 llvm::sort(C&: Regions, Comp: [this, TargetOcc](const Region *R1, const Region *R2) {
437 return R2->MaxPressure.less(MF, O: R1->MaxPressure, MaxOccupancy: TargetOcc);
438 });
439}
440
441///////////////////////////////////////////////////////////////////////////////
442// Legacy MaxOccupancy Strategy
443
444// Tries to increase occupancy applying minreg scheduler for a sequence of
445// most demanding regions. Obtained schedules are saved as BestSchedule for a
446// region.
447// TargetOcc is the best achievable occupancy for a kernel.
448// Returns better occupancy on success or current occupancy on fail.
449// BestSchedules aren't deleted on fail.
450unsigned GCNIterativeScheduler::tryMaximizeOccupancy(unsigned TargetOcc) {
451 // TODO: assert Regions are sorted descending by pressure
452 const auto &ST = MF.getSubtarget<GCNSubtarget>();
453 const unsigned DynamicVGPRBlockSize =
454 MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize();
455 const auto Occ =
456 Regions.front()->MaxPressure.getOccupancy(ST, DynamicVGPRBlockSize);
457 LLVM_DEBUG(dbgs() << "Trying to improve occupancy, target = " << TargetOcc
458 << ", current = " << Occ << '\n');
459
460 auto NewOcc = TargetOcc;
461 for (auto *R : Regions) {
462 // Always build the DAG to add mutations
463 BuildDAG DAG(*R, *this);
464
465 if (R->MaxPressure.getOccupancy(ST, DynamicVGPRBlockSize) >= NewOcc)
466 continue;
467
468 LLVM_DEBUG(printRegion(dbgs(), R->Begin, R->End, LIS, 3);
469 printLivenessInfo(dbgs(), R->Begin, R->End, LIS));
470
471 const auto MinSchedule = makeMinRegSchedule(TopRoots: DAG.getTopRoots(), DAG: *this);
472 const auto MaxRP = getSchedulePressure(R: *R, Schedule: MinSchedule);
473 LLVM_DEBUG(dbgs() << "Occupancy improvement attempt:\n";
474 printSchedRP(dbgs(), R->MaxPressure, MaxRP));
475
476 NewOcc = std::min(a: NewOcc, b: MaxRP.getOccupancy(ST, DynamicVGPRBlockSize));
477 if (NewOcc <= Occ)
478 break;
479
480 setBestSchedule(R&: *R, Schedule: MinSchedule, MaxRP);
481 }
482 LLVM_DEBUG(dbgs() << "New occupancy = " << NewOcc
483 << ", prev occupancy = " << Occ << '\n');
484 if (NewOcc > Occ) {
485 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
486 MFI->increaseOccupancy(MF, Limit: NewOcc);
487 }
488
489 return std::max(a: NewOcc, b: Occ);
490}
491
492void GCNIterativeScheduler::scheduleLegacyMaxOccupancy(
493 bool TryMaximizeOccupancy) {
494 const auto &ST = MF.getSubtarget<GCNSubtarget>();
495 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
496 auto TgtOcc = MFI->getMinAllowedOccupancy();
497 unsigned DynamicVGPRBlockSize = MFI->getDynamicVGPRBlockSize();
498
499 sortRegionsByPressure(TargetOcc: TgtOcc);
500 auto Occ =
501 Regions.front()->MaxPressure.getOccupancy(ST, DynamicVGPRBlockSize);
502
503 bool IsReentry = false;
504 if (TryMaximizeOccupancy && Occ < TgtOcc) {
505 Occ = tryMaximizeOccupancy(TargetOcc: TgtOcc);
506 IsReentry = true;
507 }
508
509 // This is really weird but for some magic scheduling regions twice
510 // gives performance improvement
511 const int NumPasses = Occ < TgtOcc ? 2 : 1;
512
513 TgtOcc = std::min(a: Occ, b: TgtOcc);
514 LLVM_DEBUG(dbgs() << "Scheduling using default scheduler, "
515 "target occupancy = "
516 << TgtOcc << '\n');
517 GCNMaxOccupancySchedStrategy LStrgy(Context, /*IsLegacyScheduler=*/true);
518 unsigned FinalOccupancy = std::min(a: Occ, b: MFI->getOccupancy());
519
520 for (int I = 0; I < NumPasses; ++I) {
521 // running first pass with TargetOccupancy = 0 mimics previous scheduling
522 // approach and is a performance magic
523 LStrgy.setTargetOccupancy(I == 0 ? 0 : TgtOcc);
524 for (auto *R : Regions) {
525 OverrideLegacyStrategy Ovr(*R, LStrgy, *this);
526 IsReentry |= I > 0;
527 swapIGLPMutations(R: *R, IsReentry);
528 Ovr.schedule();
529 const auto RP = getRegionPressure(R: *R);
530 LLVM_DEBUG(printSchedRP(dbgs(), R->MaxPressure, RP));
531
532 if (RP.getOccupancy(ST, DynamicVGPRBlockSize) < TgtOcc) {
533 LLVM_DEBUG(dbgs() << "Didn't fit into target occupancy O" << TgtOcc);
534 if (R->BestSchedule.get() && R->BestSchedule->MaxPressure.getOccupancy(
535 ST, DynamicVGPRBlockSize) >= TgtOcc) {
536 LLVM_DEBUG(dbgs() << ", scheduling minimal register\n");
537 scheduleBest(R&: *R);
538 } else {
539 LLVM_DEBUG(dbgs() << ", restoring\n");
540 Ovr.restoreOrder();
541 assert(R->MaxPressure.getOccupancy(ST, DynamicVGPRBlockSize) >=
542 TgtOcc);
543 }
544 }
545 FinalOccupancy =
546 std::min(a: FinalOccupancy, b: RP.getOccupancy(ST, DynamicVGPRBlockSize));
547 }
548 }
549 MFI->limitOccupancy(Limit: FinalOccupancy);
550}
551
552///////////////////////////////////////////////////////////////////////////////
553// Minimal Register Strategy
554
555void GCNIterativeScheduler::scheduleMinReg(bool force) {
556 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
557 const auto TgtOcc = MFI->getOccupancy();
558 sortRegionsByPressure(TargetOcc: TgtOcc);
559
560 auto MaxPressure = Regions.front()->MaxPressure;
561 for (auto *R : Regions) {
562 if (!force && R->MaxPressure.less(MF, O: MaxPressure, MaxOccupancy: TgtOcc))
563 break;
564
565 BuildDAG DAG(*R, *this);
566 const auto MinSchedule = makeMinRegSchedule(TopRoots: DAG.getTopRoots(), DAG: *this);
567
568 const auto RP = getSchedulePressure(R: *R, Schedule: MinSchedule);
569 LLVM_DEBUG(if (R->MaxPressure.less(MF, RP, TgtOcc)) {
570 dbgs() << "\nWarning: Pressure becomes worse after minreg!";
571 printSchedRP(dbgs(), R->MaxPressure, RP);
572 });
573
574 if (!force && MaxPressure.less(MF, O: RP, MaxOccupancy: TgtOcc))
575 break;
576
577 scheduleRegion(R&: *R, Schedule: MinSchedule, MaxRP: RP);
578 LLVM_DEBUG(printSchedResult(dbgs(), R, RP));
579
580 MaxPressure = RP;
581 }
582}
583
584///////////////////////////////////////////////////////////////////////////////
585// ILP scheduler port
586
587void GCNIterativeScheduler::scheduleILP(
588 bool TryMaximizeOccupancy) {
589 const auto &ST = MF.getSubtarget<GCNSubtarget>();
590 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
591 auto TgtOcc = MFI->getMinAllowedOccupancy();
592 unsigned DynamicVGPRBlockSize = MFI->getDynamicVGPRBlockSize();
593
594 sortRegionsByPressure(TargetOcc: TgtOcc);
595 auto Occ =
596 Regions.front()->MaxPressure.getOccupancy(ST, DynamicVGPRBlockSize);
597
598 bool IsReentry = false;
599 if (TryMaximizeOccupancy && Occ < TgtOcc) {
600 Occ = tryMaximizeOccupancy(TargetOcc: TgtOcc);
601 IsReentry = true;
602 }
603
604 TgtOcc = std::min(a: Occ, b: TgtOcc);
605 LLVM_DEBUG(dbgs() << "Scheduling using default scheduler, "
606 "target occupancy = "
607 << TgtOcc << '\n');
608
609 unsigned FinalOccupancy = std::min(a: Occ, b: MFI->getOccupancy());
610 for (auto *R : Regions) {
611 BuildDAG DAG(*R, *this, IsReentry);
612 const auto ILPSchedule = makeGCNILPScheduler(BotRoots: DAG.getBottomRoots(), DAG: *this);
613
614 const auto RP = getSchedulePressure(R: *R, Schedule: ILPSchedule);
615 LLVM_DEBUG(printSchedRP(dbgs(), R->MaxPressure, RP));
616
617 if (RP.getOccupancy(ST, DynamicVGPRBlockSize) < TgtOcc) {
618 LLVM_DEBUG(dbgs() << "Didn't fit into target occupancy O" << TgtOcc);
619 if (R->BestSchedule.get() && R->BestSchedule->MaxPressure.getOccupancy(
620 ST, DynamicVGPRBlockSize) >= TgtOcc) {
621 LLVM_DEBUG(dbgs() << ", scheduling minimal register\n");
622 scheduleBest(R&: *R);
623 } else {
624 restoreRegionLivenessFlags(R: *R);
625 }
626 } else {
627 scheduleRegion(R&: *R, Schedule: ILPSchedule, MaxRP: RP);
628 LLVM_DEBUG(printSchedResult(dbgs(), R, RP));
629 FinalOccupancy =
630 std::min(a: FinalOccupancy, b: RP.getOccupancy(ST, DynamicVGPRBlockSize));
631 }
632 }
633 MFI->limitOccupancy(Limit: FinalOccupancy);
634}
635