1//===-- GCNSchedStrategy.cpp - GCN Scheduler Strategy ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This contains a MachineSchedStrategy implementation for maximizing wave
11/// occupancy on GCN hardware.
12///
13/// This pass will apply multiple scheduling stages to the same function.
14/// Regions are first recorded in GCNScheduleDAGMILive::schedule. The actual
15/// entry point for the scheduling of those regions is
16/// GCNScheduleDAGMILive::runSchedStages.
17
18/// Generally, the reason for having multiple scheduling stages is to account
19/// for the kernel-wide effect of register usage on occupancy. Usually, only a
20/// few scheduling regions will have register pressure high enough to limit
21/// occupancy for the kernel, so constraints can be relaxed to improve ILP in
22/// other regions.
23///
24//===----------------------------------------------------------------------===//
25
26#include "GCNSchedStrategy.h"
27#include "AMDGPUIGroupLP.h"
28#include "GCNHazardRecognizer.h"
29#include "GCNRegPressure.h"
30#include "GCNRematStrategy.h"
31#include "SIMachineFunctionInfo.h"
32#include "Utils/AMDGPUBaseInfo.h"
33#include "llvm/ADT/BitVector.h"
34#include "llvm/ADT/STLExtras.h"
35#include "llvm/CodeGen/CalcSpillWeights.h"
36#include "llvm/CodeGen/MachineBasicBlock.h"
37#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
38#include "llvm/CodeGen/MachineOperand.h"
39#include "llvm/CodeGen/RegisterPressure.h"
40#include "llvm/MC/MCSchedule.h"
41#include "llvm/MC/TargetRegistry.h"
42#include "llvm/Support/ErrorHandling.h"
43
44#define DEBUG_TYPE "machine-scheduler"
45
46using namespace llvm;
47
48bool VGPRThresholdParser::parse(cl::Option &O, StringRef ArgName, StringRef Arg,
49 unsigned &Value) {
50 if (Arg.getAsInteger(Radix: 0, Result&: Value))
51 return O.error(Message: "'" + Arg + "' value invalid for uint argument!");
52
53 if (Value > 100)
54 return O.error(Message: "'" + Arg + "' value must be in the range [0, 100]!");
55
56 return false;
57}
58
59cl::opt<unsigned, false, VGPRThresholdParser> llvm::VGPRThresholdPercentOpt(
60 "amdgpu-vgpr-threshold-percent", cl::Hidden,
61 cl::desc("Percent of VGPR limits that we should use as RP threshold "
62 "during scheduling. We have two limits relevant to scheduling: "
63 "Critical (avoid decreasing occupancy), Excess (avoid spilling). "
64 "This flag scales both limits back by an equal percent: (0 = use "
65 " default calculation, 1-100 = use percentage), default: 0"),
66 cl::init(Val: 0));
67
68const unsigned ScheduleMetrics::ScaleFactor = 100;
69
70GCNSchedStrategy::GCNSchedStrategy(const MachineSchedContext *C)
71 : GenericScheduler(C),
72 CLOpts(C->MF->getSubtarget<GCNSubtarget>().getCLOpts()),
73 TargetOccupancy(0), MF(nullptr), DownwardTracker(*C->LIS),
74 UpwardTracker(*C->LIS), HasHighPressure(false) {
75 if (CLOpts.use_amdgpu_trackers != BoolOrDefault::Default)
76 GCNTrackersOverride = CLOpts.use_amdgpu_trackers == BoolOrDefault::True;
77 VGPRThresholdPercent = VGPRThresholdPercentOpt;
78}
79
80void GCNSchedStrategy::initialize(ScheduleDAGMI *DAG) {
81 GenericScheduler::initialize(dag: DAG);
82
83 MF = &DAG->MF;
84
85 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
86
87 SGPRExcessLimit =
88 Context->RegClassInfo->getNumAllocatableRegs(RC: &AMDGPU::SGPR_32RegClass);
89 VGPRExcessLimit =
90 Context->RegClassInfo->getNumAllocatableRegs(RC: &AMDGPU::VGPR_32RegClass);
91 AGPRExcessLimit =
92 Context->RegClassInfo->getNumAllocatableRegs(RC: &AMDGPU::AGPR_32RegClass);
93
94 ArchVGPRBankLimit = VGPRExcessLimit;
95 AGPRBankLimit = AGPRExcessLimit;
96 UnifiedVGPRExcessLimit = ST.hasGFX90AInsts() ? ST.getMaxNumVGPRs(MF: *MF) : 0;
97
98 SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
99 // Set the initial TargetOccupnacy to the maximum occupancy that we can
100 // achieve for this function. This effectively sets a lower bound on the
101 // 'Critical' register limits in the scheduler.
102 // Allow for lower occupancy targets if kernel is wave limited or memory
103 // bound, and using the relaxed occupancy feature.
104 TargetOccupancy = CLOpts.schedule_relaxed_occupancy
105 ? MFI.getMinAllowedOccupancy()
106 : MFI.getOccupancy();
107 SGPRCriticalLimit =
108 std::min(a: ST.getMaxNumSGPRs(WavesPerEU: TargetOccupancy, Addressable: true), b: SGPRExcessLimit);
109
110 if (!KnownExcessRP) {
111 // Preserve the combined occupancy budget before clamping to one bank.
112 UnifiedVGPRCriticalLimit =
113 ST.getMaxNumVGPRs(WavesPerEU: TargetOccupancy, DynamicVGPRBlockSize: MFI.getDynamicVGPRBlockSize());
114 VGPRCriticalLimit = std::min(a: UnifiedVGPRCriticalLimit, b: VGPRExcessLimit);
115 } else {
116 // This is similar to ST.getMaxNumVGPRs(TargetOccupancy) result except
117 // returns a reasonably small number for targets with lots of VGPRs, such
118 // as GFX10 and GFX11.
119 LLVM_DEBUG(dbgs() << "Region is known to spill, use alternative "
120 "VGPRCriticalLimit calculation method.\n");
121 unsigned DynamicVGPRBlockSize = MFI.getDynamicVGPRBlockSize();
122 unsigned Granule =
123 AMDGPU::IsaInfo::getVGPRAllocGranule(STI: ST, DynamicVGPRBlockSize);
124 unsigned Addressable =
125 AMDGPU::IsaInfo::getAddressableNumVGPRs(STI: ST, DynamicVGPRBlockSize);
126 unsigned VGPRBudget = alignDown(Value: Addressable / TargetOccupancy, Align: Granule);
127 VGPRBudget = std::max(a: VGPRBudget, b: Granule);
128 UnifiedVGPRCriticalLimit = VGPRBudget;
129 VGPRCriticalLimit = std::min(a: VGPRBudget, b: VGPRExcessLimit);
130 }
131
132 // Reuse VGPR critical limit
133 AGPRCriticalLimit = std::min(a: VGPRCriticalLimit, b: AGPRExcessLimit);
134
135 // Apply VGPR excess threshold percentage if specified.
136 if (VGPRThresholdPercent > 0) {
137 [[maybe_unused]] unsigned OriginalVGPRExcessLimit = VGPRExcessLimit;
138 [[maybe_unused]] unsigned OriginalVGPRCriticalLimit = VGPRCriticalLimit;
139 VGPRExcessLimit = (VGPRThresholdPercent * VGPRExcessLimit + 99) / 100;
140 VGPRCriticalLimit = (VGPRThresholdPercent * VGPRCriticalLimit + 99) / 100;
141 UnifiedVGPRCriticalLimit =
142 (VGPRThresholdPercent * UnifiedVGPRCriticalLimit + 99) / 100;
143 LLVM_DEBUG(dbgs() << "Applied VGPR excess threshold "
144 << VGPRThresholdPercent << "%, VGPRExcessLimit: "
145 << OriginalVGPRExcessLimit << " -> " << VGPRExcessLimit
146 << ". VGPRCriticalLimit: " << OriginalVGPRCriticalLimit
147 << " -> " << VGPRCriticalLimit << '\n');
148 } else {
149 VGPRExcessLimit -= std::min(a: VGPRLimitBias + ErrorMargin, b: VGPRExcessLimit);
150 VGPRCriticalLimit -=
151 std::min(a: VGPRLimitBias + ErrorMargin, b: VGPRCriticalLimit);
152 UnifiedVGPRCriticalLimit -=
153 std::min(a: VGPRLimitBias + ErrorMargin, b: UnifiedVGPRCriticalLimit);
154 }
155
156 // Subtract error margin and bias from register limits and avoid overflow.
157 SGPRCriticalLimit -= std::min(a: SGPRLimitBias + ErrorMargin, b: SGPRCriticalLimit);
158 SGPRExcessLimit -= std::min(a: SGPRLimitBias + ErrorMargin, b: SGPRExcessLimit);
159
160 AGPRExcessLimit -= std::min(a: VGPRLimitBias + ErrorMargin, b: AGPRExcessLimit);
161 AGPRCriticalLimit -= std::min(a: VGPRLimitBias + ErrorMargin, b: AGPRCriticalLimit);
162
163 LLVM_DEBUG(dbgs() << "VGPRCriticalLimit = " << VGPRCriticalLimit
164 << ", UnifiedVGPRCriticalLimit = "
165 << UnifiedVGPRCriticalLimit
166 << ", VGPRExcessLimit = " << VGPRExcessLimit
167 << ", AGPRCriticalLimit = " << AGPRCriticalLimit
168 << ", AGPRExcessLimit = " << AGPRExcessLimit
169 << ", SGPRCriticalLimit = " << SGPRCriticalLimit
170 << ", SGPRExcessLimit = " << SGPRExcessLimit << "\n\n");
171}
172
173/// Checks whether \p SU can use the cached DAG pressure diffs to compute the
174/// current register pressure.
175///
176/// This works for the common case, but it has a few exceptions that have been
177/// observed through trial and error:
178/// - Explicit physical register operands
179/// - Subregister definitions
180///
181/// In both of those cases, PressureDiff doesn't represent the actual pressure,
182/// and querying LiveIntervals through the RegPressureTracker is needed to get
183/// an accurate value.
184///
185/// We should eventually only use PressureDiff for maximum performance, but this
186/// already allows 80% of SUs to take the fast path without changing scheduling
187/// at all. Further changes would either change scheduling, or require a lot
188/// more logic to recover an accurate pressure estimate from the PressureDiffs.
189static bool canUsePressureDiffs(const SUnit &SU) {
190 if (!SU.isInstr())
191 return false;
192
193 // Cannot use pressure diffs for subregister defs or with physregs, it's
194 // imprecise in both cases. For a bundle, check the instructions inside it:
195 // the BUNDLE header only has implicit operands.
196 for (const auto &Op : const_mi_bundle_ops(MI: *SU.getInstr())) {
197 if (!Op.isReg() || Op.isImplicit())
198 continue;
199 if (Op.getReg().isPhysical() ||
200 (Op.isDef() && Op.getSubReg() != AMDGPU::NoSubRegister))
201 return false;
202 }
203 return true;
204}
205
206std::pair<unsigned, unsigned>
207GCNSchedStrategy::getBankedPressure(const GCNRegPressure &RP) const {
208 if (!CLOpts.avgpr_banking)
209 return {RP.getArchVGPRNum(), RP.getAGPRNum()};
210 return RP.getBankedVGPRNum(ArchLimit: ArchVGPRBankLimit, AccLimit: AGPRBankLimit);
211}
212
213void GCNSchedStrategy::getRegisterPressures(
214 bool AtTop, const RegPressureTracker &RPTracker, SUnit *SU,
215 std::vector<unsigned> &Pressure, std::vector<unsigned> &MaxPressure,
216 const GCNDownwardRPTrackerBase &DownwardTracker,
217 const GCNUpwardRPTracker &UpwardTracker, ScheduleDAGMI *DAG,
218 const SIRegisterInfo *SRI) {
219 // getDownwardPressure() and getUpwardPressure() make temporary changes to
220 // the tracker, so we need to pass those function a non-const copy.
221 RegPressureTracker &TempTracker = const_cast<RegPressureTracker &>(RPTracker);
222 if (!useGCNTrackers()) {
223 AtTop
224 ? TempTracker.getDownwardPressure(MI: SU->getInstr(), PressureResult&: Pressure, MaxPressureResult&: MaxPressure)
225 : TempTracker.getUpwardPressure(MI: SU->getInstr(), PressureResult&: Pressure, MaxPressureResult&: MaxPressure);
226
227 return;
228 }
229
230 // GCNTrackers
231 Pressure.resize(new_size: 4, x: 0);
232 MachineInstr *MI = SU->getInstr();
233 GCNRegPressure NewPressure;
234 if (AtTop) {
235 NewPressure = DownwardTracker.bumpDownwardPressure(MI, TRI: SRI);
236 } else {
237 GCNUpwardRPTracker TempUpwardTracker(UpwardTracker);
238 TempUpwardTracker.recede(MI: *MI);
239 NewPressure = TempUpwardTracker.getPressure();
240 }
241 auto [ArchNum, AccNum] = getBankedPressure(RP: NewPressure);
242 Pressure[AMDGPU::RegisterPressureSets::SReg_32] = NewPressure.getSGPRNum();
243 Pressure[AMDGPU::RegisterPressureSets::VGPR_32] = ArchNum;
244 Pressure[AMDGPU::RegisterPressureSets::AGPR_32] = AccNum;
245 CandUnifiedVGPRPressure =
246 GCNRegPressure::getBankedUnifiedVGPRNum(NumArchVGPRs: ArchNum, NumAGPRs: AccNum);
247}
248
249void GCNSchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU,
250 bool AtTop,
251 const RegPressureTracker &RPTracker,
252 const SIRegisterInfo *SRI,
253 unsigned SGPRPressure,
254 unsigned VGPRPressure,
255 unsigned AGPRPressure, bool IsBottomUp) {
256 Cand.SU = SU;
257 Cand.AtTop = AtTop;
258
259 if (!DAG->isTrackingPressure())
260 return;
261
262 Pressure.clear();
263 MaxPressure.clear();
264
265 // We try to use the cached PressureDiffs in the ScheduleDAG whenever
266 // possible over querying the RegPressureTracker.
267 //
268 // RegPressureTracker will make a lot of LIS queries which are very
269 // expensive, it is considered a slow function in this context.
270 //
271 // PressureDiffs are precomputed and cached, and getPressureDiff is just a
272 // trivial lookup into an array. It is pretty much free.
273 //
274 // In EXPENSIVE_CHECKS, we always query RPTracker to verify the results of
275 // PressureDiffs.
276 if (AtTop || !canUsePressureDiffs(SU: *SU) || useGCNTrackers()) {
277 getRegisterPressures(AtTop, RPTracker, SU, Pressure, MaxPressure,
278 DownwardTracker, UpwardTracker, DAG, SRI);
279 } else {
280 // Reserve 4 slots.
281 Pressure.resize(new_size: 4, x: 0);
282 Pressure[AMDGPU::RegisterPressureSets::SReg_32] = SGPRPressure;
283 Pressure[AMDGPU::RegisterPressureSets::VGPR_32] = VGPRPressure;
284 Pressure[AMDGPU::RegisterPressureSets::AGPR_32] = AGPRPressure;
285
286 for (const auto &Diff : DAG->getPressureDiff(SU)) {
287 if (!Diff.isValid())
288 continue;
289 // PressureDiffs is always bottom-up so if we're working top-down we need
290 // to invert its sign.
291 Pressure[Diff.getPSet()] +=
292 (IsBottomUp ? Diff.getUnitInc() : -Diff.getUnitInc());
293 }
294
295#ifdef EXPENSIVE_CHECKS
296 std::vector<unsigned> CheckPressure, CheckMaxPressure;
297 getRegisterPressures(AtTop, RPTracker, SU, CheckPressure, CheckMaxPressure,
298 DownwardTracker, UpwardTracker, DAG, SRI);
299 if (Pressure[AMDGPU::RegisterPressureSets::SReg_32] !=
300 CheckPressure[AMDGPU::RegisterPressureSets::SReg_32] ||
301 Pressure[AMDGPU::RegisterPressureSets::VGPR_32] !=
302 CheckPressure[AMDGPU::RegisterPressureSets::VGPR_32] ||
303 Pressure[AMDGPU::RegisterPressureSets::AGPR_32] !=
304 CheckPressure[AMDGPU::RegisterPressureSets::AGPR_32]) {
305 errs() << "Register Pressure is inaccurate when calculated through "
306 "PressureDiff\n"
307 << "SGPR got " << Pressure[AMDGPU::RegisterPressureSets::SReg_32]
308 << ", expected "
309 << CheckPressure[AMDGPU::RegisterPressureSets::SReg_32] << "\n"
310 << "VGPR got " << Pressure[AMDGPU::RegisterPressureSets::VGPR_32]
311 << ", expected "
312 << CheckPressure[AMDGPU::RegisterPressureSets::VGPR_32] << "\n"
313 << "AGPR got " << Pressure[AMDGPU::RegisterPressureSets::AGPR_32]
314 << ", expected "
315 << CheckPressure[AMDGPU::RegisterPressureSets::AGPR_32] << "\n";
316 report_fatal_error("inaccurate register pressure calculation");
317 }
318#endif
319 }
320
321 unsigned NewAGPRPressure = Pressure[AMDGPU::RegisterPressureSets::AGPR_32];
322 unsigned NewSGPRPressure = Pressure[AMDGPU::RegisterPressureSets::SReg_32];
323 unsigned NewVGPRPressure = Pressure[AMDGPU::RegisterPressureSets::VGPR_32];
324
325 // If two instructions increase the pressure of different register sets
326 // by the same amount, the generic scheduler will prefer to schedule the
327 // instruction that increases the set with the least amount of registers,
328 // which in our case would be SGPRs. This is rarely what we want, so
329 // when we report excess/critical register pressure, we do it either
330 // only for VGPRs, AGPRs or SGPRs. Priority: VGPR > AGPR > SGPR.
331
332 // FIXME: Better heuristics to determine whether to prefer SGPRs or VGPRs.
333 const unsigned MaxVGPRPressureInc = 16;
334 bool ShouldTrackVGPRs = VGPRPressure + MaxVGPRPressureInc >= VGPRExcessLimit;
335 bool ShouldTrackAGPRs = AGPRExcessLimit > 0 && !ShouldTrackVGPRs &&
336 AGPRPressure + MaxVGPRPressureInc >= AGPRExcessLimit;
337 bool ShouldTrackSGPRs =
338 !ShouldTrackVGPRs && !ShouldTrackAGPRs && SGPRPressure >= SGPRExcessLimit;
339 // FIXME: We have to enter REG-EXCESS before we reach the actual threshold
340 // to increase the likelihood we don't go over the limits. We should improve
341 // the analysis to look through dependencies to find the path with the least
342 // register pressure.
343 // We only need to update the RPDelta for instructions that increase register
344 // pressure. Instructions that decrease or keep reg pressure the same will be
345 // marked as RegExcess in tryCandidate() when they are compared with
346 // instructions that increase the register pressure.
347 if (ShouldTrackVGPRs && NewVGPRPressure >= VGPRExcessLimit) {
348 HasHighPressure = true;
349 Cand.RPDelta.Excess = PressureChange(AMDGPU::RegisterPressureSets::VGPR_32);
350 Cand.RPDelta.Excess.setUnitInc(NewVGPRPressure - VGPRExcessLimit);
351 }
352
353 if (ShouldTrackAGPRs && NewAGPRPressure >= AGPRExcessLimit) {
354 HasHighPressure = true;
355 Cand.RPDelta.Excess = PressureChange(AMDGPU::RegisterPressureSets::AGPR_32);
356 Cand.RPDelta.Excess.setUnitInc(NewAGPRPressure - AGPRExcessLimit);
357 }
358
359 if (ShouldTrackSGPRs && NewSGPRPressure >= SGPRExcessLimit) {
360 HasHighPressure = true;
361 Cand.RPDelta.Excess = PressureChange(AMDGPU::RegisterPressureSets::SReg_32);
362 Cand.RPDelta.Excess.setUnitInc(NewSGPRPressure - SGPRExcessLimit);
363 }
364
365 // Because of alignment padding, the combined file can overflow when both
366 // banks are within limits. This check catches overflow that the others above
367 // miss.
368 if (CLOpts.avgpr_banking && UnifiedVGPRExcessLimit &&
369 CandUnifiedVGPRPressure >= UnifiedVGPRExcessLimit) {
370 HasHighPressure = true;
371 int UnifiedExcess = CandUnifiedVGPRPressure - UnifiedVGPRExcessLimit;
372 PressureChange &Excess = Cand.RPDelta.Excess;
373 bool HasBankExcess =
374 Excess.isValid() &&
375 Excess.getPSet() != AMDGPU::RegisterPressureSets::SReg_32;
376 if (!HasBankExcess || UnifiedExcess > Excess.getUnitInc()) {
377 Excess = PressureChange(AMDGPU::RegisterPressureSets::VGPR_32);
378 Excess.setUnitInc(UnifiedExcess);
379 }
380 }
381
382 // Register pressure is considered 'CRITICAL' if it is approaching a value
383 // that would reduce the wave occupancy for the execution unit. When
384 // register pressure is 'CRITICAL', increasing SGPR, VGPR, and AGPR
385 // pressure all has the same cost, so we pick the most critical type.
386
387 // Unified targets allocate both vector banks from the same occupancy budget.
388 // Keep the bank pressures above for excess checks, and do not align VGPRs
389 // when there are no AGPRs, matching GCNRegPressure::getVGPRNum().
390 bool UnifiedVGPRFile = MF->getSubtarget<GCNSubtarget>().hasGFX90AInsts();
391 unsigned VectorCriticalLimit = VGPRCriticalLimit;
392 if (UnifiedVGPRFile) {
393 if (NewAGPRPressure)
394 NewVGPRPressure = GCNRegPressure::getUnifiedVGPRNum(NumArchVGPRs: NewVGPRPressure,
395 NumAGPRs: NewAGPRPressure, NumAVGPRs: 0);
396 VectorCriticalLimit = UnifiedVGPRCriticalLimit;
397 }
398
399 int SGPRDelta = NewSGPRPressure - SGPRCriticalLimit;
400 int VGPRDelta = NewVGPRPressure - VectorCriticalLimit;
401 int AGPRDelta = !UnifiedVGPRFile && AGPRExcessLimit > 0
402 ? NewAGPRPressure - AGPRCriticalLimit
403 : std::numeric_limits<int>::min();
404
405 if (SGPRDelta >= 0 || VGPRDelta >= 0 || AGPRDelta >= 0) {
406 HasHighPressure = true;
407 // Pick the most critical type.
408 if (VGPRDelta >= SGPRDelta && VGPRDelta >= AGPRDelta) {
409 Cand.RPDelta.CriticalMax =
410 PressureChange(AMDGPU::RegisterPressureSets::VGPR_32);
411 Cand.RPDelta.CriticalMax.setUnitInc(VGPRDelta);
412 } else if (AGPRDelta >= SGPRDelta) {
413 Cand.RPDelta.CriticalMax =
414 PressureChange(AMDGPU::RegisterPressureSets::AGPR_32);
415 Cand.RPDelta.CriticalMax.setUnitInc(AGPRDelta);
416 } else {
417 Cand.RPDelta.CriticalMax =
418 PressureChange(AMDGPU::RegisterPressureSets::SReg_32);
419 Cand.RPDelta.CriticalMax.setUnitInc(SGPRDelta);
420 }
421 }
422}
423
424static bool shouldCheckPending(unsigned PendingQueueLimit, SchedBoundary &Zone,
425 const TargetSchedModel *SchedModel) {
426 bool HasBufferedModel =
427 SchedModel->hasInstrSchedModel() && SchedModel->getMicroOpBufferSize();
428 unsigned Combined = Zone.Available.size() + Zone.Pending.size();
429 return Combined <= PendingQueueLimit && HasBufferedModel;
430}
431
432static SUnit *pickOnlyChoice(unsigned PendingQueueLimit, SchedBoundary &Zone,
433 const TargetSchedModel *SchedModel) {
434 // pickOnlyChoice() releases pending instructions and checks for new hazards.
435 SUnit *OnlyChoice = Zone.pickOnlyChoice();
436 if (!shouldCheckPending(PendingQueueLimit, Zone, SchedModel) ||
437 Zone.Pending.empty())
438 return OnlyChoice;
439
440 return nullptr;
441}
442
443void GCNSchedStrategy::printCandidateDecision(const SchedCandidate &Current,
444 const SchedCandidate &Preferred) {
445 LLVM_DEBUG({
446 dbgs() << "Prefer:\t\t";
447 DAG->dumpNode(*Preferred.SU);
448
449 if (Current.SU) {
450 dbgs() << "Not:\t";
451 DAG->dumpNode(*Current.SU);
452 }
453
454 dbgs() << "Reason:\t\t";
455 traceCandidate(Preferred);
456 });
457}
458
459// This function is mostly cut and pasted from
460// GenericScheduler::pickNodeFromQueue()
461void GCNSchedStrategy::pickNodeFromQueue(SchedBoundary &Zone,
462 const CandPolicy &ZonePolicy,
463 const RegPressureTracker &RPTracker,
464 SchedCandidate &Cand, bool &IsPending,
465 bool IsBottomUp) {
466 const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo *>(TRI);
467 ArrayRef<unsigned> Pressure = RPTracker.getRegSetPressureAtPos();
468 unsigned SGPRPressure = 0;
469 unsigned VGPRPressure = 0;
470 unsigned AGPRPressure = 0;
471 IsPending = false;
472 if (DAG->isTrackingPressure()) {
473 if (!useGCNTrackers()) {
474 SGPRPressure = Pressure[AMDGPU::RegisterPressureSets::SReg_32];
475 VGPRPressure = Pressure[AMDGPU::RegisterPressureSets::VGPR_32];
476 AGPRPressure = Pressure[AMDGPU::RegisterPressureSets::AGPR_32];
477 } else {
478 GCNRPTracker *T = IsBottomUp
479 ? static_cast<GCNRPTracker *>(&UpwardTracker)
480 : static_cast<GCNRPTracker *>(&DownwardTracker);
481 SGPRPressure = T->getPressure().getSGPRNum();
482 std::tie(args&: VGPRPressure, args&: AGPRPressure) =
483 getBankedPressure(RP: T->getPressure());
484 }
485 }
486 LLVM_DEBUG(dbgs() << "Available Q:\n");
487 ReadyQueue &AQ = Zone.Available;
488 for (SUnit *SU : AQ) {
489
490 SchedCandidate TryCand(ZonePolicy);
491 initCandidate(Cand&: TryCand, SU, AtTop: Zone.isTop(), RPTracker, SRI, SGPRPressure,
492 VGPRPressure, AGPRPressure, IsBottomUp);
493 // Pass SchedBoundary only when comparing nodes from the same boundary.
494 SchedBoundary *ZoneArg = Cand.AtTop == TryCand.AtTop ? &Zone : nullptr;
495 tryCandidate(Cand, TryCand, Zone: ZoneArg);
496 if (TryCand.Reason != NoCand) {
497 // Initialize resource delta if needed in case future heuristics query it.
498 if (TryCand.ResDelta == SchedResourceDelta())
499 TryCand.initResourceDelta(DAG: Zone.DAG, SchedModel);
500 LLVM_DEBUG(printCandidateDecision(Cand, TryCand));
501 Cand.setBest(TryCand);
502 } else {
503 printCandidateDecision(Current: TryCand, Preferred: Cand);
504 }
505 }
506
507 if (!shouldCheckPending(PendingQueueLimit: CLOpts.scheduler_pending_queue_limit, Zone,
508 SchedModel))
509 return;
510
511 LLVM_DEBUG(dbgs() << "Pending Q:\n");
512 ReadyQueue &PQ = Zone.Pending;
513 for (SUnit *SU : PQ) {
514
515 SchedCandidate TryCand(ZonePolicy);
516 initCandidate(Cand&: TryCand, SU, AtTop: Zone.isTop(), RPTracker, SRI, SGPRPressure,
517 VGPRPressure, AGPRPressure, IsBottomUp);
518 // Pass SchedBoundary only when comparing nodes from the same boundary.
519 SchedBoundary *ZoneArg = Cand.AtTop == TryCand.AtTop ? &Zone : nullptr;
520 tryPendingCandidate(Cand, TryCand, Zone: ZoneArg);
521 if (TryCand.Reason != NoCand) {
522 // Initialize resource delta if needed in case future heuristics query it.
523 if (TryCand.ResDelta == SchedResourceDelta())
524 TryCand.initResourceDelta(DAG: Zone.DAG, SchedModel);
525 LLVM_DEBUG(printCandidateDecision(Cand, TryCand));
526 IsPending = true;
527 Cand.setBest(TryCand);
528 } else {
529 printCandidateDecision(Current: TryCand, Preferred: Cand);
530 }
531 }
532}
533
534// This function is mostly cut and pasted from
535// GenericScheduler::pickNodeBidirectional()
536SUnit *GCNSchedStrategy::pickNodeBidirectional(bool &IsTopNode,
537 bool &PickedPending) {
538 // Schedule as far as possible in the direction of no choice. This is most
539 // efficient, but also provides the best heuristics for CriticalPSets.
540 if (SUnit *SU = pickOnlyChoice(PendingQueueLimit: CLOpts.scheduler_pending_queue_limit, Zone&: Bot,
541 SchedModel)) {
542 IsTopNode = false;
543 return SU;
544 }
545 if (SUnit *SU = pickOnlyChoice(PendingQueueLimit: CLOpts.scheduler_pending_queue_limit, Zone&: Top,
546 SchedModel)) {
547 IsTopNode = true;
548 return SU;
549 }
550 // Set the bottom-up policy based on the state of the current bottom zone
551 // and the instructions outside the zone, including the top zone.
552 CandPolicy BotPolicy;
553 setPolicy(Policy&: BotPolicy, /*IsPostRA=*/false, CurrZone&: Bot, OtherZone: &Top);
554 // Set the top-down policy based on the state of the current top zone and
555 // the instructions outside the zone, including the bottom zone.
556 CandPolicy TopPolicy;
557 setPolicy(Policy&: TopPolicy, /*IsPostRA=*/false, CurrZone&: Top, OtherZone: &Bot);
558
559 bool BotPending = false;
560 // See if BotCand is still valid (because we previously scheduled from Top).
561 LLVM_DEBUG(dbgs() << "Picking from Bot:\n");
562 if (!BotCand.isValid() || BotCand.SU->isScheduled ||
563 BotCand.Policy != BotPolicy) {
564 BotCand.reset(NewPolicy: CandPolicy());
565 pickNodeFromQueue(Zone&: Bot, ZonePolicy: BotPolicy, RPTracker: DAG->getBotRPTracker(), Cand&: BotCand,
566 IsPending&: BotPending,
567 /*IsBottomUp=*/true);
568 assert(BotCand.Reason != NoCand && "failed to find the first candidate");
569 } else {
570 LLVM_DEBUG(traceCandidate(BotCand));
571#ifndef NDEBUG
572 if (shouldVerifyScheduling()) {
573 SchedCandidate TCand;
574 TCand.reset(CandPolicy());
575 pickNodeFromQueue(Bot, BotPolicy, DAG->getBotRPTracker(), TCand,
576 BotPending,
577 /*IsBottomUp=*/true);
578 assert(TCand.SU == BotCand.SU &&
579 "Last pick result should correspond to re-picking right now");
580 }
581#endif
582 }
583
584 bool TopPending = false;
585 // Check if the top Q has a better candidate.
586 LLVM_DEBUG(dbgs() << "Picking from Top:\n");
587 if (!TopCand.isValid() || TopCand.SU->isScheduled ||
588 TopCand.Policy != TopPolicy) {
589 TopCand.reset(NewPolicy: CandPolicy());
590 pickNodeFromQueue(Zone&: Top, ZonePolicy: TopPolicy, RPTracker: DAG->getTopRPTracker(), Cand&: TopCand,
591 IsPending&: TopPending,
592 /*IsBottomUp=*/false);
593 assert(TopCand.Reason != NoCand && "failed to find the first candidate");
594 } else {
595 LLVM_DEBUG(traceCandidate(TopCand));
596#ifndef NDEBUG
597 if (shouldVerifyScheduling()) {
598 SchedCandidate TCand;
599 TCand.reset(CandPolicy());
600 pickNodeFromQueue(Top, TopPolicy, DAG->getTopRPTracker(), TCand,
601 TopPending,
602 /*IsBottomUp=*/false);
603 assert(TCand.SU == TopCand.SU &&
604 "Last pick result should correspond to re-picking right now");
605 }
606#endif
607 }
608
609 // Pick best from BotCand and TopCand.
610 LLVM_DEBUG(dbgs() << "Top Cand: "; traceCandidate(TopCand);
611 dbgs() << "Bot Cand: "; traceCandidate(BotCand););
612 SchedCandidate Cand = BotPending ? TopCand : BotCand;
613 SchedCandidate TryCand = BotPending ? BotCand : TopCand;
614 PickedPending = BotPending && TopPending;
615
616 TryCand.Reason = NoCand;
617 if (BotPending || TopPending) {
618 PickedPending |= tryPendingCandidate(Cand, TryCand&: TopCand, Zone: nullptr);
619 } else {
620 tryCandidate(Cand, TryCand, Zone: nullptr);
621 }
622
623 if (TryCand.Reason != NoCand) {
624 Cand.setBest(TryCand);
625 }
626
627 LLVM_DEBUG(dbgs() << "Picking: "; traceCandidate(Cand););
628
629 IsTopNode = Cand.AtTop;
630 return Cand.SU;
631}
632
633// This function is mostly cut and pasted from
634// GenericScheduler::pickNode()
635SUnit *GCNSchedStrategy::pickNode(bool &IsTopNode) {
636 if (DAG->top() == DAG->bottom()) {
637 assert(Top.Available.empty() && Top.Pending.empty() &&
638 Bot.Available.empty() && Bot.Pending.empty() && "ReadyQ garbage");
639 return nullptr;
640 }
641 bool PickedPending;
642 SUnit *SU;
643 do {
644 PickedPending = false;
645 if (RegionPolicy.OnlyTopDown) {
646 SU =
647 pickOnlyChoice(PendingQueueLimit: CLOpts.scheduler_pending_queue_limit, Zone&: Top, SchedModel);
648 if (!SU) {
649 CandPolicy NoPolicy;
650 TopCand.reset(NewPolicy: NoPolicy);
651 pickNodeFromQueue(Zone&: Top, ZonePolicy: NoPolicy, RPTracker: DAG->getTopRPTracker(), Cand&: TopCand,
652 IsPending&: PickedPending,
653 /*IsBottomUp=*/false);
654 assert(TopCand.Reason != NoCand && "failed to find a candidate");
655 SU = TopCand.SU;
656 }
657 IsTopNode = true;
658 } else if (RegionPolicy.OnlyBottomUp) {
659 SU =
660 pickOnlyChoice(PendingQueueLimit: CLOpts.scheduler_pending_queue_limit, Zone&: Bot, SchedModel);
661 if (!SU) {
662 CandPolicy NoPolicy;
663 BotCand.reset(NewPolicy: NoPolicy);
664 pickNodeFromQueue(Zone&: Bot, ZonePolicy: NoPolicy, RPTracker: DAG->getBotRPTracker(), Cand&: BotCand,
665 IsPending&: PickedPending,
666 /*IsBottomUp=*/true);
667 assert(BotCand.Reason != NoCand && "failed to find a candidate");
668 SU = BotCand.SU;
669 }
670 IsTopNode = false;
671 } else {
672 SU = pickNodeBidirectional(IsTopNode, PickedPending);
673 }
674 } while (SU->isScheduled);
675
676 if (PickedPending) {
677 unsigned ReadyCycle = IsTopNode ? SU->TopReadyCycle : SU->BotReadyCycle;
678 SchedBoundary &Zone = IsTopNode ? Top : Bot;
679 unsigned CurrentCycle = Zone.getCurrCycle();
680 if (ReadyCycle > CurrentCycle)
681 Zone.bumpCycle(NextCycle: ReadyCycle);
682
683 // FIXME: checkHazard() doesn't give information about which cycle the
684 // hazard will resolve so just keep bumping the cycle by 1. This could be
685 // made more efficient if checkHazard() returned more details.
686 while (Zone.checkHazard(SU))
687 Zone.bumpCycle(NextCycle: Zone.getCurrCycle() + 1);
688
689 Zone.releasePending();
690 }
691
692 if (SU->isTopReady())
693 Top.removeReady(SU);
694 if (SU->isBottomReady())
695 Bot.removeReady(SU);
696
697 LLVM_DEBUG(dbgs() << "Scheduling " << *SU << " " << *SU->getInstr());
698 return SU;
699}
700
701void GCNSchedStrategy::schedNode(SUnit *SU, bool IsTopNode) {
702 if (useGCNTrackers()) {
703 MachineInstr *MI = SU->getInstr();
704 IsTopNode ? DownwardTracker.advance(MI: *MI) : UpwardTracker.recede(MI: *MI);
705 }
706
707 return GenericScheduler::schedNode(SU, IsTopNode);
708}
709
710GCNSchedStageID GCNSchedStrategy::getCurrentStage() {
711 assert(CurrentStage && CurrentStage != SchedStages.end());
712 return *CurrentStage;
713}
714
715bool GCNSchedStrategy::advanceStage() {
716 assert(CurrentStage != SchedStages.end());
717 if (!CurrentStage)
718 CurrentStage = SchedStages.begin();
719 else
720 CurrentStage++;
721
722 return CurrentStage != SchedStages.end();
723}
724
725bool GCNSchedStrategy::hasNextStage() const {
726 assert(CurrentStage);
727 return std::next(x: CurrentStage) != SchedStages.end();
728}
729
730bool GCNSchedStrategy::tryPendingCandidate(SchedCandidate &Cand,
731 SchedCandidate &TryCand,
732 SchedBoundary *Zone) const {
733 // Initialize the candidate if needed.
734 if (!Cand.isValid()) {
735 TryCand.Reason = NodeOrder;
736 return true;
737 }
738
739 // Bias PhysReg Defs and copies to their uses and defined respectively.
740 if (tryGreater(TryVal: biasPhysReg(SU: TryCand.SU, isTop: TryCand.AtTop),
741 CandVal: biasPhysReg(SU: Cand.SU, isTop: Cand.AtTop), TryCand, Cand, Reason: PhysReg))
742 return TryCand.Reason != NoCand;
743
744 // Avoid exceeding the target's limit.
745 if (DAG->isTrackingPressure() &&
746 tryPressure(TryP: TryCand.RPDelta.Excess, CandP: Cand.RPDelta.Excess, TryCand, Cand,
747 Reason: RegExcess, TRI, MF: DAG->MF))
748 return TryCand.Reason != NoCand;
749
750 // Avoid increasing the max critical pressure in the scheduled region.
751 if (DAG->isTrackingPressure() &&
752 tryPressure(TryP: TryCand.RPDelta.CriticalMax, CandP: Cand.RPDelta.CriticalMax,
753 TryCand, Cand, Reason: RegCritical, TRI, MF: DAG->MF))
754 return TryCand.Reason != NoCand;
755
756 bool SameBoundary = Zone != nullptr;
757 if (SameBoundary) {
758 TryCand.initResourceDelta(DAG, SchedModel);
759 if (tryLess(TryVal: TryCand.ResDelta.CritResources, CandVal: Cand.ResDelta.CritResources,
760 TryCand, Cand, Reason: ResourceReduce))
761 return TryCand.Reason != NoCand;
762 if (tryGreater(TryVal: TryCand.ResDelta.DemandedResources,
763 CandVal: Cand.ResDelta.DemandedResources, TryCand, Cand,
764 Reason: ResourceDemand))
765 return TryCand.Reason != NoCand;
766 }
767
768 return false;
769}
770
771GCNMaxOccupancySchedStrategy::GCNMaxOccupancySchedStrategy(
772 const MachineSchedContext *C, bool IsLegacyScheduler)
773 : GCNSchedStrategy(C) {
774 SchedStages.push_back(Elt: GCNSchedStageID::OccInitialSchedule);
775 if (CLOpts.rewrite_mfma_form_sched_stage)
776 SchedStages.push_back(Elt: GCNSchedStageID::RewriteMFMAForm);
777 SchedStages.push_back(Elt: GCNSchedStageID::UnclusteredHighRPReschedule);
778 SchedStages.push_back(Elt: GCNSchedStageID::ClusteredLowOccupancyReschedule);
779 SchedStages.push_back(Elt: GCNSchedStageID::PreRARematerialize);
780 if (IsLegacyScheduler)
781 GCNTrackersOverride = std::nullopt;
782}
783
784GCNMaxILPSchedStrategy::GCNMaxILPSchedStrategy(const MachineSchedContext *C)
785 : GCNSchedStrategy(C) {
786 SchedStages.push_back(Elt: GCNSchedStageID::ILPInitialSchedule);
787}
788
789bool GCNMaxILPSchedStrategy::tryCandidate(SchedCandidate &Cand,
790 SchedCandidate &TryCand,
791 SchedBoundary *Zone) const {
792 // Initialize the candidate if needed.
793 if (!Cand.isValid()) {
794 TryCand.Reason = NodeOrder;
795 return true;
796 }
797
798 // Avoid spilling by exceeding the register limit.
799 if (DAG->isTrackingPressure() &&
800 tryPressure(TryP: TryCand.RPDelta.Excess, CandP: Cand.RPDelta.Excess, TryCand, Cand,
801 Reason: RegExcess, TRI, MF: DAG->MF))
802 return TryCand.Reason != NoCand;
803
804 // Bias PhysReg Defs and copies to their uses and defined respectively.
805 if (tryGreater(TryVal: biasPhysReg(SU: TryCand.SU, isTop: TryCand.AtTop),
806 CandVal: biasPhysReg(SU: Cand.SU, isTop: Cand.AtTop), TryCand, Cand, Reason: PhysReg))
807 return TryCand.Reason != NoCand;
808
809 bool SameBoundary = Zone != nullptr;
810 if (SameBoundary) {
811 // Prioritize instructions that read unbuffered resources by stall cycles.
812 if (tryLess(TryVal: Zone->getLatencyStallCycles(SU: TryCand.SU),
813 CandVal: Zone->getLatencyStallCycles(SU: Cand.SU), TryCand, Cand, Reason: Stall))
814 return TryCand.Reason != NoCand;
815
816 // Avoid critical resource consumption and balance the schedule.
817 TryCand.initResourceDelta(DAG, SchedModel);
818 if (tryLess(TryVal: TryCand.ResDelta.CritResources, CandVal: Cand.ResDelta.CritResources,
819 TryCand, Cand, Reason: ResourceReduce))
820 return TryCand.Reason != NoCand;
821 if (tryGreater(TryVal: TryCand.ResDelta.DemandedResources,
822 CandVal: Cand.ResDelta.DemandedResources, TryCand, Cand,
823 Reason: ResourceDemand))
824 return TryCand.Reason != NoCand;
825
826 // Unconditionally try to reduce latency.
827 if (tryLatency(TryCand, Cand, Zone&: *Zone))
828 return TryCand.Reason != NoCand;
829
830 // Weak edges are for clustering and other constraints.
831 if (tryLess(TryVal: getWeakLeft(SU: TryCand.SU, isTop: TryCand.AtTop),
832 CandVal: getWeakLeft(SU: Cand.SU, isTop: Cand.AtTop), TryCand, Cand, Reason: Weak))
833 return TryCand.Reason != NoCand;
834 }
835
836 // Keep clustered nodes together to encourage downstream peephole
837 // optimizations which may reduce resource requirements.
838 //
839 // This is a best effort to set things up for a post-RA pass. Optimizations
840 // like generating loads of multiple registers should ideally be done within
841 // the scheduler pass by combining the loads during DAG postprocessing.
842 unsigned CandZoneCluster = Cand.AtTop ? TopClusterID : BotClusterID;
843 unsigned TryCandZoneCluster = TryCand.AtTop ? TopClusterID : BotClusterID;
844 bool CandIsClusterSucc =
845 isTheSameCluster(A: CandZoneCluster, B: Cand.SU->ParentClusterIdx);
846 bool TryCandIsClusterSucc =
847 isTheSameCluster(A: TryCandZoneCluster, B: TryCand.SU->ParentClusterIdx);
848 if (tryGreater(TryVal: TryCandIsClusterSucc, CandVal: CandIsClusterSucc, TryCand, Cand,
849 Reason: Cluster))
850 return TryCand.Reason != NoCand;
851
852 // Avoid increasing the max critical pressure in the scheduled region.
853 if (DAG->isTrackingPressure() &&
854 tryPressure(TryP: TryCand.RPDelta.CriticalMax, CandP: Cand.RPDelta.CriticalMax,
855 TryCand, Cand, Reason: RegCritical, TRI, MF: DAG->MF))
856 return TryCand.Reason != NoCand;
857
858 // Avoid increasing the max pressure of the entire region.
859 if (DAG->isTrackingPressure() &&
860 tryPressure(TryP: TryCand.RPDelta.CurrentMax, CandP: Cand.RPDelta.CurrentMax, TryCand,
861 Cand, Reason: RegMax, TRI, MF: DAG->MF))
862 return TryCand.Reason != NoCand;
863
864 if (SameBoundary) {
865 // Fall through to original instruction order.
866 if ((Zone->isTop() && TryCand.SU->NodeNum < Cand.SU->NodeNum) ||
867 (!Zone->isTop() && TryCand.SU->NodeNum > Cand.SU->NodeNum)) {
868 TryCand.Reason = NodeOrder;
869 return true;
870 }
871 }
872 return false;
873}
874
875GCNMaxMemoryClauseSchedStrategy::GCNMaxMemoryClauseSchedStrategy(
876 const MachineSchedContext *C)
877 : GCNSchedStrategy(C) {
878 SchedStages.push_back(Elt: GCNSchedStageID::MemoryClauseInitialSchedule);
879}
880
881/// GCNMaxMemoryClauseSchedStrategy tries best to clause memory instructions as
882/// much as possible. This is achieved by:
883// 1. Prioritize clustered operations before stall latency heuristic.
884// 2. Prioritize long-latency-load before stall latency heuristic.
885///
886/// \param Cand provides the policy and current best candidate.
887/// \param TryCand refers to the next SUnit candidate, otherwise uninitialized.
888/// \param Zone describes the scheduled zone that we are extending, or nullptr
889/// if Cand is from a different zone than TryCand.
890/// \return \c true if TryCand is better than Cand (Reason is NOT NoCand)
891bool GCNMaxMemoryClauseSchedStrategy::tryCandidate(SchedCandidate &Cand,
892 SchedCandidate &TryCand,
893 SchedBoundary *Zone) const {
894 // Initialize the candidate if needed.
895 if (!Cand.isValid()) {
896 TryCand.Reason = NodeOrder;
897 return true;
898 }
899
900 // Bias PhysReg Defs and copies to their uses and defined respectively.
901 if (tryGreater(TryVal: biasPhysReg(SU: TryCand.SU, isTop: TryCand.AtTop),
902 CandVal: biasPhysReg(SU: Cand.SU, isTop: Cand.AtTop), TryCand, Cand, Reason: PhysReg))
903 return TryCand.Reason != NoCand;
904
905 if (DAG->isTrackingPressure()) {
906 // Avoid exceeding the target's limit.
907 if (tryPressure(TryP: TryCand.RPDelta.Excess, CandP: Cand.RPDelta.Excess, TryCand, Cand,
908 Reason: RegExcess, TRI, MF: DAG->MF))
909 return TryCand.Reason != NoCand;
910
911 // Avoid increasing the max critical pressure in the scheduled region.
912 if (tryPressure(TryP: TryCand.RPDelta.CriticalMax, CandP: Cand.RPDelta.CriticalMax,
913 TryCand, Cand, Reason: RegCritical, TRI, MF: DAG->MF))
914 return TryCand.Reason != NoCand;
915 }
916
917 // MaxMemoryClause-specific: We prioritize clustered instructions as we would
918 // get more benefit from clausing these memory instructions.
919 unsigned CandZoneCluster = Cand.AtTop ? TopClusterID : BotClusterID;
920 unsigned TryCandZoneCluster = TryCand.AtTop ? TopClusterID : BotClusterID;
921 bool CandIsClusterSucc =
922 isTheSameCluster(A: CandZoneCluster, B: Cand.SU->ParentClusterIdx);
923 bool TryCandIsClusterSucc =
924 isTheSameCluster(A: TryCandZoneCluster, B: TryCand.SU->ParentClusterIdx);
925 if (tryGreater(TryVal: TryCandIsClusterSucc, CandVal: CandIsClusterSucc, TryCand, Cand,
926 Reason: Cluster))
927 return TryCand.Reason != NoCand;
928
929 // We only compare a subset of features when comparing nodes between
930 // Top and Bottom boundary. Some properties are simply incomparable, in many
931 // other instances we should only override the other boundary if something
932 // is a clear good pick on one boundary. Skip heuristics that are more
933 // "tie-breaking" in nature.
934 bool SameBoundary = Zone != nullptr;
935 if (SameBoundary) {
936 // For loops that are acyclic path limited, aggressively schedule for
937 // latency. Within an single cycle, whenever CurrMOps > 0, allow normal
938 // heuristics to take precedence.
939 if (Rem.IsAcyclicLatencyLimited && !Zone->getCurrMOps() &&
940 tryLatency(TryCand, Cand, Zone&: *Zone))
941 return TryCand.Reason != NoCand;
942
943 // MaxMemoryClause-specific: Prioritize long latency memory load
944 // instructions in top-bottom order to hide more latency. The mayLoad check
945 // is used to exclude store-like instructions, which we do not want to
946 // scheduler them too early.
947 bool TryMayLoad =
948 TryCand.SU->isInstr() && TryCand.SU->getInstr()->mayLoad();
949 bool CandMayLoad = Cand.SU->isInstr() && Cand.SU->getInstr()->mayLoad();
950
951 if (TryMayLoad || CandMayLoad) {
952 bool TryLongLatency =
953 TryCand.SU->Latency > 10 * Cand.SU->Latency && TryMayLoad;
954 bool CandLongLatency =
955 10 * TryCand.SU->Latency < Cand.SU->Latency && CandMayLoad;
956
957 if (tryGreater(TryVal: Zone->isTop() ? TryLongLatency : CandLongLatency,
958 CandVal: Zone->isTop() ? CandLongLatency : TryLongLatency, TryCand,
959 Cand, Reason: Stall))
960 return TryCand.Reason != NoCand;
961 }
962 // Prioritize instructions that read unbuffered resources by stall cycles.
963 if (tryLess(TryVal: Zone->getLatencyStallCycles(SU: TryCand.SU),
964 CandVal: Zone->getLatencyStallCycles(SU: Cand.SU), TryCand, Cand, Reason: Stall))
965 return TryCand.Reason != NoCand;
966 }
967
968 if (SameBoundary) {
969 // Weak edges are for clustering and other constraints.
970 if (tryLess(TryVal: getWeakLeft(SU: TryCand.SU, isTop: TryCand.AtTop),
971 CandVal: getWeakLeft(SU: Cand.SU, isTop: Cand.AtTop), TryCand, Cand, Reason: Weak))
972 return TryCand.Reason != NoCand;
973 }
974
975 // Avoid increasing the max pressure of the entire region.
976 if (DAG->isTrackingPressure() &&
977 tryPressure(TryP: TryCand.RPDelta.CurrentMax, CandP: Cand.RPDelta.CurrentMax, TryCand,
978 Cand, Reason: RegMax, TRI, MF: DAG->MF))
979 return TryCand.Reason != NoCand;
980
981 if (SameBoundary) {
982 // Avoid critical resource consumption and balance the schedule.
983 TryCand.initResourceDelta(DAG, SchedModel);
984 if (tryLess(TryVal: TryCand.ResDelta.CritResources, CandVal: Cand.ResDelta.CritResources,
985 TryCand, Cand, Reason: ResourceReduce))
986 return TryCand.Reason != NoCand;
987 if (tryGreater(TryVal: TryCand.ResDelta.DemandedResources,
988 CandVal: Cand.ResDelta.DemandedResources, TryCand, Cand,
989 Reason: ResourceDemand))
990 return TryCand.Reason != NoCand;
991
992 // Avoid serializing long latency dependence chains.
993 // For acyclic path limited loops, latency was already checked above.
994 if (!RegionPolicy.DisableLatencyHeuristic && TryCand.Policy.ReduceLatency &&
995 !Rem.IsAcyclicLatencyLimited && tryLatency(TryCand, Cand, Zone&: *Zone))
996 return TryCand.Reason != NoCand;
997
998 // Fall through to original instruction order.
999 if (Zone->isTop() == (TryCand.SU->NodeNum < Cand.SU->NodeNum)) {
1000 assert(TryCand.SU->NodeNum != Cand.SU->NodeNum);
1001 TryCand.Reason = NodeOrder;
1002 return true;
1003 }
1004 }
1005
1006 return false;
1007}
1008
1009GCNScheduleDAGMILive::GCNScheduleDAGMILive(
1010 MachineSchedContext *C, std::unique_ptr<MachineSchedStrategy> S)
1011 : ScheduleDAGMILive(C, std::move(S)), ST(MF.getSubtarget<GCNSubtarget>()),
1012 MFI(*MF.getInfo<SIMachineFunctionInfo>()),
1013 StartingOccupancy(MFI.getOccupancy()), MinOccupancy(StartingOccupancy),
1014 RegionLiveOuts(this, /*IsLiveOut=*/true) {
1015
1016 // We want regions with a single MI to be scheduled so that we can reason
1017 // about them correctly during scheduling stages that move MIs between regions
1018 // (e.g., rematerialization).
1019 ScheduleSingleMIRegions = true;
1020 LLVM_DEBUG(dbgs() << "Starting occupancy is " << StartingOccupancy << ".\n");
1021 if (ST.getCLOpts().schedule_relaxed_occupancy) {
1022 MinOccupancy = std::min(a: MFI.getMinAllowedOccupancy(), b: StartingOccupancy);
1023 if (MinOccupancy != StartingOccupancy)
1024 LLVM_DEBUG(dbgs() << "Allowing Occupancy drops to " << MinOccupancy
1025 << ".\n");
1026 }
1027}
1028
1029std::unique_ptr<GCNSchedStage>
1030GCNScheduleDAGMILive::createSchedStage(GCNSchedStageID SchedStageID) {
1031 switch (SchedStageID) {
1032 case GCNSchedStageID::OccInitialSchedule:
1033 return std::make_unique<OccInitialScheduleStage>(args&: SchedStageID, args&: *this);
1034 case GCNSchedStageID::RewriteMFMAForm:
1035 return std::make_unique<RewriteMFMAFormStage>(args&: SchedStageID, args&: *this);
1036 case GCNSchedStageID::UnclusteredHighRPReschedule:
1037 return std::make_unique<UnclusteredHighRPStage>(args&: SchedStageID, args&: *this);
1038 case GCNSchedStageID::ClusteredLowOccupancyReschedule:
1039 return std::make_unique<ClusteredLowOccStage>(args&: SchedStageID, args&: *this);
1040 case GCNSchedStageID::PreRARematerialize:
1041 return std::make_unique<PreRARematStage>(args&: SchedStageID, args&: *this);
1042 case GCNSchedStageID::ILPInitialSchedule:
1043 return std::make_unique<ILPInitialScheduleStage>(args&: SchedStageID, args&: *this);
1044 case GCNSchedStageID::MemoryClauseInitialSchedule:
1045 return std::make_unique<MemoryClauseInitialScheduleStage>(args&: SchedStageID,
1046 args&: *this);
1047 case GCNSchedStageID::LiveIntervalRPReschedule:
1048 return std::make_unique<LiveIntervalRPStage>(args&: SchedStageID, args&: *this);
1049 }
1050
1051 llvm_unreachable("Unknown SchedStageID.");
1052}
1053
1054void GCNScheduleDAGMILive::schedule() {
1055 // Collect all scheduling regions. The actual scheduling is performed in
1056 // GCNScheduleDAGMILive::finalizeSchedule.
1057 Regions.push_back(Elt: std::pair(RegionBegin, RegionEnd));
1058}
1059
1060GCNRegPressure GCNScheduleDAGMILive::getRealRegPressure(
1061 unsigned RegionIdx,
1062 std::function<void(const GCNRegPressure &)> PressureCallback) const {
1063 if (Regions[RegionIdx].first == Regions[RegionIdx].second) {
1064 GCNRegPressure RP = llvm::getRegPressure(MRI, LiveRegs: LiveIns[RegionIdx]);
1065 if (PressureCallback)
1066 PressureCallback(RP);
1067 return RP;
1068 }
1069 GCNDownwardRPTracker RPTracker(*LIS);
1070 bool HasInstrs =
1071 RPTracker.reset(MI: *Regions[RegionIdx].first, End: Regions[RegionIdx].second,
1072 LiveRegsCopy: &LiveIns[RegionIdx]);
1073 // reset() does not pass the live-in pressure to the callback.
1074 if (PressureCallback) {
1075 PressureCallback(RPTracker.getPressure());
1076 RPTracker.setPressureCallback(std::move(PressureCallback));
1077 }
1078 if (HasInstrs)
1079 RPTracker.advance(End: Regions[RegionIdx].second);
1080 return RPTracker.moveMaxPressure();
1081}
1082
1083static MachineInstr *getLastMIForRegion(MachineBasicBlock::iterator RegionBegin,
1084 MachineBasicBlock::iterator RegionEnd) {
1085 assert(RegionBegin != RegionEnd && "Region must not be empty");
1086 return &*skipDebugInstructionsBackward(It: std::prev(x: RegionEnd), Begin: RegionBegin);
1087}
1088
1089void GCNScheduleDAGMILive::computeBlockPressure(unsigned RegionIdx,
1090 const MachineBasicBlock *MBB) {
1091 GCNDownwardRPTracker RPTracker(*LIS);
1092
1093 // If the block has the only successor then live-ins of that successor are
1094 // live-outs of the current block. We can reuse calculated live set if the
1095 // successor will be sent to scheduling past current block.
1096
1097 // However, due to the bug in LiveInterval analysis it may happen that two
1098 // predecessors of the same successor block have different lane bitmasks for
1099 // a live-out register. Workaround that by sticking to one-to-one relationship
1100 // i.e. one predecessor with one successor block.
1101 const MachineBasicBlock *OnlySucc = nullptr;
1102 if (MBB->succ_size() == 1) {
1103 auto *Candidate = *MBB->succ_begin();
1104 if (!Candidate->empty() && Candidate->pred_size() == 1) {
1105 SlotIndexes *Ind = LIS->getSlotIndexes();
1106 if (Ind->getMBBStartIdx(mbb: MBB) < Ind->getMBBStartIdx(mbb: Candidate))
1107 OnlySucc = Candidate;
1108 }
1109 }
1110
1111 // Scheduler sends regions from the end of the block upwards.
1112 size_t CurRegion = RegionIdx;
1113 for (size_t E = Regions.size(); CurRegion != E; ++CurRegion)
1114 if (Regions[CurRegion].first->getParent() != MBB)
1115 break;
1116 --CurRegion;
1117
1118 auto I = MBB->begin();
1119 auto LiveInIt = MBBLiveIns.find(Val: MBB);
1120 auto &Rgn = Regions[CurRegion];
1121 auto *NonDbgMI = &*skipDebugInstructionsForward(It: Rgn.first, End: Rgn.second);
1122 if (LiveInIt != MBBLiveIns.end()) {
1123 auto LiveIn = std::move(LiveInIt->second);
1124 RPTracker.reset(MI: *MBB->begin(), End: MBB->end(), LiveRegsCopy: &LiveIn);
1125 MBBLiveIns.erase(I: LiveInIt);
1126 } else {
1127 I = Rgn.first;
1128 auto LRS = BBLiveInMap.lookup(Val: NonDbgMI);
1129#ifdef EXPENSIVE_CHECKS
1130 assert(isEqual(getLiveRegsBefore(*NonDbgMI, *LIS), LRS));
1131#endif
1132 RPTracker.reset(MI: *I, End: I->getParent()->end(), LiveRegsCopy: &LRS);
1133 }
1134
1135 for (;;) {
1136 I = RPTracker.getNext();
1137
1138 if (Regions[CurRegion].first == I || NonDbgMI == I) {
1139 LiveIns[CurRegion] = RPTracker.getLiveRegs();
1140 RPTracker.clearMaxPressure();
1141 }
1142
1143 if (Regions[CurRegion].second == I) {
1144 Pressure[CurRegion] = RPTracker.moveMaxPressure();
1145 if (CurRegion-- == RegionIdx)
1146 break;
1147 auto &Rgn = Regions[CurRegion];
1148 NonDbgMI = &*skipDebugInstructionsForward(It: Rgn.first, End: Rgn.second);
1149 }
1150 RPTracker.advanceBeforeNext();
1151 RPTracker.advanceToNext();
1152 }
1153
1154 if (OnlySucc) {
1155 if (I != MBB->end()) {
1156 RPTracker.advanceBeforeNext();
1157 RPTracker.advanceToNext();
1158 RPTracker.advance(End: MBB->end());
1159 }
1160 MBBLiveIns[OnlySucc] = RPTracker.moveLiveRegs();
1161 }
1162}
1163
1164DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet>
1165GCNScheduleDAGMILive::getRegionLiveInMap() const {
1166 assert(!Regions.empty());
1167 std::vector<MachineInstr *> RegionFirstMIs;
1168 RegionFirstMIs.reserve(n: Regions.size());
1169 for (auto &[RegionBegin, RegionEnd] : reverse(C: Regions))
1170 RegionFirstMIs.push_back(
1171 x: &*skipDebugInstructionsForward(It: RegionBegin, End: RegionEnd));
1172
1173 return getLiveRegMap(R&: RegionFirstMIs, /*After=*/false, LIS&: *LIS);
1174}
1175
1176DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet>
1177GCNScheduleDAGMILive::getRegionLiveOutMap() const {
1178 assert(!Regions.empty());
1179 std::vector<MachineInstr *> RegionLastMIs;
1180 RegionLastMIs.reserve(n: Regions.size());
1181 for (auto &[RegionBegin, RegionEnd] : reverse(C: Regions)) {
1182 // Skip empty regions.
1183 if (RegionBegin == RegionEnd)
1184 continue;
1185 RegionLastMIs.push_back(x: getLastMIForRegion(RegionBegin, RegionEnd));
1186 }
1187 return getLiveRegMap(R&: RegionLastMIs, /*After=*/true, LIS&: *LIS);
1188}
1189
1190void RegionPressureMap::buildLiveRegMap() {
1191 IdxToInstruction.clear();
1192
1193 RegionLiveRegMap =
1194 IsLiveOut ? DAG->getRegionLiveOutMap() : DAG->getRegionLiveInMap();
1195 for (unsigned I = 0; I < DAG->Regions.size(); I++) {
1196 auto &[RegionBegin, RegionEnd] = DAG->Regions[I];
1197 // Skip empty regions.
1198 if (RegionBegin == RegionEnd)
1199 continue;
1200 MachineInstr *RegionKey =
1201 IsLiveOut ? getLastMIForRegion(RegionBegin, RegionEnd) : &*RegionBegin;
1202 IdxToInstruction[I] = RegionKey;
1203 }
1204}
1205
1206void GCNScheduleDAGMILive::finalizeSchedule() {
1207 // Start actual scheduling here. This function is called by the base
1208 // MachineScheduler after all regions have been recorded by
1209 // GCNScheduleDAGMILive::schedule().
1210 LiveIns.resize(N: Regions.size());
1211 Pressure.resize(N: Regions.size());
1212 RegionsWithHighRP.resize(N: static_cast<unsigned>(Regions.size()));
1213 RegionsWithExcessRP.resize(N: static_cast<unsigned>(Regions.size()));
1214 RegionsWithIGLPInstrs.resize(N: static_cast<unsigned>(Regions.size()));
1215 RegionsWithHighRP.reset();
1216 RegionsWithExcessRP.reset();
1217 RegionsWithIGLPInstrs.reset();
1218
1219 runSchedStages();
1220}
1221
1222void GCNScheduleDAGMILive::runSchedStages() {
1223 LLVM_DEBUG(dbgs() << "All regions recorded, starting actual scheduling.\n");
1224
1225 GCNSchedStrategy &S = static_cast<GCNSchedStrategy &>(*SchedImpl);
1226 if (!Regions.empty()) {
1227 BBLiveInMap = getRegionLiveInMap();
1228 if (S.useGCNTrackers())
1229 RegionLiveOuts.buildLiveRegMap();
1230 }
1231
1232#ifndef NDEBUG
1233 if (ST.getCLOpts().print_max_reg_pressure_regusage_before_scheduler) {
1234 dumpMaxRegPressure(MF, GCNRegPressure::VGPR, *LIS, MLI);
1235 dumpMaxRegPressure(MF, GCNRegPressure::SGPR, *LIS, MLI);
1236 LIS->dump();
1237 }
1238#endif
1239
1240 while (S.advanceStage()) {
1241 auto Stage = createSchedStage(SchedStageID: S.getCurrentStage());
1242 if (!Stage->initGCNSchedStage())
1243 continue;
1244
1245 for (auto Region : Regions) {
1246 RegionBegin = Region.first;
1247 RegionEnd = Region.second;
1248 // Setup for scheduling the region and check whether it should be skipped.
1249 if (!Stage->initGCNRegion()) {
1250 Stage->advanceRegion();
1251 exitRegion();
1252 continue;
1253 }
1254
1255 if (S.useGCNTrackers()) {
1256 const unsigned RegionIdx = Stage->getRegionIdx();
1257 S.getDownwardTracker().reset(MRI, LiveRegs: LiveIns[RegionIdx]);
1258 S.getUpwardTracker().reset(
1259 MRI, LiveRegs: RegionLiveOuts.getLiveRegsForRegionIdx(RegionIdx));
1260 }
1261
1262 ScheduleDAGMILive::schedule();
1263 Stage->finalizeGCNRegion();
1264 Stage->advanceRegion();
1265 exitRegion();
1266 }
1267
1268 Stage->finalizeGCNSchedStage();
1269 }
1270
1271#ifndef NDEBUG
1272 if (ST.getCLOpts().print_max_reg_pressure_regusage_after_scheduler) {
1273 dumpMaxRegPressure(MF, GCNRegPressure::VGPR, *LIS, MLI);
1274 dumpMaxRegPressure(MF, GCNRegPressure::SGPR, *LIS, MLI);
1275 LIS->dump();
1276 }
1277#endif
1278}
1279
1280#ifndef NDEBUG
1281raw_ostream &llvm::operator<<(raw_ostream &OS, const GCNSchedStageID &StageID) {
1282 switch (StageID) {
1283 case GCNSchedStageID::OccInitialSchedule:
1284 OS << "Max Occupancy Initial Schedule";
1285 break;
1286 case GCNSchedStageID::RewriteMFMAForm:
1287 OS << "Instruction Rewriting Reschedule";
1288 break;
1289 case GCNSchedStageID::UnclusteredHighRPReschedule:
1290 OS << "Unclustered High Register Pressure Reschedule";
1291 break;
1292 case GCNSchedStageID::ClusteredLowOccupancyReschedule:
1293 OS << "Clustered Low Occupancy Reschedule";
1294 break;
1295 case GCNSchedStageID::PreRARematerialize:
1296 OS << "Pre-RA Rematerialize";
1297 break;
1298 case GCNSchedStageID::ILPInitialSchedule:
1299 OS << "Max ILP Initial Schedule";
1300 break;
1301 case GCNSchedStageID::MemoryClauseInitialSchedule:
1302 OS << "Max memory clause Initial Schedule";
1303 break;
1304 case GCNSchedStageID::LiveIntervalRPReschedule:
1305 OS << "Live Interval RP Reschedule";
1306 break;
1307 }
1308
1309 return OS;
1310}
1311#endif
1312
1313GCNSchedStage::GCNSchedStage(GCNSchedStageID StageID, GCNScheduleDAGMILive &DAG)
1314 : DAG(DAG), S(static_cast<GCNSchedStrategy &>(*DAG.SchedImpl)), MF(DAG.MF),
1315 MFI(DAG.MFI), ST(DAG.ST), StageID(StageID) {}
1316
1317bool GCNSchedStage::initGCNSchedStage() {
1318 if (!DAG.LIS)
1319 return false;
1320
1321 LLVM_DEBUG(dbgs() << "Starting scheduling stage: " << StageID << "\n");
1322 return true;
1323}
1324
1325void RewriteMFMAFormStage::findReachingDefs(
1326 MachineOperand &UseMO, LiveIntervals *LIS,
1327 SmallVectorImpl<SlotIndex> &DefIdxs) {
1328 MachineInstr *UseMI = UseMO.getParent();
1329 LiveInterval &UseLI = LIS->getInterval(Reg: UseMO.getReg());
1330 VNInfo *VNI = UseLI.getVNInfoAt(Idx: LIS->getInstructionIndex(Instr: *UseMI));
1331
1332 // If the def is not a PHI, then it must be the only reaching def.
1333 if (!VNI->isPHIDef()) {
1334 DefIdxs.push_back(Elt: VNI->def);
1335 return;
1336 }
1337
1338 SmallPtrSet<MachineBasicBlock *, 8> Visited = {UseMI->getParent()};
1339 SmallVector<MachineBasicBlock *, 8> Worklist;
1340
1341 // Mark the predecessor blocks for traversal
1342 for (MachineBasicBlock *PredMBB : UseMI->getParent()->predecessors()) {
1343 Worklist.push_back(Elt: PredMBB);
1344 Visited.insert(Ptr: PredMBB);
1345 }
1346
1347 while (!Worklist.empty()) {
1348 MachineBasicBlock *CurrMBB = Worklist.pop_back_val();
1349
1350 SlotIndex CurrMBBEnd = LIS->getMBBEndIdx(mbb: CurrMBB);
1351 VNInfo *VNI = UseLI.getVNInfoAt(Idx: CurrMBBEnd.getPrevSlot());
1352
1353 MachineBasicBlock *DefMBB = LIS->getMBBFromIndex(index: VNI->def);
1354
1355 // If there is a def in this block, then add it to the list. This is the
1356 // reaching def of this path.
1357 if (!VNI->isPHIDef()) {
1358 DefIdxs.push_back(Elt: VNI->def);
1359 continue;
1360 }
1361
1362 for (MachineBasicBlock *PredMBB : DefMBB->predecessors()) {
1363 if (Visited.insert(Ptr: PredMBB).second)
1364 Worklist.push_back(Elt: PredMBB);
1365 }
1366 }
1367}
1368
1369void RewriteMFMAFormStage::findReachingUses(
1370 const MachineInstr *DefMI, LiveIntervals *LIS,
1371 SmallVectorImpl<MachineOperand *> &ReachingUses) {
1372 SlotIndex DefIdx = LIS->getInstructionIndex(Instr: *DefMI);
1373 for (MachineOperand &UseMO :
1374 DAG.MRI.use_nodbg_operands(Reg: DefMI->getOperand(i: 0).getReg())) {
1375 SmallVector<SlotIndex, 8> ReachingDefIndexes;
1376 findReachingDefs(UseMO, LIS, DefIdxs&: ReachingDefIndexes);
1377
1378 // If we find a use that contains this DefMI in its reachingDefs, then it is
1379 // a reaching use.
1380 if (any_of(Range&: ReachingDefIndexes, P: [DefIdx](SlotIndex RDIdx) {
1381 return SlotIndex::isSameInstr(A: RDIdx, B: DefIdx);
1382 }))
1383 ReachingUses.push_back(Elt: &UseMO);
1384 }
1385}
1386
1387bool RewriteMFMAFormStage::initGCNSchedStage() {
1388 // We only need to run this pass if the architecture supports AGPRs.
1389 // Additionally, we don't use AGPRs at occupancy levels above 1 so there
1390 // is no need for this pass in that case, either.
1391 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1392 if (!ST.hasGFX90AInsts() || MFI.getMinWavesPerEU() > 1)
1393 return false;
1394
1395 RegionsWithExcessArchVGPR.resize(N: static_cast<unsigned>(DAG.Regions.size()));
1396 RegionsWithExcessArchVGPR.reset();
1397 for (unsigned Region = 0; Region < DAG.Regions.size(); Region++) {
1398 GCNRegPressure PressureBefore = DAG.Pressure[Region];
1399 if (PressureBefore.getArchVGPRNum() > ST.getAddressableNumArchVGPRs())
1400 RegionsWithExcessArchVGPR[Region] = true;
1401 }
1402
1403 if (RegionsWithExcessArchVGPR.none())
1404 return false;
1405
1406 TII = ST.getInstrInfo();
1407 SRI = ST.getRegisterInfo();
1408
1409 std::vector<std::pair<MachineInstr *, unsigned>> RewriteCands;
1410 DenseMap<MachineBasicBlock *, std::set<Register>> CopyForUse;
1411 SmallPtrSet<MachineInstr *, 8> CopyForDef;
1412
1413 if (!initHeuristics(RewriteCands, CopyForUse, CopyForDef))
1414 return false;
1415
1416 int64_t Cost = getRewriteCost(RewriteCands, CopyForUse, CopyForDef);
1417
1418 // If we haven't found the beneficial conditions, prefer the VGPR form which
1419 // may result in less cross RC copies.
1420 if (Cost > 0)
1421 return false;
1422
1423 return rewrite(RewriteCands);
1424}
1425
1426bool UnclusteredHighRPStage::initGCNSchedStage() {
1427 if (!ST.getCLOpts().unclustered_high_rp_reschedule)
1428 return false;
1429
1430 if (!GCNSchedStage::initGCNSchedStage())
1431 return false;
1432
1433 if (DAG.RegionsWithHighRP.none() && DAG.RegionsWithExcessRP.none())
1434 return false;
1435
1436 SavedMutations.swap(x&: DAG.Mutations);
1437 DAG.addMutation(
1438 Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::PreRAReentry));
1439
1440 InitialOccupancy = DAG.MinOccupancy;
1441 // Aggressively try to reduce register pressure in the unclustered high RP
1442 // stage. Temporarily increase occupancy target in the region.
1443 TempTargetOccupancy = MFI.getMaxWavesPerEU() > DAG.MinOccupancy
1444 ? InitialOccupancy + 1
1445 : InitialOccupancy;
1446 IsAnyRegionScheduled = false;
1447 S.SGPRLimitBias = S.HighRPSGPRBias;
1448 S.VGPRLimitBias = S.HighRPVGPRBias;
1449
1450 LLVM_DEBUG(
1451 dbgs()
1452 << "Retrying function scheduling without clustering. "
1453 "Aggressively try to reduce register pressure to achieve occupancy "
1454 << TempTargetOccupancy << ".\n");
1455
1456 return true;
1457}
1458
1459bool ClusteredLowOccStage::initGCNSchedStage() {
1460 if (!ST.getCLOpts().clustered_low_occupancy_reschedule)
1461 return false;
1462
1463 if (!GCNSchedStage::initGCNSchedStage())
1464 return false;
1465
1466 // Don't bother trying to improve ILP in lower RP regions if occupancy has not
1467 // been dropped. All regions will have already been scheduled with the ideal
1468 // occupancy targets.
1469 if (DAG.StartingOccupancy <= DAG.MinOccupancy)
1470 return false;
1471
1472 LLVM_DEBUG(
1473 dbgs() << "Retrying function scheduling with lowest recorded occupancy "
1474 << DAG.MinOccupancy << ".\n");
1475 return true;
1476}
1477
1478bool PreRARematStage::initGCNSchedStage() {
1479 // FIXME: This pass will invalidate cached BBLiveInMap and MBBLiveIns for
1480 // regions inbetween the defs and region we sinked the def to. Will need to be
1481 // fixed if there is another pass after this pass.
1482 assert(!S.hasNextStage());
1483
1484 if (!GCNSchedStage::initGCNSchedStage() || DAG.Regions.size() <= 1)
1485 return false;
1486
1487 // Set up "spilling targets" for all regions.
1488 const Function &F = MF.getFunction();
1489 unsigned MaxSGPRs = ST.getMaxNumSGPRs(F);
1490 unsigned MaxVGPRs = ST.getMaxNumVGPRs(F);
1491
1492 const unsigned VGPRBlockSize = MFI.getDynamicVGPRBlockSize();
1493 bool HasVectorRegisterExcess = false;
1494 for (const GCNRegPressure &RP : DAG.Pressure) {
1495 if (GCNRPTarget(MaxSGPRs, MaxVGPRs, MF, RP).hasVectorRegisterExcess()) {
1496 HasVectorRegisterExcess = true;
1497 break;
1498 }
1499 }
1500 if (HasVectorRegisterExcess || DAG.MinOccupancy >= MFI.getMaxWavesPerEU()) {
1501 // In addition to register usage being above addressable limits, occupancy
1502 // below the minimum is considered like "spilling" as well.
1503 TargetOcc = std::nullopt;
1504 } else {
1505 // There is no spilling and room to improve occupancy; set up "increased
1506 // occupancy targets" for all regions.
1507 TargetOcc = DAG.MinOccupancy + 1;
1508 MaxSGPRs = ST.getMaxNumSGPRs(WavesPerEU: *TargetOcc, Addressable: false);
1509 MaxVGPRs = ST.getMaxNumVGPRs(WavesPerEU: *TargetOcc, DynamicVGPRBlockSize: VGPRBlockSize);
1510 }
1511
1512 LLVM_DEBUG({
1513 dbgs() << "In function ";
1514 MF.getFunction().printAsOperand(dbgs(), false);
1515 dbgs() << ": rematerializing to ";
1516 if (TargetOcc) {
1517 dbgs() << "increase occupancy from " << *TargetOcc - 1 << '\n';
1518 } else {
1519 dbgs() << "reduce spilling (minimum target occupancy is "
1520 << MFI.getMinWavesPerEU() << ")\n";
1521 }
1522 });
1523
1524 LiveOuts.reserve(N: DAG.Regions.size());
1525 DAG.RegionLiveOuts.buildLiveRegMap();
1526 for (unsigned RegionIdx : seq(Size: DAG.Regions.size()))
1527 LiveOuts.push_back(Elt: DAG.RegionLiveOuts.getLiveRegsForRegionIdx(RegionIdx));
1528
1529 Strategy = std::make_unique<GCNRematStrategy>(args&: MF, args&: DAG.Regions, args&: DAG.LiveIns,
1530 args&: LiveOuts, args&: *DAG.LIS, args&: *DAG.MBFI);
1531 if (TargetOcc.has_value()) {
1532 // Rematerialization is likely to move instructions into higher frequency
1533 // regions, increasing their total sum latency. This is acceptable if we are
1534 // eliminating spills in the process, but when the goal is increasing
1535 // occupancy we get nothing out of rematerialization if occupancy is not
1536 // increased in the end; in such cases we want to be able to roll back
1537 // rematerializations.
1538 Strategy->startRecordingRemats();
1539 }
1540
1541 RescheduleRegions.resize(N: DAG.Regions.size());
1542 MutableArrayRef<GCNRegPressure> MaxPressure = DAG.Pressure;
1543 Strategy->rematerializeFor(MaxSGPRs, MaxVGPRs,
1544 /*ReduceSpill=*/SpillReduction: !TargetOcc.has_value(),
1545 AffectedRegions&: RescheduleRegions, MaxPressure: &MaxPressure);
1546 if (RescheduleRegions.none())
1547 return false;
1548
1549 // Reflect changes in the DAG and compute achieved occupancy.
1550 LLVM_DEBUG(dbgs() << "==== REMAT RESULTS ====\n");
1551 for (unsigned RegionIdx : RescheduleRegions.set_bits()) {
1552 DAG.RegionLiveOuts.getLiveRegsForRegionIdx(RegionIdx) = LiveOuts[RegionIdx];
1553 LLVM_DEBUG(dbgs() << '[' << RegionIdx << "] Achieved occupancy "
1554 << DAG.Pressure[RegionIdx].getOccupancy(ST, VGPRBlockSize)
1555 << '\n');
1556 }
1557 AchievedOcc = MFI.getMaxWavesPerEU();
1558 for (const GCNRegPressure &RP : DAG.Pressure)
1559 AchievedOcc = std::min(a: AchievedOcc, b: RP.getOccupancy(ST, DynamicVGPRBlockSize: VGPRBlockSize));
1560
1561 LLVM_DEBUG({
1562 dbgs() << "Retrying function scheduling with new min. occupancy of "
1563 << AchievedOcc << " from rematerializing (original was "
1564 << DAG.MinOccupancy;
1565 if (TargetOcc)
1566 dbgs() << ", target was " << *TargetOcc;
1567 dbgs() << ")\n";
1568 });
1569
1570 DAG.setTargetOccupancy(getStageTargetOccupancy());
1571 return true;
1572}
1573
1574void GCNSchedStage::finalizeGCNSchedStage() {
1575 DAG.finishBlock();
1576 LLVM_DEBUG(dbgs() << "Ending scheduling stage: " << StageID << "\n");
1577}
1578
1579void UnclusteredHighRPStage::finalizeGCNSchedStage() {
1580 SavedMutations.swap(x&: DAG.Mutations);
1581 S.SGPRLimitBias = S.VGPRLimitBias = 0;
1582 if (DAG.MinOccupancy > InitialOccupancy) {
1583 assert(IsAnyRegionScheduled);
1584 LLVM_DEBUG(dbgs() << StageID
1585 << " stage successfully increased occupancy to "
1586 << DAG.MinOccupancy << '\n');
1587 } else if (!IsAnyRegionScheduled) {
1588 assert(DAG.MinOccupancy == InitialOccupancy);
1589 LLVM_DEBUG(dbgs() << StageID
1590 << ": No regions scheduled, min occupancy stays at "
1591 << DAG.MinOccupancy << ", MFI occupancy stays at "
1592 << MFI.getOccupancy() << ".\n");
1593 }
1594
1595 GCNSchedStage::finalizeGCNSchedStage();
1596}
1597
1598bool GCNSchedStage::initGCNRegion() {
1599 // Skip empty scheduling region.
1600 if (DAG.begin() == DAG.end())
1601 return false;
1602
1603 // Check whether this new region is also a new block.
1604 if (DAG.RegionBegin->getParent() != CurrentMBB)
1605 setupNewBlock();
1606
1607 unsigned NumRegionInstrs =
1608 static_cast<unsigned>(std::distance(first: DAG.begin(), last: DAG.end()));
1609 DAG.enterRegion(bb: CurrentMBB, begin: DAG.begin(), end: DAG.end(), regioninstrs: NumRegionInstrs);
1610
1611 // Skip regions with 1 schedulable instruction.
1612 if (DAG.begin() == std::prev(x: DAG.end()))
1613 return false;
1614
1615 LLVM_DEBUG(dbgs() << "********** MI Scheduling **********\n");
1616 LLVM_DEBUG(dbgs() << MF.getName() << ":" << printMBBReference(*CurrentMBB)
1617 << " " << CurrentMBB->getName()
1618 << "\n From: " << *DAG.begin() << " To: ";
1619 if (DAG.RegionEnd != CurrentMBB->end()) dbgs() << *DAG.RegionEnd;
1620 else dbgs() << "End";
1621 dbgs() << " RegionInstrs: " << NumRegionInstrs << '\n');
1622
1623 // Save original instruction order before scheduling for possible revert.
1624 Unsched.clear();
1625 Unsched.reserve(n: DAG.NumRegionInstrs);
1626 if (StageID == GCNSchedStageID::OccInitialSchedule ||
1627 StageID == GCNSchedStageID::ILPInitialSchedule) {
1628 const SIInstrInfo *SII = static_cast<const SIInstrInfo *>(DAG.TII);
1629 for (auto &I : DAG) {
1630 Unsched.push_back(x: &I);
1631 if (SII->isIGLPMutationOnly(Opcode: I.getOpcode()))
1632 DAG.RegionsWithIGLPInstrs[RegionIdx] = true;
1633 }
1634 } else {
1635 for (auto &I : DAG)
1636 Unsched.push_back(x: &I);
1637 }
1638
1639 PressureBefore = DAG.Pressure[RegionIdx];
1640
1641 LLVM_DEBUG(
1642 dbgs() << "Pressure before scheduling:\nRegion live-ins:"
1643 << print(DAG.LiveIns[RegionIdx], DAG.MRI)
1644 << "Region live-in pressure: "
1645 << print(llvm::getRegPressure(DAG.MRI, DAG.LiveIns[RegionIdx]))
1646 << "Region register pressure: " << print(PressureBefore));
1647
1648 S.HasHighPressure = false;
1649 S.KnownExcessRP = isRegionWithExcessRP();
1650
1651 if (DAG.RegionsWithIGLPInstrs[RegionIdx] &&
1652 StageID != GCNSchedStageID::UnclusteredHighRPReschedule) {
1653 SavedMutations.clear();
1654 SavedMutations.swap(x&: DAG.Mutations);
1655 bool IsInitialStage = StageID == GCNSchedStageID::OccInitialSchedule ||
1656 StageID == GCNSchedStageID::ILPInitialSchedule;
1657 DAG.addMutation(Mutation: createIGroupLPDAGMutation(
1658 Phase: IsInitialStage ? AMDGPU::SchedulingPhase::Initial
1659 : AMDGPU::SchedulingPhase::PreRAReentry));
1660 }
1661
1662 return true;
1663}
1664
1665bool UnclusteredHighRPStage::initGCNRegion() {
1666 // Only reschedule regions that have excess register pressure (i.e. spilling)
1667 // or had minimum occupancy at the beginning of the stage (as long as
1668 // rescheduling of previous regions did not make occupancy drop back down to
1669 // the initial minimum).
1670 unsigned DynamicVGPRBlockSize = DAG.MFI.getDynamicVGPRBlockSize();
1671 // If no region has been scheduled yet, the DAG has not yet been updated with
1672 // the occupancy target. So retrieve it from the temporary.
1673 unsigned CurrentTargetOccupancy =
1674 IsAnyRegionScheduled ? DAG.MinOccupancy : TempTargetOccupancy;
1675 if (!DAG.RegionsWithExcessRP[RegionIdx] &&
1676 (CurrentTargetOccupancy <= InitialOccupancy ||
1677 DAG.Pressure[RegionIdx].getOccupancy(ST, DynamicVGPRBlockSize) !=
1678 InitialOccupancy))
1679 return false;
1680
1681 bool IsSchedulingThisRegion = GCNSchedStage::initGCNRegion();
1682 // If this is the first region scheduled during this stage, make the target
1683 // occupancy changes in the DAG and MFI.
1684 if (!IsAnyRegionScheduled && IsSchedulingThisRegion) {
1685 IsAnyRegionScheduled = true;
1686 if (MFI.getMaxWavesPerEU() > DAG.MinOccupancy)
1687 DAG.setTargetOccupancy(TempTargetOccupancy);
1688 }
1689 return IsSchedulingThisRegion;
1690}
1691
1692bool ClusteredLowOccStage::initGCNRegion() {
1693 // We may need to reschedule this region if it wasn't rescheduled in the last
1694 // stage, or if we found it was testing critical register pressure limits in
1695 // the unclustered reschedule stage. The later is because we may not have been
1696 // able to raise the min occupancy in the previous stage so the region may be
1697 // overly constrained even if it was already rescheduled.
1698 if (!DAG.RegionsWithHighRP[RegionIdx])
1699 return false;
1700
1701 return GCNSchedStage::initGCNRegion();
1702}
1703
1704bool PreRARematStage::initGCNRegion() {
1705 return !RevertAllRegions && RescheduleRegions[RegionIdx] &&
1706 GCNSchedStage::initGCNRegion();
1707}
1708
1709void GCNSchedStage::setupNewBlock() {
1710 if (CurrentMBB)
1711 DAG.finishBlock();
1712
1713 CurrentMBB = DAG.RegionBegin->getParent();
1714 DAG.startBlock(bb: CurrentMBB);
1715 // Get real RP for the region if it hasn't be calculated before. After the
1716 // initial schedule stage real RP will be collected after scheduling.
1717 if (StageID == GCNSchedStageID::OccInitialSchedule ||
1718 StageID == GCNSchedStageID::ILPInitialSchedule ||
1719 StageID == GCNSchedStageID::MemoryClauseInitialSchedule)
1720 DAG.computeBlockPressure(RegionIdx, MBB: CurrentMBB);
1721}
1722
1723void GCNSchedStage::finalizeGCNRegion() {
1724 DAG.Regions[RegionIdx] = std::pair(DAG.RegionBegin, DAG.RegionEnd);
1725 if (S.HasHighPressure)
1726 DAG.RegionsWithHighRP[RegionIdx] = true;
1727
1728 // Revert scheduling if we have dropped occupancy or there is some other
1729 // reason that the original schedule is better.
1730 checkScheduling();
1731
1732 if (DAG.RegionsWithIGLPInstrs[RegionIdx] &&
1733 StageID != GCNSchedStageID::UnclusteredHighRPReschedule)
1734 SavedMutations.swap(x&: DAG.Mutations);
1735}
1736
1737void PreRARematStage::finalizeGCNRegion() {
1738 GCNSchedStage::finalizeGCNRegion();
1739 // When the goal is to increase occupancy, all regions must reach the target
1740 // occupancy for rematerializations to be possibly useful, otherwise we will
1741 // just hurt latency for no benefit. If minimum occupancy drops below the
1742 // target there is no point in trying to re-schedule further regions.
1743 if (!TargetOcc)
1744 return;
1745 RegionReverts.emplace_back(Args&: RegionIdx, Args&: Unsched, Args&: PressureBefore);
1746 if (DAG.MinOccupancy < *TargetOcc) {
1747 LLVM_DEBUG(dbgs() << "Region " << RegionIdx
1748 << " cannot meet occupancy target, interrupting "
1749 "re-scheduling in all regions\n");
1750 RevertAllRegions = true;
1751 }
1752}
1753
1754void GCNSchedStage::checkScheduling() {
1755 // The maximum number of arch VGPR on non-unified register file, or the
1756 // maximum VGPR + AGPR in the unified register file case.
1757 unsigned MaxVGPRs = ST.getMaxNumVGPRs(MF);
1758 // The maximum number of arch VGPR for both unified and non-unified register
1759 // file.
1760 unsigned MaxArchVGPRs = std::min(a: MaxVGPRs, b: ST.getAddressableNumArchVGPRs());
1761
1762 // The region maximum keeps the VGPR and AVGPR pressure of a single point, so
1763 // distributing it across the banks can miss a rigid peak at another point.
1764 // Distribute the pressure of each point instead.
1765 bool AVGPRBanking = ST.getCLOpts().avgpr_banking;
1766 unsigned ArchVGPRBankLimit, AGPRBankLimit;
1767 std::tie(args&: ArchVGPRBankLimit, args&: AGPRBankLimit) = S.getBankLimits();
1768 unsigned BankedArchVGPRPeak = 0, BankedAGPRPeak = 0, BankedUnifiedPeak = 0;
1769 std::function<void(const GCNRegPressure &)> TrackBankedPeaks;
1770 if (AVGPRBanking) {
1771 TrackBankedPeaks = [&](const GCNRegPressure &RP) {
1772 auto [ArchNum, AccNum] =
1773 RP.getBankedVGPRNum(ArchLimit: ArchVGPRBankLimit, AccLimit: AGPRBankLimit);
1774 BankedArchVGPRPeak = std::max(a: BankedArchVGPRPeak, b: ArchNum);
1775 BankedAGPRPeak = std::max(a: BankedAGPRPeak, b: AccNum);
1776 BankedUnifiedPeak =
1777 std::max(a: BankedUnifiedPeak,
1778 b: GCNRegPressure::getBankedUnifiedVGPRNum(NumArchVGPRs: ArchNum, NumAGPRs: AccNum));
1779 };
1780 }
1781
1782 // Check the results of scheduling.
1783 PressureAfter = DAG.getRealRegPressure(RegionIdx, PressureCallback: TrackBankedPeaks);
1784
1785 LLVM_DEBUG(dbgs() << "Pressure after scheduling: " << print(PressureAfter));
1786 LLVM_DEBUG(dbgs() << "Region: " << RegionIdx << ".\n");
1787
1788 unsigned DynamicVGPRBlockSize = DAG.MFI.getDynamicVGPRBlockSize();
1789
1790 bool UnifiedVGPRFile = ST.hasGFX90AInsts();
1791 unsigned VectorCriticalLimit =
1792 UnifiedVGPRFile ? S.UnifiedVGPRCriticalLimit : S.VGPRCriticalLimit;
1793 // Fitting the combined occupancy budget does not rule out spilling in one
1794 // bank. Keep the per-bank excess checks before taking this early exit.
1795 if (PressureAfter.getSGPRNum() <= S.SGPRCriticalLimit &&
1796 PressureAfter.getVGPRNum(UnifiedVGPRFile) <= VectorCriticalLimit &&
1797 (!UnifiedVGPRFile ||
1798 (PressureAfter.getArchVGPRNum() <= S.VGPRExcessLimit &&
1799 PressureAfter.getAGPRNum() <= S.AGPRExcessLimit))) {
1800 DAG.Pressure[RegionIdx] = PressureAfter;
1801
1802 // Early out if we have achieved the occupancy target.
1803 LLVM_DEBUG(dbgs() << "Pressure in desired limits, done.\n");
1804 return;
1805 }
1806
1807 unsigned TargetOccupancy = std::min(
1808 a: S.getTargetOccupancy(), b: ST.getOccupancyWithWorkGroupSizes(MF).second);
1809 unsigned WavesAfter = std::min(
1810 a: TargetOccupancy, b: PressureAfter.getOccupancy(ST, DynamicVGPRBlockSize));
1811 unsigned WavesBefore = std::min(
1812 a: TargetOccupancy, b: PressureBefore.getOccupancy(ST, DynamicVGPRBlockSize));
1813 LLVM_DEBUG(dbgs() << "Occupancy before scheduling: " << WavesBefore
1814 << ", after " << WavesAfter << ".\n");
1815
1816 // We may not be able to keep the current target occupancy because of the just
1817 // scheduled region. We might still be able to revert scheduling if the
1818 // occupancy before was higher, or if the current schedule has register
1819 // pressure higher than the excess limits which could lead to more spilling.
1820 unsigned NewOccupancy = std::max(a: WavesAfter, b: WavesBefore);
1821
1822 // Allow memory bound functions to drop to 4 waves if not limited by an
1823 // attribute.
1824 if (WavesAfter < WavesBefore && WavesAfter < DAG.MinOccupancy &&
1825 WavesAfter >= MFI.getMinAllowedOccupancy()) {
1826 LLVM_DEBUG(dbgs() << "Function is memory bound, allow occupancy drop up to "
1827 << MFI.getMinAllowedOccupancy() << " waves\n");
1828 NewOccupancy = WavesAfter;
1829 }
1830
1831 if (NewOccupancy < DAG.MinOccupancy) {
1832 DAG.MinOccupancy = NewOccupancy;
1833 MFI.limitOccupancy(Limit: DAG.MinOccupancy);
1834 LLVM_DEBUG(dbgs() << "Occupancy lowered for the function to "
1835 << DAG.MinOccupancy << ".\n");
1836 }
1837 unsigned MaxSGPRs = ST.getMaxNumSGPRs(MF);
1838
1839 bool BankExcess = AVGPRBanking
1840 ? BankedArchVGPRPeak > ArchVGPRBankLimit ||
1841 BankedAGPRPeak > AGPRBankLimit
1842 : PressureAfter.getArchVGPRNum() > MaxArchVGPRs ||
1843 PressureAfter.getAGPRNum() > MaxArchVGPRs;
1844
1845 unsigned VGPRNum = AVGPRBanking && UnifiedVGPRFile
1846 ? BankedUnifiedPeak
1847 : PressureAfter.getVGPRNum(UnifiedVGPRFile);
1848 if (VGPRNum > MaxVGPRs || BankExcess ||
1849 PressureAfter.getSGPRNum() > MaxSGPRs) {
1850 DAG.RegionsWithHighRP[RegionIdx] = true;
1851 DAG.RegionsWithExcessRP[RegionIdx] = true;
1852 }
1853
1854 // Revert if this region's schedule would cause a drop in occupancy or
1855 // spilling.
1856 if (shouldRevertScheduling(WavesAfter)) {
1857 modifyRegionSchedule(RegionIdx, MIOrder: Unsched);
1858 std::tie(args&: DAG.RegionBegin, args&: DAG.RegionEnd) = DAG.Regions[RegionIdx];
1859 } else {
1860 DAG.Pressure[RegionIdx] = PressureAfter;
1861 }
1862}
1863
1864unsigned
1865GCNSchedStage::computeSUnitReadyCycle(const SUnit &SU, unsigned CurrCycle,
1866 DenseMap<unsigned, unsigned> &ReadyCycles,
1867 const TargetSchedModel &SM) {
1868 unsigned ReadyCycle = CurrCycle;
1869 for (auto &D : SU.Preds) {
1870 if (D.isAssignedRegDep()) {
1871 MachineInstr *DefMI = D.getSUnit()->getInstr();
1872 unsigned Latency = SM.computeInstrLatency(MI: DefMI);
1873 unsigned DefReady = ReadyCycles[DAG.getSUnit(MI: DefMI)->NodeNum];
1874 ReadyCycle = std::max(a: ReadyCycle, b: DefReady + Latency);
1875 }
1876 }
1877 ReadyCycles[SU.NodeNum] = ReadyCycle;
1878 return ReadyCycle;
1879}
1880
1881#ifndef NDEBUG
1882struct EarlierIssuingCycle {
1883 bool operator()(std::pair<MachineInstr *, unsigned> A,
1884 std::pair<MachineInstr *, unsigned> B) const {
1885 return A.second < B.second;
1886 }
1887};
1888
1889static void printScheduleModel(std::set<std::pair<MachineInstr *, unsigned>,
1890 EarlierIssuingCycle> &ReadyCycles) {
1891 if (ReadyCycles.empty())
1892 return;
1893 unsigned BBNum = ReadyCycles.begin()->first->getParent()->getNumber();
1894 dbgs() << "\n################## Schedule time ReadyCycles for MBB : " << BBNum
1895 << " ##################\n# Cycle #\t\t\tInstruction "
1896 " "
1897 " \n";
1898 unsigned IPrev = 1;
1899 for (auto &I : ReadyCycles) {
1900 if (I.second > IPrev + 1)
1901 dbgs() << "****************************** BUBBLE OF " << I.second - IPrev
1902 << " CYCLES DETECTED ******************************\n\n";
1903 dbgs() << "[ " << I.second << " ] : " << *I.first << "\n";
1904 IPrev = I.second;
1905 }
1906}
1907#endif
1908
1909ScheduleMetrics
1910GCNSchedStage::getScheduleMetrics(const std::vector<SUnit> &InputSchedule) {
1911#ifndef NDEBUG
1912 std::set<std::pair<MachineInstr *, unsigned>, EarlierIssuingCycle>
1913 ReadyCyclesSorted;
1914#endif
1915 const TargetSchedModel &SM = ST.getInstrInfo()->getSchedModel();
1916 unsigned SumBubbles = 0;
1917 DenseMap<unsigned, unsigned> ReadyCycles;
1918 unsigned CurrCycle = 0;
1919 for (auto &SU : InputSchedule) {
1920 unsigned ReadyCycle =
1921 computeSUnitReadyCycle(SU, CurrCycle, ReadyCycles, SM);
1922 SumBubbles += ReadyCycle - CurrCycle;
1923#ifndef NDEBUG
1924 ReadyCyclesSorted.insert(std::make_pair(SU.getInstr(), ReadyCycle));
1925#endif
1926 CurrCycle = ++ReadyCycle;
1927 }
1928#ifndef NDEBUG
1929 LLVM_DEBUG(
1930 printScheduleModel(ReadyCyclesSorted);
1931 dbgs() << "\n\t"
1932 << "Metric: "
1933 << (SumBubbles
1934 ? (SumBubbles * ScheduleMetrics::ScaleFactor) / CurrCycle
1935 : 1)
1936 << "\n\n");
1937#endif
1938
1939 return ScheduleMetrics(CurrCycle, SumBubbles);
1940}
1941
1942ScheduleMetrics
1943GCNSchedStage::getScheduleMetrics(const GCNScheduleDAGMILive &DAG) {
1944#ifndef NDEBUG
1945 std::set<std::pair<MachineInstr *, unsigned>, EarlierIssuingCycle>
1946 ReadyCyclesSorted;
1947#endif
1948 const TargetSchedModel &SM = ST.getInstrInfo()->getSchedModel();
1949 unsigned SumBubbles = 0;
1950 DenseMap<unsigned, unsigned> ReadyCycles;
1951 unsigned CurrCycle = 0;
1952 for (auto &MI : DAG) {
1953 SUnit *SU = DAG.getSUnit(MI: &MI);
1954 if (!SU)
1955 continue;
1956 unsigned ReadyCycle =
1957 computeSUnitReadyCycle(SU: *SU, CurrCycle, ReadyCycles, SM);
1958 SumBubbles += ReadyCycle - CurrCycle;
1959#ifndef NDEBUG
1960 ReadyCyclesSorted.insert(std::make_pair(SU->getInstr(), ReadyCycle));
1961#endif
1962 CurrCycle = ++ReadyCycle;
1963 }
1964#ifndef NDEBUG
1965 LLVM_DEBUG(
1966 printScheduleModel(ReadyCyclesSorted);
1967 dbgs() << "\n\t"
1968 << "Metric: "
1969 << (SumBubbles
1970 ? (SumBubbles * ScheduleMetrics::ScaleFactor) / CurrCycle
1971 : 1)
1972 << "\n\n");
1973#endif
1974
1975 return ScheduleMetrics(CurrCycle, SumBubbles);
1976}
1977
1978bool GCNSchedStage::shouldRevertScheduling(unsigned WavesAfter) {
1979 if (WavesAfter < DAG.MinOccupancy)
1980 return true;
1981
1982 // For dynamic VGPR mode, we don't want to waste any VGPR blocks.
1983 if (DAG.MFI.isDynamicVGPREnabled()) {
1984 unsigned BlocksBefore = AMDGPU::IsaInfo::getAllocatedNumVGPRBlocks(
1985 STI: ST, NumVGPRs: PressureBefore.getVGPRNum(UnifiedVGPRFile: false),
1986 DynamicVGPRBlockSize: DAG.MFI.getDynamicVGPRBlockSize());
1987 unsigned BlocksAfter = AMDGPU::IsaInfo::getAllocatedNumVGPRBlocks(
1988 STI: ST, NumVGPRs: PressureAfter.getVGPRNum(UnifiedVGPRFile: false), DynamicVGPRBlockSize: DAG.MFI.getDynamicVGPRBlockSize());
1989 if (BlocksAfter > BlocksBefore)
1990 return true;
1991 }
1992
1993 return false;
1994}
1995
1996bool OccInitialScheduleStage::shouldRevertScheduling(unsigned WavesAfter) {
1997 if (PressureAfter == PressureBefore)
1998 return false;
1999
2000 if (GCNSchedStage::shouldRevertScheduling(WavesAfter))
2001 return true;
2002
2003 if (mayCauseSpilling(WavesAfter))
2004 return true;
2005
2006 return false;
2007}
2008
2009bool UnclusteredHighRPStage::shouldRevertScheduling(unsigned WavesAfter) {
2010 // If RP is not reduced in the unclustered reschedule stage, revert to the
2011 // old schedule.
2012 if ((WavesAfter <=
2013 PressureBefore.getOccupancy(ST, DynamicVGPRBlockSize: DAG.MFI.getDynamicVGPRBlockSize()) &&
2014 mayCauseSpilling(WavesAfter)) ||
2015 GCNSchedStage::shouldRevertScheduling(WavesAfter)) {
2016 LLVM_DEBUG(dbgs() << "Unclustered reschedule did not help.\n");
2017 return true;
2018 }
2019
2020 // Do not attempt to relax schedule even more if we are already spilling.
2021 if (isRegionWithExcessRP())
2022 return false;
2023
2024 LLVM_DEBUG(
2025 dbgs()
2026 << "\n\t *** In shouldRevertScheduling ***\n"
2027 << " *********** BEFORE UnclusteredHighRPStage ***********\n");
2028 ScheduleMetrics MBefore = getScheduleMetrics(InputSchedule: DAG.SUnits);
2029 LLVM_DEBUG(
2030 dbgs()
2031 << "\n *********** AFTER UnclusteredHighRPStage ***********\n");
2032 ScheduleMetrics MAfter = getScheduleMetrics(DAG);
2033 unsigned OldMetric = MBefore.getMetric();
2034 unsigned NewMetric = MAfter.getMetric();
2035 unsigned ScheduleMetricBias = ST.getCLOpts().schedule_metric_bias;
2036 unsigned WavesBefore = std::min(
2037 a: S.getTargetOccupancy(),
2038 b: PressureBefore.getOccupancy(ST, DynamicVGPRBlockSize: DAG.MFI.getDynamicVGPRBlockSize()));
2039 unsigned Profit =
2040 ((WavesAfter * ScheduleMetrics::ScaleFactor) / WavesBefore *
2041 ((OldMetric + ScheduleMetricBias) * ScheduleMetrics::ScaleFactor) /
2042 NewMetric) /
2043 ScheduleMetrics::ScaleFactor;
2044 LLVM_DEBUG(dbgs() << "\tMetric before " << MBefore << "\tMetric after "
2045 << MAfter << "Profit: " << Profit << "\n");
2046 return Profit < ScheduleMetrics::ScaleFactor;
2047}
2048
2049bool ClusteredLowOccStage::shouldRevertScheduling(unsigned WavesAfter) {
2050 if (PressureAfter == PressureBefore)
2051 return false;
2052
2053 if (GCNSchedStage::shouldRevertScheduling(WavesAfter))
2054 return true;
2055
2056 if (mayCauseSpilling(WavesAfter))
2057 return true;
2058
2059 return false;
2060}
2061
2062bool PreRARematStage::shouldRevertScheduling(unsigned WavesAfter) {
2063 return GCNSchedStage::shouldRevertScheduling(WavesAfter) ||
2064 mayCauseSpilling(WavesAfter);
2065}
2066
2067bool ILPInitialScheduleStage::shouldRevertScheduling(unsigned WavesAfter) {
2068 if (mayCauseSpilling(WavesAfter))
2069 return true;
2070
2071 return false;
2072}
2073
2074bool MemoryClauseInitialScheduleStage::shouldRevertScheduling(
2075 unsigned WavesAfter) {
2076 return mayCauseSpilling(WavesAfter);
2077}
2078
2079bool LiveIntervalRPStage::initGCNSchedStage() {
2080 if (!ST.getCLOpts().lirp_reschedule)
2081 return false;
2082
2083 if (!GCNSchedStage::initGCNSchedStage())
2084 return false;
2085
2086 if (!S.VGPRThresholdPercent) {
2087 LLVM_DEBUG(dbgs() << "LIRP: expected VGPRThresholdPercent to be enabled, "
2088 "not using live interval RP reschedule stage\n");
2089 return false;
2090 }
2091
2092 return true;
2093}
2094
2095bool LiveIntervalRPStage::initGCNRegion() {
2096 const AMDGPUOptions &CLOpts = ST.getCLOpts();
2097 unsigned InstantRP = DAG.Pressure[RegionIdx].getArchVGPRNum();
2098 auto [RegionBegin, RegionEnd] = DAG.Regions[RegionIdx];
2099 if (RegionBegin == RegionEnd)
2100 return false;
2101
2102 unsigned LIRP = estimateGreedyVGPRPressure(
2103 RegionBegin, RegionEnd, LiveIns: DAG.LiveIns[RegionIdx], LIS: *DAG.getLIS(),
2104 MRI: DAG.MF.getRegInfo(), TRI: static_cast<const SIRegisterInfo &>(*DAG.TRI));
2105
2106 unsigned NewVGPRThresholdPercent =
2107 (S.VGPRThresholdPercent * CLOpts.lirp_vgpr_reduction + 99) / 100;
2108
2109 LLVM_DEBUG(dbgs() << "LIRP: Region " << RegionIdx
2110 << ", VGPRThresholdPercent: " << S.VGPRThresholdPercent
2111 << " -> " << NewVGPRThresholdPercent
2112 << ", VGPRExcessLimit=" << S.VGPRExcessLimit
2113 << ", VGPRCriticalLimit=" << S.VGPRCriticalLimit
2114 << ", InstantRP=" << InstantRP << ", LIRP=" << LIRP);
2115
2116 bool DoRescheduling = false;
2117 // Lower bound on InstantRP to skip over tiny regions.
2118 unsigned InstantRPLowerBound =
2119 S.VGPRExcessLimit * CLOpts.lirp_instant_lower_bound / 100;
2120 if (LIRP > S.VGPRExcessLimit) {
2121 LLVM_DEBUG(dbgs() << " [LIRP exceeds the limit (" << S.VGPRExcessLimit
2122 << "), rescheduling]");
2123 DoRescheduling = true;
2124 } else if (LIRP > InstantRP && InstantRP > InstantRPLowerBound) {
2125 unsigned IncreasePercent = ((LIRP - InstantRP) * 100) / InstantRP;
2126 if (IncreasePercent > CLOpts.lirp_threshold) {
2127 LLVM_DEBUG(dbgs() << " [" << IncreasePercent << "% > "
2128 << CLOpts.lirp_threshold << "%, rescheduling]");
2129 DoRescheduling = true;
2130 }
2131 }
2132 LLVM_DEBUG(dbgs() << '\n');
2133
2134 if (DoRescheduling && GCNSchedStage::initGCNRegion()) {
2135 SavedVGPRExcessLimit = S.VGPRExcessLimit;
2136 SavedVGPRCriticalLimit = S.VGPRCriticalLimit;
2137 SavedUnifiedVGPRCriticalLimit = S.UnifiedVGPRCriticalLimit;
2138 SavedVGPRThresholdPercent = S.VGPRThresholdPercent;
2139 S.VGPRThresholdPercent = NewVGPRThresholdPercent;
2140 return true;
2141 }
2142
2143 return false;
2144}
2145
2146void LiveIntervalRPStage::finalizeGCNRegion() {
2147 S.VGPRExcessLimit = SavedVGPRExcessLimit;
2148 S.VGPRCriticalLimit = SavedVGPRCriticalLimit;
2149 S.UnifiedVGPRCriticalLimit = SavedUnifiedVGPRCriticalLimit;
2150 S.VGPRThresholdPercent = SavedVGPRThresholdPercent;
2151 GCNSchedStage::finalizeGCNRegion();
2152}
2153
2154bool GCNSchedStage::mayCauseSpilling(unsigned WavesAfter) {
2155 if (WavesAfter <= MFI.getMinWavesPerEU() && isRegionWithExcessRP() &&
2156 !PressureAfter.less(MF, O: PressureBefore)) {
2157 LLVM_DEBUG(dbgs() << "New pressure will result in more spilling.\n");
2158 return true;
2159 }
2160
2161 return false;
2162}
2163
2164void GCNSchedStage::modifyRegionSchedule(unsigned RegionIdx,
2165 ArrayRef<MachineInstr *> MIOrder) {
2166 assert(static_cast<size_t>(std::distance(DAG.Regions[RegionIdx].first,
2167 DAG.Regions[RegionIdx].second)) ==
2168 MIOrder.size() &&
2169 "instruction number mismatch");
2170 if (MIOrder.empty())
2171 return;
2172
2173 LLVM_DEBUG(dbgs() << "Reverting scheduling for region " << RegionIdx << '\n');
2174
2175 // Reconstruct MI sequence by moving instructions in desired order before
2176 // the current region's start.
2177 MachineBasicBlock::iterator RegionEnd = DAG.Regions[RegionIdx].first;
2178 MachineBasicBlock *MBB = MIOrder.front()->getParent();
2179 for (MachineInstr *MI : MIOrder) {
2180 // Either move the next MI in order before the end of the region or move the
2181 // region end past the MI if it is at the correct position.
2182 MachineBasicBlock::iterator MII = MI->getIterator();
2183 if (MII != RegionEnd) {
2184 // Will subsequent splice move MI up past a non-debug instruction?
2185 bool NonDebugReordered =
2186 !MI->isDebugInstr() &&
2187 skipDebugInstructionsForward(It: RegionEnd, End: MII) != MII;
2188 MBB->splice(Where: RegionEnd, Other: MBB, From: MI);
2189 // Only update LiveIntervals information if non-debug instructions are
2190 // reordered. Otherwise debug instructions could cause code generation to
2191 // change.
2192 if (NonDebugReordered)
2193 DAG.LIS->handleMove(MI&: *MI, UpdateFlags: true);
2194 } else {
2195 // MI is already at the expected position. However, earlier splices in
2196 // this loop may have changed neighboring slot indices, so this MI's
2197 // slot index can become non-monotonic w.r.t. the physical MBB order.
2198 // Only re-seat when monotonicity is actually violated to avoid
2199 // unnecessary LiveInterval changes that could perturb scheduling.
2200 if (!MI->isDebugInstr()) {
2201 SlotIndex MIIdx = DAG.LIS->getInstructionIndex(Instr: *MI);
2202 SlotIndex PrevIdx = DAG.LIS->getSlotIndexes()->getIndexBefore(MI: *MI);
2203 if (PrevIdx >= MIIdx)
2204 DAG.LIS->handleMove(MI&: *MI, UpdateFlags: true);
2205 }
2206 ++RegionEnd;
2207 }
2208 if (MI->isDebugInstr()) {
2209 LLVM_DEBUG(dbgs() << "Scheduling " << *MI);
2210 continue;
2211 }
2212
2213 // Reset read-undef flags and update them later.
2214 RegisterOperands::restoreLivenessFlags(MI&: *MI, TRI: *DAG.TRI, MRI: DAG.MRI, LIS&: *DAG.LIS,
2215 TrackLaneMasks: DAG.ShouldTrackLaneMasks);
2216 LLVM_DEBUG(dbgs() << "Scheduling " << *MI);
2217 }
2218
2219 // The region end doesn't change throughout scheduling since it itself is
2220 // outside the region (whether that is a MBB end or a terminator MI).
2221 assert(RegionEnd == DAG.Regions[RegionIdx].second && "region end mismatch");
2222 DAG.Regions[RegionIdx].first = MIOrder.front();
2223}
2224
2225/// Returns true if reaching def \p RD will be in AGPR form after the rewrite
2226/// and so needs no bridge copy: a candidate MFMA in \p RewriteSet, an
2227/// AV_MOV_*_IMM_PSEUDO, or a copy from a candidate src2 reg in \p CandSrc2Regs.
2228/// A non-candidate MFMA stays in VGPR form and still needs a bridge.
2229static bool isReachingDefAGPRForm(
2230 MachineInstr *RD, const SmallPtrSetImpl<MachineInstr *> &RewriteSet,
2231 const DenseSet<Register> &CandSrc2Regs, const SIInstrInfo &TII) {
2232 if (TII.isMAI(MI: *RD))
2233 return RewriteSet.contains(Ptr: RD);
2234 if (RD->getOpcode() == AMDGPU::AV_MOV_B32_IMM_PSEUDO ||
2235 RD->getOpcode() == AMDGPU::AV_MOV_B64_IMM_PSEUDO)
2236 return true;
2237 if (RD->isCopy() && CandSrc2Regs.contains(V: RD->getOperand(i: 1).getReg()))
2238 return true;
2239 return false;
2240}
2241
2242bool RewriteMFMAFormStage::hasUseRequiringVGPR(
2243 ArrayRef<SlotIndex> Src2ReachingDefs,
2244 const SmallPtrSetImpl<MachineInstr *> &RewriteSet) {
2245 for (SlotIndex RDIdx : Src2ReachingDefs) {
2246 const MachineInstr *RD = DAG.LIS->getInstructionFromIndex(index: RDIdx);
2247 SmallVector<MachineOperand *, 8> ReachingUses;
2248 findReachingUses(DefMI: RD, LIS: DAG.LIS, ReachingUses);
2249 for (const MachineOperand *UseMO : ReachingUses) {
2250 const MachineInstr *UseMI = UseMO->getParent();
2251 if (UseMI->isCopy())
2252 continue;
2253 if (TII->isMAI(MI: *UseMI) && RewriteSet.contains(Ptr: UseMI))
2254 continue;
2255 return true;
2256 }
2257 }
2258 return false;
2259}
2260
2261void RewriteMFMAFormStage::resetRewriteCandsToVGPR(
2262 ArrayRef<std::pair<MachineInstr *, unsigned>> RewriteCands) {
2263 for (auto [MI, OriginalOpcode] : RewriteCands) {
2264 assert(TII->isMAI(*MI));
2265 const TargetRegisterClass *ADefRC =
2266 DAG.MRI.getRegClass(Reg: MI->getOperand(i: 0).getReg());
2267 const TargetRegisterClass *VDefRC = SRI->getEquivalentVGPRClass(SRC: ADefRC);
2268 DAG.MRI.setRegClass(Reg: MI->getOperand(i: 0).getReg(), RC: VDefRC);
2269 MI->setDesc(TII->get(Opcode: OriginalOpcode));
2270
2271 MachineOperand *Src2 = TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src2);
2272 if (!Src2->isReg())
2273 continue;
2274
2275 // Have to get src types separately since subregs may cause C and D
2276 // registers to be different types even though the actual operand is
2277 // the same size.
2278 const TargetRegisterClass *AUseRC = DAG.MRI.getRegClass(Reg: Src2->getReg());
2279 const TargetRegisterClass *VUseRC = SRI->getEquivalentVGPRClass(SRC: AUseRC);
2280 DAG.MRI.setRegClass(Reg: Src2->getReg(), RC: VUseRC);
2281 }
2282}
2283
2284bool RewriteMFMAFormStage::isRewriteCandidate(MachineInstr *MI) const {
2285 if (!static_cast<const SIInstrInfo *>(DAG.TII)->isMAI(MI: *MI))
2286 return false;
2287 if (AMDGPU::getAGPRFormOp(Opcode: MI->getOpcode()) == -1)
2288 return false;
2289 // Reject candidates whose users force an unavoidable bridge copy.
2290 Register DstReg = MI->getOperand(i: 0).getReg();
2291 for (const MachineInstr &UseMI : DAG.MRI.use_nodbg_instructions(Reg: DstReg)) {
2292 if (!TII->isMAI(MI: UseMI) && !UseMI.isCopy())
2293 return false;
2294 }
2295 return true;
2296}
2297
2298bool RewriteMFMAFormStage::initHeuristics(
2299 std::vector<std::pair<MachineInstr *, unsigned>> &RewriteCands,
2300 DenseMap<MachineBasicBlock *, std::set<Register>> &CopyForUse,
2301 SmallPtrSetImpl<MachineInstr *> &CopyForDef) {
2302 bool Changed = false;
2303
2304 // Collect the candidate group, its members share AGPR-form operands
2305 // post-rewrite, so reaching defs feeding any member don't need bridge copy.
2306 SmallPtrSet<MachineInstr *, 16> RewriteSet;
2307 DenseSet<Register> CandSrc2Regs;
2308 for (MachineBasicBlock &MBB : MF) {
2309 for (MachineInstr &MI : MBB) {
2310 if (!isRewriteCandidate(MI: &MI))
2311 continue;
2312 RewriteSet.insert(Ptr: &MI);
2313 MachineOperand *Src2 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
2314 if (Src2 && Src2->isReg())
2315 CandSrc2Regs.insert(V: Src2->getReg());
2316 }
2317 }
2318
2319 // Prepare for the heuristics
2320 for (MachineBasicBlock &MBB : MF) {
2321 for (MachineInstr &MI : MBB) {
2322 if (!isRewriteCandidate(MI: &MI))
2323 continue;
2324
2325 int ReplacementOp = AMDGPU::getAGPRFormOp(Opcode: MI.getOpcode());
2326 assert(ReplacementOp != -1);
2327
2328 RewriteCands.push_back(x: {&MI, MI.getOpcode()});
2329 MI.setDesc(TII->get(Opcode: ReplacementOp));
2330
2331 MachineOperand *Src2 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
2332 if (Src2->isReg()) {
2333 SmallVector<SlotIndex, 8> Src2ReachingDefs;
2334 findReachingDefs(UseMO&: *Src2, LIS: DAG.LIS, DefIdxs&: Src2ReachingDefs);
2335
2336 // If src2 has a use that must remain VGPR, it cannot be reclassified to
2337 // AGPR.
2338 bool Src2NeedsVGPR = hasUseRequiringVGPR(Src2ReachingDefs, RewriteSet);
2339 Src2NeedsVGPRCache[&MI] = Src2NeedsVGPR;
2340
2341 for (SlotIndex RDIdx : Src2ReachingDefs) {
2342 MachineInstr *RD = DAG.LIS->getInstructionFromIndex(index: RDIdx);
2343 if (!Src2NeedsVGPR &&
2344 isReachingDefAGPRForm(RD, RewriteSet, CandSrc2Regs, TII: *TII))
2345 continue;
2346 CopyForDef.insert(Ptr: RD);
2347 }
2348 }
2349
2350 MachineOperand &Dst = MI.getOperand(i: 0);
2351 SmallVector<MachineOperand *, 8> DstReachingUses;
2352
2353 findReachingUses(DefMI: &MI, LIS: DAG.LIS, ReachingUses&: DstReachingUses);
2354
2355 for (MachineOperand *RUOp : DstReachingUses) {
2356 MachineInstr *UserMI = RUOp->getParent();
2357 // Group members read the AGPR result directly.
2358 if (TII->isMAI(MI: *UserMI) && RewriteSet.contains(Ptr: UserMI))
2359 continue;
2360
2361 // For any user of the result of the MFMA which is not an MFMA, we
2362 // insert a copy. For a given register, we will only insert one copy
2363 // per user block.
2364 CopyForUse[UserMI->getParent()].insert(x: RUOp->getReg());
2365
2366 if (TII->isMAI(MI: *UserMI))
2367 continue;
2368
2369 SmallVector<SlotIndex, 8> DstUsesReachingDefs;
2370 findReachingDefs(UseMO&: *RUOp, LIS: DAG.LIS, DefIdxs&: DstUsesReachingDefs);
2371
2372 for (SlotIndex RDIndex : DstUsesReachingDefs) {
2373 MachineInstr *RD = DAG.LIS->getInstructionFromIndex(index: RDIndex);
2374 if (TII->isMAI(MI: *RD))
2375 continue;
2376
2377 // For any definition of the user of the MFMA which is not an MFMA,
2378 // we insert a copy. We do this to transform all the reaching defs
2379 // of this use to AGPR. By doing this, we can insert a copy from
2380 // AGPR to VGPR at the user rather than after the MFMA.
2381 CopyForDef.insert(Ptr: RD);
2382 }
2383 }
2384
2385 // Do the rewrite to allow for updated RP calculation.
2386 const TargetRegisterClass *VDefRC = DAG.MRI.getRegClass(Reg: Dst.getReg());
2387 const TargetRegisterClass *ADefRC = SRI->getEquivalentAGPRClass(SRC: VDefRC);
2388 DAG.MRI.setRegClass(Reg: Dst.getReg(), RC: ADefRC);
2389 if (Src2->isReg()) {
2390 // Have to get src types separately since subregs may cause C and D
2391 // registers to be different types even though the actual operand is
2392 // the same size.
2393 const TargetRegisterClass *VUseRC = DAG.MRI.getRegClass(Reg: Src2->getReg());
2394 const TargetRegisterClass *AUseRC = SRI->getEquivalentAGPRClass(SRC: VUseRC);
2395 DAG.MRI.setRegClass(Reg: Src2->getReg(), RC: AUseRC);
2396 }
2397 Changed = true;
2398 }
2399 }
2400
2401 return Changed;
2402}
2403
2404int64_t RewriteMFMAFormStage::getRewriteCost(
2405 ArrayRef<std::pair<MachineInstr *, unsigned>> RewriteCands,
2406 const DenseMap<MachineBasicBlock *, std::set<Register>> &CopyForUse,
2407 const SmallPtrSetImpl<MachineInstr *> &CopyForDef) {
2408 MachineBlockFrequencyInfo *MBFI = DAG.MBFI;
2409
2410 int64_t BestSpillCost = 0;
2411 int64_t Cost = 0;
2412 uint64_t EntryFreq = MBFI->getEntryFreq().getFrequency();
2413
2414 std::pair<unsigned, unsigned> MaxVectorRegs =
2415 ST.getMaxNumVectorRegs(F: MF.getFunction());
2416 unsigned ArchVGPRThreshold = MaxVectorRegs.first;
2417 unsigned AGPRThreshold = MaxVectorRegs.second;
2418 unsigned CombinedThreshold = ST.getMaxNumVGPRs(MF);
2419
2420 for (unsigned Region = 0; Region < DAG.Regions.size(); Region++) {
2421 if (!RegionsWithExcessArchVGPR[Region])
2422 continue;
2423
2424 GCNRegPressure &PressureBefore = DAG.Pressure[Region];
2425 unsigned SpillCostBefore = PressureBefore.getVGPRSpills(
2426 MF, ArchVGPRThreshold, AGPRThreshold, CombinedThreshold);
2427
2428 // For the cases we care about (i.e. ArchVGPR usage is greater than the
2429 // addressable limit), rewriting alone should bring pressure to manageable
2430 // level. If we find any such region, then the rewrite is potentially
2431 // beneficial.
2432 GCNRegPressure PressureAfter = DAG.getRealRegPressure(RegionIdx: Region);
2433 unsigned SpillCostAfter = PressureAfter.getVGPRSpills(
2434 MF, ArchVGPRThreshold, AGPRThreshold, CombinedThreshold);
2435
2436 uint64_t BlockFreq =
2437 MBFI->getBlockFreq(MBB: DAG.Regions[Region].first->getParent())
2438 .getFrequency();
2439
2440 bool RelativeFreqIsDenom = EntryFreq > BlockFreq;
2441 uint64_t RelativeFreq = EntryFreq && BlockFreq
2442 ? (RelativeFreqIsDenom ? EntryFreq / BlockFreq
2443 : BlockFreq / EntryFreq)
2444 : 1;
2445
2446 // This assumes perfect spilling / splitting -- using one spill / copy
2447 // instruction and one restoreFrom / copy for each excess register,
2448 int64_t SpillCost = ((int)SpillCostAfter - (int)SpillCostBefore) * 2;
2449
2450 // Also account for the block frequency.
2451 if (RelativeFreqIsDenom)
2452 SpillCost /= (int64_t)RelativeFreq;
2453 else
2454 SpillCost *= (int64_t)RelativeFreq;
2455
2456 // If we have increased spilling in any block, just bail.
2457 if (SpillCost > 0) {
2458 resetRewriteCandsToVGPR(RewriteCands);
2459 return SpillCost;
2460 }
2461
2462 if (SpillCost < BestSpillCost)
2463 BestSpillCost = SpillCost;
2464 }
2465
2466 // Set the cost to the largest decrease in spill cost in order to not double
2467 // count spill reductions.
2468 Cost = BestSpillCost;
2469 assert(Cost <= 0);
2470
2471 unsigned CopyCost = 0;
2472
2473 // For each CopyForDef, increase the cost by the register size while
2474 // accounting for block frequency.
2475 for (MachineInstr *DefMI : CopyForDef) {
2476 Register DefReg = DefMI->getOperand(i: 0).getReg();
2477 uint64_t DefFreq =
2478 EntryFreq
2479 ? MBFI->getBlockFreq(MBB: DefMI->getParent()).getFrequency() / EntryFreq
2480 : 1;
2481
2482 const TargetRegisterClass *RC = DAG.MRI.getRegClass(Reg: DefReg);
2483 CopyCost += static_cast<unsigned>(RC->getCopyCost() * DefFreq);
2484 }
2485
2486 // Account for CopyForUse copies in each block that the register is used.
2487 for (auto &[UseBlock, UseRegs] : CopyForUse) {
2488 uint64_t UseFreq =
2489 EntryFreq ? MBFI->getBlockFreq(MBB: UseBlock).getFrequency() / EntryFreq : 1;
2490
2491 for (Register UseReg : UseRegs) {
2492 const TargetRegisterClass *RC = DAG.MRI.getRegClass(Reg: UseReg);
2493 CopyCost += static_cast<unsigned>(RC->getCopyCost() * UseFreq);
2494 }
2495 }
2496
2497 // Reset the classes that were changed to AGPR for better register bank
2498 // analysis. We must do rewriting after copy-insertion, as some defs of the
2499 // register may require VGPR. Additionally, if we bail out and don't perform
2500 // the rewrite then these need to be restored anyway.
2501 resetRewriteCandsToVGPR(RewriteCands);
2502
2503 return Cost + CopyCost;
2504}
2505
2506bool RewriteMFMAFormStage::rewrite(
2507 ArrayRef<std::pair<MachineInstr *, unsigned>> RewriteCands) {
2508 DenseMap<MachineInstr *, unsigned> FirstMIToRegion;
2509 DenseMap<MachineInstr *, unsigned> LastMIToRegion;
2510
2511 for (unsigned Region = 0; Region < DAG.Regions.size(); Region++) {
2512 RegionBoundaries Entry = DAG.Regions[Region];
2513 if (Entry.first == Entry.second)
2514 continue;
2515
2516 FirstMIToRegion[&*Entry.first] = Region;
2517 if (Entry.second != Entry.first->getParent()->end())
2518 LastMIToRegion[&*Entry.second] = Region;
2519 }
2520
2521 // Rewrite the MFMAs to AGPR, and insert any copies as needed.
2522 // The general assumption of the algorithm (and the previous cost calculation)
2523 // is that it is better to insert the copies in the MBB of the def of the src2
2524 // operands, and in the MBB of the user of the dest operands. This is based on
2525 // the assumption that the MFMAs are likely to appear in loop bodies, while
2526 // the src2 and dest operands are live-in / live-out of the loop. Due to this
2527 // design, the algorithm for finding copy insertion points is more
2528 // complicated.
2529 //
2530 // There are three main cases to handle: 1. the reaching defs of the src2
2531 // operands, 2. the reaching uses of the dst operands, and 3. the reaching
2532 // defs of the reaching uses of the dst operand.
2533 //
2534 // In the first case, we simply insert copies after each of the reaching
2535 // definitions. In the second case, we collect all the uses of a given dest
2536 // and organize them by MBB. Then, we insert 1 copy for each MBB before the
2537 // earliest use. Since the use may have multiple reaching defs, and since we
2538 // want to replace the register it is using with the result of the copy, we
2539 // must handle case 3. In the third case, we simply insert a copy after each
2540 // of the reaching defs to connect to the copy of the reaching uses of the dst
2541 // reg. This allows us to avoid inserting copies next to the MFMAs.
2542 //
2543 // While inserting the copies, we maintain a map of operands which will use
2544 // different regs (i.e. the result of the copies). For example, a case 1 src2
2545 // operand will use the register result of the copies after the reaching defs,
2546 // as opposed to the original register. Now that we have completed our copy
2547 // analysis and placement, we can bulk update the registers. We do this
2548 // separately as to avoid complicating the reachingDef and reachingUse
2549 // queries.
2550 //
2551 // While inserting the copies, we also maintain a list or registers which we
2552 // will want to reclassify as AGPR. After doing the copy insertion and the
2553 // register replacement, we can finally do the reclassification. This uses the
2554 // redef map, as the registers we are interested in reclassifying may be
2555 // replaced by the result of a copy. We must do this after the copy analysis
2556 // and placement as we must have an accurate redef map -- otherwise we may end
2557 // up creating illegal instructions.
2558
2559 // The original registers of the MFMA that need to be reclassified as AGPR.
2560 DenseSet<Register> RewriteRegs;
2561 // The map of an original register in the MFMA to a new register (result of a
2562 // copy) that it should be replaced with.
2563 DenseMap<Register, Register> RedefMap;
2564 // The map of the original MFMA registers to the relevant MFMA operands.
2565 DenseMap<Register, DenseSet<MachineOperand *>> ReplaceMap;
2566 // The map of reaching defs for a given register -- to avoid duplicate copies.
2567 DenseMap<Register, SmallPtrSet<MachineInstr *, 8>> ReachingDefCopyMap;
2568 // The map of reaching uses for a given register by basic block -- to avoid
2569 // duplicate copies and to calculate per MBB insert pts.
2570 DenseMap<unsigned, DenseMap<Register, SmallPtrSet<MachineOperand *, 8>>>
2571 ReachingUseTracker;
2572
2573 // Collect the candidate group; its members share AGPR-form operands
2574 // post-rewrite, so reaching defs feeding any member need no bridge copy.
2575 SmallPtrSet<MachineInstr *, 16> RewriteCandsSet;
2576 DenseSet<Register> RewriteSrc2Regs;
2577 for (auto &[MI, OriginalOpcode] : RewriteCands) {
2578 RewriteCandsSet.insert(Ptr: MI);
2579 MachineOperand *Src2 = TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src2);
2580 if (Src2 && Src2->isReg())
2581 RewriteSrc2Regs.insert(V: Src2->getReg());
2582 }
2583
2584 for (auto &[MI, OriginalOpcode] : RewriteCands) {
2585 int ReplacementOp = AMDGPU::getAGPRFormOp(Opcode: MI->getOpcode());
2586 if (ReplacementOp == -1)
2587 continue;
2588 MI->setDesc(TII->get(Opcode: ReplacementOp));
2589
2590 // Case 1: insert copies for the reaching defs of the Src2Reg.
2591 MachineOperand *Src2 = TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::src2);
2592 if (Src2->isReg()) {
2593 Register Src2Reg = Src2->getReg();
2594 if (!Src2Reg.isVirtual())
2595 return false;
2596
2597 Register MappedReg = Src2->getReg();
2598 SmallVector<SlotIndex, 8> Src2ReachingDefs;
2599 findReachingDefs(UseMO&: *Src2, LIS: DAG.LIS, DefIdxs&: Src2ReachingDefs);
2600 SmallSetVector<MachineInstr *, 8> Src2DefsReplace;
2601
2602 // If src2 has a use that must remain VGPR, it cannot be reclassified to
2603 // AGPR.
2604 bool Src2NeedsVGPR = Src2NeedsVGPRCache.lookup(Val: MI);
2605
2606 for (SlotIndex RDIndex : Src2ReachingDefs) {
2607 MachineInstr *RD = DAG.LIS->getInstructionFromIndex(index: RDIndex);
2608 if (!Src2NeedsVGPR &&
2609 isReachingDefAGPRForm(RD, RewriteSet: RewriteCandsSet, CandSrc2Regs: RewriteSrc2Regs, TII: *TII))
2610 continue;
2611
2612 Src2DefsReplace.insert(X: RD);
2613 }
2614
2615 if (!Src2DefsReplace.empty()) {
2616 auto RI = RedefMap.find(Val: Src2Reg);
2617 if (RI != RedefMap.end()) {
2618 MappedReg = RI->second;
2619 } else {
2620 assert(!ReachingDefCopyMap.contains(Src2Reg));
2621 const TargetRegisterClass *Src2RC = DAG.MRI.getRegClass(Reg: Src2Reg);
2622 const TargetRegisterClass *VGPRRC =
2623 SRI->getEquivalentVGPRClass(SRC: Src2RC);
2624
2625 // Track the mapping of the original register to the new register.
2626 MappedReg = DAG.MRI.createVirtualRegister(RegClass: VGPRRC);
2627 RedefMap[Src2Reg] = MappedReg;
2628 }
2629
2630 // If none exists, create a copy from this reaching def.
2631 // We may have inserted a copy already in an earlier iteration.
2632 for (MachineInstr *RD : Src2DefsReplace) {
2633 // Do not create redundant copies.
2634 if (ReachingDefCopyMap[Src2Reg].insert(Ptr: RD).second) {
2635 MachineInstrBuilder VGPRCopy =
2636 BuildMI(BB&: *RD->getParent(), I: std::next(x: RD->getIterator()),
2637 MIMD: RD->getDebugLoc(), MCID: TII->get(Opcode: TargetOpcode::COPY))
2638 .addDef(RegNo: MappedReg, Flags: {}, SubReg: 0)
2639 .addUse(RegNo: Src2Reg, Flags: {}, SubReg: 0);
2640 DAG.LIS->InsertMachineInstrInMaps(MI&: *VGPRCopy);
2641
2642 // If this reaching def was the last MI in the region, update the
2643 // region boundaries.
2644 if (LastMIToRegion.contains(Val: RD)) {
2645 unsigned UpdateRegion = LastMIToRegion[RD];
2646 DAG.Regions[UpdateRegion].second = VGPRCopy;
2647 LastMIToRegion.erase(Val: RD);
2648 }
2649 }
2650 }
2651 }
2652
2653 // Track the register for reclassification
2654 RewriteRegs.insert(V: Src2Reg);
2655
2656 // Always insert the operand for replacement. If this corresponds with a
2657 // chain of tied-def we may not see the VGPR requirement until later.
2658 ReplaceMap[Src2Reg].insert(V: Src2);
2659 }
2660
2661 // Case 2 and Case 3: insert copies before the reaching uses of the dsts,
2662 // and after the reaching defs of the reaching uses of the dsts.
2663
2664 MachineOperand *Dst = &MI->getOperand(i: 0);
2665 Register DstReg = Dst->getReg();
2666 if (!DstReg.isVirtual())
2667 return false;
2668
2669 Register MappedReg = DstReg;
2670 SmallVector<MachineOperand *, 8> DstReachingUses;
2671
2672 SmallVector<MachineOperand *, 8> DstReachingUseCopies;
2673 SmallVector<MachineInstr *, 8> DstUseDefsReplace;
2674
2675 findReachingUses(DefMI: MI, LIS: DAG.LIS, ReachingUses&: DstReachingUses);
2676
2677 for (MachineOperand *RUOp : DstReachingUses) {
2678 MachineInstr *UserMI = RUOp->getParent();
2679 // Group members read the AGPR result directly.
2680 if (TII->isMAI(MI: *UserMI) && RewriteCandsSet.contains(Ptr: UserMI))
2681 continue;
2682
2683 // If there is a non mai reaching use, then we need a copy.
2684 if (find(Range&: DstReachingUseCopies, Val: RUOp) == DstReachingUseCopies.end())
2685 DstReachingUseCopies.push_back(Elt: RUOp);
2686
2687 // Non-rewritten MAI: its defs aren't being reclassified.
2688 if (TII->isMAI(MI: *UserMI))
2689 continue;
2690
2691 SmallVector<SlotIndex, 8> DstUsesReachingDefs;
2692 findReachingDefs(UseMO&: *RUOp, LIS: DAG.LIS, DefIdxs&: DstUsesReachingDefs);
2693
2694 for (SlotIndex RDIndex : DstUsesReachingDefs) {
2695 MachineInstr *RD = DAG.LIS->getInstructionFromIndex(index: RDIndex);
2696 if (TII->isMAI(MI: *RD))
2697 continue;
2698
2699 // If there is a non mai reaching def of this reaching use, then we will
2700 // need a copy.
2701 if (find(Range&: DstUseDefsReplace, Val: RD) == DstUseDefsReplace.end())
2702 DstUseDefsReplace.push_back(Elt: RD);
2703 }
2704 }
2705
2706 if (!DstUseDefsReplace.empty()) {
2707 auto RI = RedefMap.find(Val: DstReg);
2708 if (RI != RedefMap.end()) {
2709 MappedReg = RI->second;
2710 } else {
2711 assert(!ReachingDefCopyMap.contains(DstReg));
2712 const TargetRegisterClass *DstRC = DAG.MRI.getRegClass(Reg: DstReg);
2713 const TargetRegisterClass *VGPRRC = SRI->getEquivalentVGPRClass(SRC: DstRC);
2714
2715 // Track the mapping of the original register to the new register.
2716 MappedReg = DAG.MRI.createVirtualRegister(RegClass: VGPRRC);
2717 RedefMap[DstReg] = MappedReg;
2718 }
2719
2720 // If none exists, create a copy from this reaching def.
2721 // We may have inserted a copy already in an earlier iteration.
2722 for (MachineInstr *RD : DstUseDefsReplace) {
2723 // Do not create reundant copies.
2724 if (ReachingDefCopyMap[DstReg].insert(Ptr: RD).second) {
2725 MachineInstrBuilder VGPRCopy =
2726 BuildMI(BB&: *RD->getParent(), I: std::next(x: RD->getIterator()),
2727 MIMD: RD->getDebugLoc(), MCID: TII->get(Opcode: TargetOpcode::COPY))
2728 .addDef(RegNo: MappedReg, Flags: {}, SubReg: 0)
2729 .addUse(RegNo: DstReg, Flags: {}, SubReg: 0);
2730 DAG.LIS->InsertMachineInstrInMaps(MI&: *VGPRCopy);
2731
2732 // If this reaching def was the last MI in the region, update the
2733 // region boundaries.
2734 auto LMI = LastMIToRegion.find(Val: RD);
2735 if (LMI != LastMIToRegion.end()) {
2736 unsigned UpdateRegion = LMI->second;
2737 DAG.Regions[UpdateRegion].second = VGPRCopy;
2738 LastMIToRegion.erase(Val: RD);
2739 }
2740 }
2741 }
2742 }
2743
2744 DenseSet<MachineOperand *> &DstRegSet = ReplaceMap[DstReg];
2745 // One AGPR→VGPR copy per dst register, shared by all same-block uses.
2746 Register SameBlockCopyReg;
2747 MachineInstr *EarliestSameBlockUse = nullptr;
2748 for (MachineOperand *RU : DstReachingUseCopies) {
2749 MachineBasicBlock *RUBlock = RU->getParent()->getParent();
2750 // Just keep track of the reaching use of this register by block. After we
2751 // have scanned all the MFMAs we can find optimal insert pts.
2752 if (RUBlock != MI->getParent()) {
2753 ReachingUseTracker[RUBlock->getNumber()][DstReg].insert(Ptr: RU);
2754 continue;
2755 }
2756
2757 // Lazily create the copy register on first same-block use.
2758 if (!SameBlockCopyReg.isValid()) {
2759 const TargetRegisterClass *DstRC = DAG.MRI.getRegClass(Reg: DstReg);
2760 const TargetRegisterClass *VGPRRC = SRI->getEquivalentVGPRClass(SRC: DstRC);
2761 SameBlockCopyReg = DAG.MRI.createVirtualRegister(RegClass: VGPRRC);
2762 }
2763
2764 // Track the earliest use for copy insertion point.
2765 MachineInstr *UseInst = RU->getParent();
2766 if (!EarliestSameBlockUse ||
2767 SlotIndex::isEarlierInstr(
2768 A: DAG.LIS->getInstructionIndex(Instr: *UseInst),
2769 B: DAG.LIS->getInstructionIndex(Instr: *EarliestSameBlockUse)))
2770 EarliestSameBlockUse = UseInst;
2771 RU->setReg(SameBlockCopyReg);
2772 }
2773
2774 // Insert the copy before the earliest same-block use.
2775 if (SameBlockCopyReg.isValid()) {
2776 MachineInstrBuilder VGPRCopy =
2777 BuildMI(BB&: *EarliestSameBlockUse->getParent(),
2778 I: EarliestSameBlockUse->getIterator(), MIMD: DebugLoc(),
2779 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: SameBlockCopyReg)
2780 .addUse(RegNo: DstReg, Flags: {}, SubReg: 0);
2781 DAG.LIS->InsertMachineInstrInMaps(MI&: *VGPRCopy);
2782 DstRegSet.insert(V: &VGPRCopy->getOperand(i: 1));
2783 }
2784
2785 // Track the register for reclassification
2786 RewriteRegs.insert(V: DstReg);
2787
2788 // Insert the dst operand for replacement. If this dst is in a chain of
2789 // tied-def MFMAs, and the first src2 needs to be replaced with a new reg,
2790 // all the correspond operands need to be replaced.
2791 DstRegSet.insert(V: Dst);
2792 }
2793
2794 // Handle the copies for dst uses.
2795 using RUBType =
2796 std::pair<unsigned, DenseMap<Register, SmallPtrSet<MachineOperand *, 8>>>;
2797 for (RUBType RUBlockEntry : ReachingUseTracker) {
2798 using RUDType = std::pair<Register, SmallPtrSet<MachineOperand *, 8>>;
2799 for (RUDType RUDst : RUBlockEntry.second) {
2800 MachineOperand *OpBegin = *RUDst.second.begin();
2801 SlotIndex InstPt = DAG.LIS->getInstructionIndex(Instr: *OpBegin->getParent());
2802
2803 // Find the earliest use in this block.
2804 for (MachineOperand *User : RUDst.second) {
2805 SlotIndex NewInstPt = DAG.LIS->getInstructionIndex(Instr: *User->getParent());
2806 if (SlotIndex::isEarlierInstr(A: NewInstPt, B: InstPt))
2807 InstPt = NewInstPt;
2808 }
2809
2810 const TargetRegisterClass *DstRC = DAG.MRI.getRegClass(Reg: RUDst.first);
2811 const TargetRegisterClass *VGPRRC = SRI->getEquivalentVGPRClass(SRC: DstRC);
2812 Register NewUseReg = DAG.MRI.createVirtualRegister(RegClass: VGPRRC);
2813 MachineInstr *UseInst = DAG.LIS->getInstructionFromIndex(index: InstPt);
2814
2815 MachineInstrBuilder VGPRCopy =
2816 BuildMI(BB&: *UseInst->getParent(), I: UseInst->getIterator(),
2817 MIMD: UseInst->getDebugLoc(), MCID: TII->get(Opcode: TargetOpcode::COPY))
2818 .addDef(RegNo: NewUseReg, Flags: {}, SubReg: 0)
2819 .addUse(RegNo: RUDst.first, Flags: {}, SubReg: 0);
2820 DAG.LIS->InsertMachineInstrInMaps(MI&: *VGPRCopy);
2821
2822 // If this UseInst was the first MI in the region, update the region
2823 // boundaries.
2824 auto FI = FirstMIToRegion.find(Val: UseInst);
2825 if (FI != FirstMIToRegion.end()) {
2826 unsigned UpdateRegion = FI->second;
2827 DAG.Regions[UpdateRegion].first = VGPRCopy;
2828 FirstMIToRegion.erase(Val: UseInst);
2829 }
2830
2831 // Replace the operand for all users.
2832 for (MachineOperand *User : RUDst.second) {
2833 User->setReg(NewUseReg);
2834 }
2835
2836 // Track the copy source operand for replacement.
2837 ReplaceMap[RUDst.first].insert(V: &VGPRCopy->getOperand(i: 1));
2838 }
2839 }
2840
2841 // We may have needed to insert copies after the reaching defs of the MFMAs.
2842 // Replace the original register with the result of the copy for all relevant
2843 // operands.
2844 for (std::pair<Register, Register> NewDef : RedefMap) {
2845 Register OldReg = NewDef.first;
2846 Register NewReg = NewDef.second;
2847
2848 // Replace the register for any associated operand in the MFMA chain.
2849 for (MachineOperand *ReplaceOp : ReplaceMap[OldReg])
2850 ReplaceOp->setReg(NewReg);
2851 }
2852
2853 // Finally, do the reclassification of the MFMA registers.
2854 for (Register RewriteReg : RewriteRegs) {
2855 Register RegToRewrite = RewriteReg;
2856
2857 // Be sure to update the replacement register and not the original.
2858 auto RI = RedefMap.find(Val: RewriteReg);
2859 if (RI != RedefMap.end())
2860 RegToRewrite = RI->second;
2861
2862 const TargetRegisterClass *CurrRC = DAG.MRI.getRegClass(Reg: RegToRewrite);
2863 const TargetRegisterClass *AGPRRC = SRI->getEquivalentAGPRClass(SRC: CurrRC);
2864
2865 DAG.MRI.setRegClass(Reg: RegToRewrite, RC: AGPRRC);
2866 }
2867
2868 // Bulk update the LIS.
2869 DAG.LIS->reanalyze(MF&: DAG.MF);
2870 // Liveins may have been modified for cross RC copies
2871 RegionPressureMap LiveInUpdater(&DAG, false);
2872 LiveInUpdater.buildLiveRegMap();
2873
2874 for (unsigned Region = 0; Region < DAG.Regions.size(); Region++)
2875 DAG.LiveIns[Region] = LiveInUpdater.getLiveRegsForRegionIdx(RegionIdx: Region);
2876
2877 DAG.Pressure[RegionIdx] = DAG.getRealRegPressure(RegionIdx);
2878
2879 return true;
2880}
2881
2882unsigned PreRARematStage::getStageTargetOccupancy() const {
2883 return TargetOcc ? *TargetOcc : MFI.getMinWavesPerEU();
2884}
2885
2886void PreRARematStage::finalizeGCNSchedStage() {
2887 // We consider that reducing spilling is always beneficial so we never
2888 // rollback rematerializations or revert scheduling in such cases.
2889 if (!TargetOcc)
2890 return;
2891
2892 // When increasing occupancy, it is possible that re-scheduling is not able to
2893 // achieve the target occupancy in all regions, in which case re-scheduling in
2894 // all regions should be reverted.
2895 if (DAG.MinOccupancy >= *TargetOcc)
2896 return;
2897
2898 // Revert re-scheduling in all affected regions.
2899 for (const auto &[RegionIdx, OrigMIOrder, MaxPressure] : RegionReverts) {
2900 LLVM_DEBUG(dbgs() << "Reverting re-scheduling in region " << RegionIdx
2901 << '\n');
2902 DAG.Pressure[RegionIdx] = MaxPressure;
2903 modifyRegionSchedule(RegionIdx, MIOrder: OrigMIOrder);
2904 }
2905
2906 // It is possible that re-scheduling lowers occupancy over the one achieved
2907 // just through rematerializations, in which case we revert re-scheduling in
2908 // all regions but do not roll back rematerializations.
2909 if (AchievedOcc >= *TargetOcc) {
2910 DAG.setTargetOccupancy(AchievedOcc);
2911 return;
2912 }
2913
2914 // Reset the target occupancy to what it was pre-rematerialization.
2915 DAG.setTargetOccupancy(*TargetOcc - 1);
2916 Strategy->rollbackRecordedRemats();
2917 for (unsigned RegionIdx : RescheduleRegions.set_bits()) {
2918 DAG.RegionLiveOuts.getLiveRegsForRegionIdx(RegionIdx) = LiveOuts[RegionIdx];
2919 DAG.Pressure[RegionIdx] = DAG.getRealRegPressure(RegionIdx);
2920 }
2921
2922#ifdef EXPENSIVE_CHECKS
2923 // In particular, we want to check for coherent MI/slot order in regions in
2924 // which reverts and/or rollbacks may have happened.
2925 MF.verify();
2926#endif
2927
2928 GCNSchedStage::finalizeGCNSchedStage();
2929}
2930
2931void GCNScheduleDAGMILive::setTargetOccupancy(unsigned TargetOccupancy) {
2932 MinOccupancy = TargetOccupancy;
2933 if (MFI.getOccupancy() < TargetOccupancy)
2934 MFI.increaseOccupancy(MF, Limit: MinOccupancy);
2935 else
2936 MFI.limitOccupancy(Limit: MinOccupancy);
2937}
2938
2939static bool hasIGLPInstrs(ScheduleDAGInstrs *DAG) {
2940 const SIInstrInfo *SII = static_cast<const SIInstrInfo *>(DAG->TII);
2941 return any_of(Range&: *DAG, P: [SII](MachineBasicBlock::iterator MI) {
2942 return SII->isIGLPMutationOnly(Opcode: MI->getOpcode());
2943 });
2944}
2945
2946GCNPostScheduleDAGMILive::GCNPostScheduleDAGMILive(
2947 MachineSchedContext *C, std::unique_ptr<MachineSchedStrategy> S,
2948 bool RemoveKillFlags)
2949 : ScheduleDAGMI(C, std::move(S), RemoveKillFlags) {}
2950
2951void GCNPostScheduleDAGMILive::schedule() {
2952 HasIGLPInstrs = hasIGLPInstrs(DAG: this);
2953 if (HasIGLPInstrs) {
2954 SavedMutations.clear();
2955 SavedMutations.swap(x&: Mutations);
2956 addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::PostRA));
2957 }
2958
2959 ScheduleDAGMI::schedule();
2960}
2961
2962void GCNPostScheduleDAGMILive::finalizeSchedule() {
2963 if (HasIGLPInstrs)
2964 SavedMutations.swap(x&: Mutations);
2965
2966 ScheduleDAGMI::finalizeSchedule();
2967}
2968