1//===- GCNRegPressure.cpp -------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the GCNRegPressure class.
11///
12//===----------------------------------------------------------------------===//
13
14#include "GCNRegPressure.h"
15#include "AMDGPU.h"
16#include "SIMachineFunctionInfo.h"
17#include "llvm/ADT/SetVector.h"
18#include "llvm/CodeGen/LiveIntervalUnion.h"
19#include "llvm/CodeGen/MachineBasicBlock.h"
20#include "llvm/CodeGen/MachineLoopInfo.h"
21#include "llvm/CodeGen/RegisterPressure.h"
22
23using namespace llvm;
24
25#define DEBUG_TYPE "machine-scheduler"
26
27bool llvm::isEqual(const GCNRPTracker::LiveRegSet &S1,
28 const GCNRPTracker::LiveRegSet &S2) {
29 if (S1.size() != S2.size())
30 return false;
31
32 for (const auto &P : S1) {
33 auto I = S2.find(Val: P.first);
34 if (I == S2.end() || I->second != P.second)
35 return false;
36 }
37 return true;
38}
39
40///////////////////////////////////////////////////////////////////////////////
41// GCNRegPressure
42
43unsigned GCNRegPressure::getRegKind(const TargetRegisterClass *RC,
44 const SIRegisterInfo *STI) {
45 return STI->isSGPRClass(RC)
46 ? SGPR
47 : (STI->isAGPRClass(RC)
48 ? AGPR
49 : (STI->isVectorSuperClass(RC) ? AVGPR : VGPR));
50}
51
52void GCNRegPressure::inc(unsigned Reg,
53 LaneBitmask PrevMask,
54 LaneBitmask NewMask,
55 const MachineRegisterInfo &MRI) {
56 unsigned NewNumCoveredRegs = SIRegisterInfo::getNumCoveredRegs(LM: NewMask);
57 unsigned PrevNumCoveredRegs = SIRegisterInfo::getNumCoveredRegs(LM: PrevMask);
58 if (NewNumCoveredRegs == PrevNumCoveredRegs)
59 return;
60
61 int Sign = 1;
62 if (NewMask < PrevMask) {
63 std::swap(a&: NewMask, b&: PrevMask);
64 std::swap(a&: NewNumCoveredRegs, b&: PrevNumCoveredRegs);
65 Sign = -1;
66 }
67 assert(PrevMask < NewMask && PrevNumCoveredRegs < NewNumCoveredRegs &&
68 "prev mask should always be lesser than new");
69
70 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
71 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
72 const SIRegisterInfo *STI = static_cast<const SIRegisterInfo *>(TRI);
73 unsigned RegKind = getRegKind(RC, STI);
74 if (TRI->getRegSizeInBits(RC: *RC) != 32) {
75 // Reg is from a tuple register class.
76 if (PrevMask.none()) {
77 unsigned TupleIdx = TOTAL_KINDS + RegKind;
78 Value[TupleIdx] += Sign * TRI->getRegClassWeight(RC).RegWeight;
79 }
80 // Pressure scales with number of new registers covered by the new mask.
81 // Note when true16 is enabled, we can no longer safely use the following
82 // approach to calculate the difference in the number of 32-bit registers
83 // between two masks:
84 //
85 // Sign *= SIRegisterInfo::getNumCoveredRegs(~PrevMask & NewMask);
86 //
87 // The issue is that the mask calculation `~PrevMask & NewMask` doesn't
88 // properly account for partial usage of a 32-bit register when dealing with
89 // 16-bit registers.
90 //
91 // Consider this example:
92 // Assume PrevMask = 0b0010 and NewMask = 0b1111. Here, the correct register
93 // usage difference should be 1, because even though PrevMask uses only half
94 // of a 32-bit register, it should still be counted as a full register use.
95 // However, the mask calculation yields `~PrevMask & NewMask = 0b1101`, and
96 // calling `getNumCoveredRegs` returns 2 instead of 1. This incorrect
97 // calculation can lead to integer overflow when Sign = -1.
98 Sign *= NewNumCoveredRegs - PrevNumCoveredRegs;
99 }
100 Value[RegKind] += Sign;
101}
102
103namespace {
104struct RegExcess {
105 unsigned SGPR = 0;
106 unsigned VGPR = 0;
107 unsigned ArchVGPR = 0;
108 unsigned AGPR = 0;
109
110 bool anyExcess() const { return SGPR || VGPR || ArchVGPR || AGPR; }
111 bool hasVectorRegisterExcess() const { return VGPR || ArchVGPR || AGPR; }
112
113 RegExcess(const MachineFunction &MF, const GCNRegPressure &RP)
114 : RegExcess(MF, RP, GCNRPTarget(MF, RP)) {}
115 RegExcess(const MachineFunction &MF, const GCNRegPressure &RP,
116 const GCNRPTarget &Target) {
117 unsigned MaxSGPRs = Target.getMaxSGPRs();
118 unsigned MaxVGPRs = Target.getMaxVGPRs();
119
120 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
121 SGPR = std::max(a: static_cast<int>(RP.getSGPRNum() - MaxSGPRs), b: 0);
122
123 // The number of virtual VGPRs required to handle excess SGPR
124 unsigned WaveSize = ST.getWavefrontSize();
125 unsigned VGPRForSGPRSpills = divideCeil(Numerator: SGPR, Denominator: WaveSize);
126
127 unsigned MaxArchVGPRs = ST.getAddressableNumArchVGPRs();
128
129 // Unified excess pressure conditions, accounting for VGPRs used for SGPR
130 // spills
131 VGPR = std::max(a: static_cast<int>(RP.getVGPRNum(UnifiedVGPRFile: ST.hasGFX90AInsts()) +
132 VGPRForSGPRSpills - MaxVGPRs),
133 b: 0);
134
135 unsigned ArchVGPRLimit = ST.hasGFX90AInsts() ? MaxArchVGPRs : MaxVGPRs;
136 // Arch VGPR excess pressure conditions, accounting for VGPRs used for SGPR
137 // spills
138 ArchVGPR = std::max(a: static_cast<int>(RP.getArchVGPRNum() +
139 VGPRForSGPRSpills - ArchVGPRLimit),
140 b: 0);
141
142 // AGPR excess pressure conditions
143 AGPR = std::max(a: static_cast<int>(RP.getAGPRNum() - ArchVGPRLimit), b: 0);
144 }
145};
146} // namespace
147
148bool GCNRegPressure::less(const MachineFunction &MF, const GCNRegPressure &O,
149 unsigned MaxOccupancy) const {
150 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
151 unsigned DynamicVGPRBlockSize =
152 MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize();
153
154 const auto SGPROcc = std::min(a: MaxOccupancy,
155 b: ST.getOccupancyWithNumSGPRs(SGPRs: getSGPRNum()));
156 const auto VGPROcc = std::min(
157 a: MaxOccupancy, b: ST.getOccupancyWithNumVGPRs(VGPRs: getVGPRNum(UnifiedVGPRFile: ST.hasGFX90AInsts()),
158 DynamicVGPRBlockSize));
159 const auto OtherSGPROcc = std::min(a: MaxOccupancy,
160 b: ST.getOccupancyWithNumSGPRs(SGPRs: O.getSGPRNum()));
161 const auto OtherVGPROcc =
162 std::min(a: MaxOccupancy,
163 b: ST.getOccupancyWithNumVGPRs(VGPRs: O.getVGPRNum(UnifiedVGPRFile: ST.hasGFX90AInsts()),
164 DynamicVGPRBlockSize));
165
166 const auto Occ = std::min(a: SGPROcc, b: VGPROcc);
167 const auto OtherOcc = std::min(a: OtherSGPROcc, b: OtherVGPROcc);
168
169 // Give first precedence to the better occupancy.
170 if (Occ != OtherOcc)
171 return Occ > OtherOcc;
172
173 unsigned MaxVGPRs = ST.getMaxNumVGPRs(MF);
174
175 RegExcess Excess(MF, *this);
176 RegExcess OtherExcess(MF, O);
177
178 unsigned MaxArchVGPRs = ST.getAddressableNumArchVGPRs();
179
180 bool ExcessRP = Excess.anyExcess();
181 bool OtherExcessRP = OtherExcess.anyExcess();
182
183 // Give second precedence to the reduced number of spills to hold the register
184 // pressure.
185 if (ExcessRP || OtherExcessRP) {
186 // The difference in excess VGPR pressure, after including VGPRs used for
187 // SGPR spills
188 int VGPRDiff =
189 ((OtherExcess.VGPR + OtherExcess.ArchVGPR + OtherExcess.AGPR) -
190 (Excess.VGPR + Excess.ArchVGPR + Excess.AGPR));
191
192 int SGPRDiff = OtherExcess.SGPR - Excess.SGPR;
193
194 if (VGPRDiff != 0)
195 return VGPRDiff > 0;
196 if (SGPRDiff != 0) {
197 unsigned PureExcessVGPR =
198 std::max(a: static_cast<int>(getVGPRNum(UnifiedVGPRFile: ST.hasGFX90AInsts()) - MaxVGPRs),
199 b: 0) +
200 std::max(a: static_cast<int>(getVGPRNum(UnifiedVGPRFile: false) - MaxArchVGPRs), b: 0);
201 unsigned OtherPureExcessVGPR =
202 std::max(
203 a: static_cast<int>(O.getVGPRNum(UnifiedVGPRFile: ST.hasGFX90AInsts()) - MaxVGPRs),
204 b: 0) +
205 std::max(a: static_cast<int>(O.getVGPRNum(UnifiedVGPRFile: false) - MaxArchVGPRs), b: 0);
206
207 // If we have a special case where there is a tie in excess VGPR, but one
208 // of the pressures has VGPR usage from SGPR spills, prefer the pressure
209 // with SGPR spills.
210 if (PureExcessVGPR != OtherPureExcessVGPR)
211 return SGPRDiff < 0;
212 // If both pressures have the same excess pressure before and after
213 // accounting for SGPR spills, prefer fewer SGPR spills.
214 return SGPRDiff > 0;
215 }
216 }
217
218 bool SGPRImportant = SGPROcc < VGPROcc;
219 const bool OtherSGPRImportant = OtherSGPROcc < OtherVGPROcc;
220
221 // If both pressures disagree on what is more important compare vgprs.
222 if (SGPRImportant != OtherSGPRImportant) {
223 SGPRImportant = false;
224 }
225
226 // Give third precedence to lower register tuple pressure.
227 bool SGPRFirst = SGPRImportant;
228 for (int I = 2; I > 0; --I, SGPRFirst = !SGPRFirst) {
229 if (SGPRFirst) {
230 auto SW = getSGPRTuplesWeight();
231 auto OtherSW = O.getSGPRTuplesWeight();
232 if (SW != OtherSW)
233 return SW < OtherSW;
234 } else {
235 auto VW = getVGPRTuplesWeight();
236 auto OtherVW = O.getVGPRTuplesWeight();
237 if (VW != OtherVW)
238 return VW < OtherVW;
239 }
240 }
241
242 // Give final precedence to lower general RP.
243 return SGPRImportant ? (getSGPRNum() < O.getSGPRNum()):
244 (getVGPRNum(UnifiedVGPRFile: ST.hasGFX90AInsts()) <
245 O.getVGPRNum(UnifiedVGPRFile: ST.hasGFX90AInsts()));
246}
247
248Printable llvm::print(const GCNRegPressure &RP, const GCNSubtarget *ST,
249 unsigned DynamicVGPRBlockSize) {
250 return Printable([&RP, ST, DynamicVGPRBlockSize](raw_ostream &OS) {
251 OS << "VGPRs: " << RP.getArchVGPRNum();
252 if (unsigned NumAVGPRs = RP.getAVGPRNum())
253 OS << " (AV: " << NumAVGPRs << ')';
254 OS << " AGPRs: " << RP.getAGPRNum();
255 if (ST)
256 OS << "(O"
257 << ST->getOccupancyWithNumVGPRs(VGPRs: RP.getVGPRNum(UnifiedVGPRFile: ST->hasGFX90AInsts()),
258 DynamicVGPRBlockSize)
259 << ')';
260 OS << ", SGPRs: " << RP.getSGPRNum();
261 if (ST)
262 OS << "(O" << ST->getOccupancyWithNumSGPRs(SGPRs: RP.getSGPRNum()) << ')';
263 OS << ", LVGPR WT: " << RP.getVGPRTuplesWeight()
264 << ", LSGPR WT: " << RP.getSGPRTuplesWeight();
265 if (ST)
266 OS << " -> Occ: " << RP.getOccupancy(ST: *ST, DynamicVGPRBlockSize);
267 OS << '\n';
268 });
269}
270
271static LaneBitmask getDefRegMask(const MachineOperand &MO,
272 const MachineRegisterInfo &MRI) {
273 assert(MO.isDef() && MO.isReg() && MO.getReg().isVirtual());
274
275 // We don't rely on read-undef flag because in case of tentative schedule
276 // tracking it isn't set correctly yet. This works correctly however since
277 // use mask has been tracked before using LIS.
278 return MO.getSubReg() == 0 ?
279 MRI.getMaxLaneMaskForVReg(Reg: MO.getReg()) :
280 MRI.getTargetRegisterInfo()->getSubRegIndexLaneMask(SubIdx: MO.getSubReg());
281}
282
283static void
284collectVirtualRegUses(SmallVectorImpl<VRegMaskOrUnit> &VRegMaskOrUnits,
285 const MachineInstr &MI, const LiveIntervals &LIS,
286 const MachineRegisterInfo &MRI) {
287
288 auto &TRI = *MRI.getTargetRegisterInfo();
289 for (const auto &MO : MI.operands()) {
290 if (!MO.isReg() || !MO.getReg().isVirtual())
291 continue;
292 if (!MO.isUse() || !MO.readsReg())
293 continue;
294
295 Register Reg = MO.getReg();
296 auto I = llvm::find_if(Range&: VRegMaskOrUnits, P: [Reg](const VRegMaskOrUnit &RM) {
297 return RM.VRegOrUnit.asVirtualReg() == Reg;
298 });
299
300 auto &P = I == VRegMaskOrUnits.end()
301 ? VRegMaskOrUnits.emplace_back(Args: VirtRegOrUnit(Reg),
302 Args: LaneBitmask::getNone())
303 : *I;
304
305 P.LaneMask |= MO.getSubReg() ? TRI.getSubRegIndexLaneMask(SubIdx: MO.getSubReg())
306 : MRI.getMaxLaneMaskForVReg(Reg);
307 }
308
309 SlotIndex InstrSI;
310 for (auto &P : VRegMaskOrUnits) {
311 auto &LI = LIS.getInterval(Reg: P.VRegOrUnit.asVirtualReg());
312 if (!LI.hasSubRanges())
313 continue;
314
315 // For a tentative schedule LIS isn't updated yet but livemask should
316 // remain the same on any schedule. Subreg defs can be reordered but they
317 // all must dominate uses anyway.
318 if (!InstrSI)
319 InstrSI = LIS.getInstructionIndex(Instr: MI).getBaseIndex();
320
321 P.LaneMask = getLiveLaneMask(LI, SI: InstrSI, MRI, LaneMaskFilter: P.LaneMask);
322 }
323}
324
325/// Mostly copy/paste from CodeGen/RegisterPressure.cpp
326static LaneBitmask getLanesWithProperty(
327 const LiveIntervals &LIS, const MachineRegisterInfo &MRI,
328 bool TrackLaneMasks, Register Reg, SlotIndex Pos,
329 function_ref<bool(const LiveRange &LR, SlotIndex Pos)> Property) {
330 assert(Reg.isVirtual());
331 const LiveInterval &LI = LIS.getInterval(Reg);
332 LaneBitmask Result;
333 if (TrackLaneMasks && LI.hasSubRanges()) {
334 for (const LiveInterval::SubRange &SR : LI.subranges()) {
335 if (Property(SR, Pos))
336 Result |= SR.LaneMask;
337 }
338 } else if (Property(LI, Pos)) {
339 Result =
340 TrackLaneMasks ? MRI.getMaxLaneMaskForVReg(Reg) : LaneBitmask::getAll();
341 }
342
343 return Result;
344}
345
346/// Mostly copy/paste from CodeGen/RegisterPressure.cpp
347/// Helper to find a vreg use between two indices {PriorUseIdx, NextUseIdx}.
348/// The query starts with a lane bitmask which gets lanes/bits removed for every
349/// use we find.
350static LaneBitmask findUseBetween(unsigned Reg, LaneBitmask LastUseMask,
351 SlotIndex PriorUseIdx, SlotIndex NextUseIdx,
352 const MachineRegisterInfo &MRI,
353 const SIRegisterInfo *TRI,
354 const LiveIntervals *LIS,
355 bool Upward = false) {
356 for (const MachineOperand &MO : MRI.use_nodbg_operands(Reg)) {
357 if (MO.isUndef())
358 continue;
359 const MachineInstr *MI = MO.getParent();
360 SlotIndex InstSlot = LIS->getInstructionIndex(Instr: *MI).getRegSlot();
361 bool InRange = Upward ? (InstSlot > PriorUseIdx && InstSlot <= NextUseIdx)
362 : (InstSlot >= PriorUseIdx && InstSlot < NextUseIdx);
363 if (!InRange)
364 continue;
365
366 unsigned SubRegIdx = MO.getSubReg();
367 LaneBitmask UseMask = TRI->getSubRegIndexLaneMask(SubIdx: SubRegIdx);
368 LastUseMask &= ~UseMask;
369 if (LastUseMask.none())
370 return LaneBitmask::getNone();
371 }
372 return LastUseMask;
373}
374
375////////////////////////////////////////////////////////////////////////////////
376// GCNRPTarget
377
378GCNRPTarget::GCNRPTarget(const MachineFunction &MF, const GCNRegPressure &RP)
379 : GCNRPTarget(RP, MF) {
380 const Function &F = MF.getFunction();
381 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
382 setTarget(NumSGPRs: ST.getMaxNumSGPRs(F), NumVGPRs: ST.getMaxNumVGPRs(F));
383}
384
385GCNRPTarget::GCNRPTarget(unsigned NumSGPRs, unsigned NumVGPRs,
386 const MachineFunction &MF, const GCNRegPressure &RP)
387 : GCNRPTarget(RP, MF) {
388 setTarget(NumSGPRs, NumVGPRs);
389}
390
391GCNRPTarget::GCNRPTarget(unsigned Occupancy, const MachineFunction &MF,
392 const GCNRegPressure &RP)
393 : GCNRPTarget(RP, MF) {
394 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
395 unsigned DynamicVGPRBlockSize =
396 MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize();
397 setTarget(NumSGPRs: ST.getMaxNumSGPRs(WavesPerEU: Occupancy, /*Addressable=*/false),
398 NumVGPRs: ST.getMaxNumVGPRs(WavesPerEU: Occupancy, DynamicVGPRBlockSize));
399}
400
401void GCNRPTarget::setTarget(unsigned NumSGPRs, unsigned NumVGPRs) {
402 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
403 MaxSGPRs = std::min(a: ST.getAddressableNumSGPRs(), b: NumSGPRs);
404 MaxVGPRs = std::min(a: ST.getAddressableNumArchVGPRs(), b: NumVGPRs);
405 if (UnifiedRF) {
406 unsigned DynamicVGPRBlockSize =
407 MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize();
408 MaxUnifiedVGPRs =
409 std::min(a: ST.getAddressableNumVGPRs(DynamicVGPRBlockSize), b: NumVGPRs);
410 } else {
411 MaxUnifiedVGPRs = 0;
412 }
413}
414
415bool GCNRPTarget::isSaveBeneficial(Register Reg) const {
416 const MachineRegisterInfo &MRI = MF.getRegInfo();
417 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
418 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
419 const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo *>(TRI);
420
421 RegExcess Excess(MF, RP, *this);
422
423 if (SRI->isSGPRClass(RC))
424 return Excess.SGPR;
425
426 if (SRI->isAGPRClass(RC))
427 return (UnifiedRF && Excess.VGPR) || Excess.AGPR;
428
429 return (UnifiedRF && Excess.VGPR) || Excess.ArchVGPR;
430}
431
432bool GCNRPTarget::isSaveBeneficial(const GCNRegPressure &SaveRP) const {
433 RegExcess Excess(MF, RP, *this);
434 if (SaveRP.getSGPRNum() != 0 && Excess.SGPR != 0)
435 return true;
436 if (SaveRP.getArchVGPRNum() != 0 && Excess.ArchVGPR != 0)
437 return true;
438 if (SaveRP.getAGPRNum() != 0 && Excess.AGPR != 0)
439 return true;
440 if (UnifiedRF && Excess.VGPR != 0)
441 return SaveRP.getArchVGPRNum() != 0 || SaveRP.getAGPRNum() != 0;
442 return false;
443}
444
445unsigned GCNRPTarget::getNumRegsBenefit(const GCNRegPressure &SaveRP) const {
446 RegExcess Excess(MF, RP, *this);
447 const unsigned NumVGPRAboveAddrLimit =
448 std::min(a: Excess.ArchVGPR, b: SaveRP.getArchVGPRNum()) +
449 std::min(a: Excess.AGPR, b: SaveRP.getAGPRNum());
450 unsigned NumRegsSaved =
451 std::min(a: Excess.SGPR, b: SaveRP.getSGPRNum()) + NumVGPRAboveAddrLimit;
452
453 if (UnifiedRF && Excess.VGPR) {
454 // We have already accounted for excess pressure above addressive limits for
455 // the individual VGPR classes. However for targets with unified RFs there
456 // is also a unified VGPR pressure (ArchVGPR + AGPR combination) limit to
457 // honor that may be more restrictive that the per-VGPR-class limits. We
458 // must also be careful not to double-count VGPR saves that may contribute
459 // to lowering pressure both above the addressable limit in their respective
460 // class as well as in the unified VGPR limit.
461 const unsigned VGPRSave = SaveRP.getArchVGPRNum() + SaveRP.getAGPRNum();
462 if (NumVGPRAboveAddrLimit < VGPRSave)
463 NumRegsSaved += std::min(a: Excess.VGPR, b: VGPRSave - NumVGPRAboveAddrLimit);
464 }
465
466 return NumRegsSaved;
467}
468
469bool GCNRPTarget::satisfied(const GCNRegPressure &TestRP) const {
470 if (TestRP.getSGPRNum() > MaxSGPRs || TestRP.getVGPRNum(UnifiedVGPRFile: false) > MaxVGPRs)
471 return false;
472 if (UnifiedRF && TestRP.getVGPRNum(UnifiedVGPRFile: true) > MaxUnifiedVGPRs)
473 return false;
474 return true;
475}
476
477bool GCNRPTarget::hasVectorRegisterExcess() const {
478 RegExcess Excess(MF, RP, *this);
479 return Excess.hasVectorRegisterExcess();
480}
481
482///////////////////////////////////////////////////////////////////////////////
483// GCNRPTracker
484
485LaneBitmask llvm::getLiveLaneMask(unsigned Reg, SlotIndex SI,
486 const LiveIntervals &LIS,
487 const MachineRegisterInfo &MRI,
488 LaneBitmask LaneMaskFilter) {
489 return getLiveLaneMask(LI: LIS.getInterval(Reg), SI, MRI, LaneMaskFilter);
490}
491
492LaneBitmask llvm::getLiveLaneMask(const LiveInterval &LI, SlotIndex SI,
493 const MachineRegisterInfo &MRI,
494 LaneBitmask LaneMaskFilter) {
495 LaneBitmask LiveMask;
496 if (LI.hasSubRanges()) {
497 for (const auto &S : LI.subranges())
498 if ((S.LaneMask & LaneMaskFilter).any() && S.liveAt(index: SI)) {
499 LiveMask |= S.LaneMask;
500 assert(LiveMask == (LiveMask & MRI.getMaxLaneMaskForVReg(LI.reg())));
501 }
502 } else if (LI.liveAt(index: SI)) {
503 LiveMask = MRI.getMaxLaneMaskForVReg(Reg: LI.reg());
504 }
505 LiveMask &= LaneMaskFilter;
506 return LiveMask;
507}
508
509GCNRPTracker::LiveRegSet llvm::getLiveRegs(SlotIndex SI,
510 const LiveIntervals &LIS,
511 const MachineRegisterInfo &MRI,
512 GCNRegPressure::RegKind RegKind) {
513 GCNRPTracker::LiveRegSet LiveRegs;
514 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
515 auto Reg = Register::index2VirtReg(Index: I);
516 if (RegKind != GCNRegPressure::TOTAL_KINDS &&
517 GCNRegPressure::getRegKind(Reg, MRI) != RegKind)
518 continue;
519 if (!LIS.hasInterval(Reg))
520 continue;
521 auto LiveMask = getLiveLaneMask(Reg, SI, LIS, MRI);
522 if (LiveMask.any())
523 LiveRegs[Reg] = LiveMask;
524 }
525 return LiveRegs;
526}
527
528void GCNRPTracker::reset(const MachineInstr &MI, bool After) {
529 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
530 if (!MI.isDebugInstr()) {
531 SlotIndex SI = LIS.getInstructionIndex(Instr: MI);
532 if (After)
533 SI = SI.getDeadSlot();
534 reset(MRI, SI);
535 return;
536 }
537
538 // Look for the first valid index after the provided debug MI.
539 MachineBasicBlock::const_iterator It = MI.getIterator(),
540 MBBEnd = MI.getParent()->end();
541 MachineBasicBlock::const_iterator NonDbgMI =
542 skipDebugInstructionsForward(It, End: MBBEnd);
543 if (NonDbgMI == MBBEnd) {
544 // There are no non-debug instruction between MI and the end of the
545 // block, so we reset the tracker at the end of the block.
546 reset(MBB: *MI.getParent(), /*End=*/true);
547 return;
548 }
549 // MI is a debug instruction so register pressure before or after it is
550 // identical. Since we moved forward to finding a non-debug instruction
551 // in the block, we reset the tracker before that instruction i.e., at its
552 // base index.
553 reset(MRI, SI: LIS.getInstructionIndex(Instr: *NonDbgMI));
554}
555
556void GCNRPTracker::reset(const MachineBasicBlock &MBB, bool End) {
557 SlotIndex SI = End ? LIS.getSlotIndexes()->getMBBLastIdx(MBB: &MBB)
558 : LIS.getMBBStartIdx(mbb: &MBB);
559 reset(MRI: MBB.getParent()->getRegInfo(), SI);
560}
561
562void GCNRPTracker::reset(const MachineRegisterInfo &MRI, SlotIndex SI) {
563 this->MRI = &MRI;
564 LastTrackedMI = nullptr;
565 LiveRegs = llvm::getLiveRegs(SI, LIS, MRI);
566 MaxPressure = CurPressure = getRegPressure(MRI, LiveRegs);
567}
568
569void GCNRPTracker::reset(const MachineRegisterInfo &MRI,
570 const LiveRegSet &LiveRegs) {
571 this->MRI = &MRI;
572 LastTrackedMI = nullptr;
573 if (&this->LiveRegs != &LiveRegs)
574 this->LiveRegs = LiveRegs;
575 MaxPressure = CurPressure = getRegPressure(MRI, LiveRegs);
576}
577
578/// Mostly copy/paste from CodeGen/RegisterPressure.cpp
579LaneBitmask GCNRPTracker::getLastUsedLanes(Register Reg, SlotIndex Pos) const {
580 return getLanesWithProperty(
581 LIS, MRI: *MRI, TrackLaneMasks: true, Reg, Pos: Pos.getBaseIndex(),
582 Property: [](const LiveRange &LR, SlotIndex Pos) {
583 const LiveRange::Segment *S = LR.getSegmentContaining(Idx: Pos);
584 return S != nullptr && S->end == Pos.getRegSlot();
585 });
586}
587
588////////////////////////////////////////////////////////////////////////////////
589// GCNUpwardRPTracker
590
591void GCNUpwardRPTracker::recede(const MachineInstr &MI) {
592 assert(MRI && "call reset first");
593
594 LastTrackedMI = &MI;
595
596 if (MI.isDebugInstr())
597 return;
598
599 // Kill all defs.
600 GCNRegPressure DefPressure, ECDefPressure;
601 bool HasECDefs = false;
602 for (const MachineOperand &MO : MI.all_defs()) {
603 if (!MO.getReg().isVirtual())
604 continue;
605
606 Register Reg = MO.getReg();
607 LaneBitmask DefMask = getDefRegMask(MO, MRI: *MRI);
608
609 // Treat a def as fully live at the moment of definition: keep a record.
610 if (MO.isEarlyClobber()) {
611 ECDefPressure.inc(Reg, PrevMask: LaneBitmask::getNone(), NewMask: DefMask, MRI: *MRI);
612 HasECDefs = true;
613 } else
614 DefPressure.inc(Reg, PrevMask: LaneBitmask::getNone(), NewMask: DefMask, MRI: *MRI);
615
616 auto I = LiveRegs.find(Val: Reg);
617 if (I == LiveRegs.end())
618 continue;
619
620 LaneBitmask &LiveMask = I->second;
621 LaneBitmask PrevMask = LiveMask;
622 LiveMask &= ~DefMask;
623 CurPressure.inc(Reg, PrevMask, NewMask: LiveMask, MRI: *MRI);
624 if (LiveMask.none())
625 LiveRegs.erase(I);
626 }
627
628 // Update MaxPressure with defs pressure.
629 DefPressure += CurPressure;
630 if (HasECDefs)
631 DefPressure += ECDefPressure;
632 updateMaxPressure(RP: DefPressure);
633
634 // Make uses alive.
635 SmallVector<VRegMaskOrUnit, 8> RegUses;
636 collectVirtualRegUses(VRegMaskOrUnits&: RegUses, MI, LIS, MRI: *MRI);
637 for (const VRegMaskOrUnit &U : RegUses) {
638 LaneBitmask &LiveMask = LiveRegs[U.VRegOrUnit.asVirtualReg()];
639 LaneBitmask PrevMask = LiveMask;
640 LiveMask |= U.LaneMask;
641 CurPressure.inc(Reg: U.VRegOrUnit.asVirtualReg(), PrevMask, NewMask: LiveMask, MRI: *MRI);
642 }
643
644 // Update MaxPressure with uses plus early-clobber defs pressure.
645 updateMaxPressure(RP: HasECDefs ? CurPressure + ECDefPressure : CurPressure);
646
647 assert(CurPressure == getRegPressure(*MRI, LiveRegs));
648}
649
650////////////////////////////////////////////////////////////////////////////////
651// GCNDownwardRPTracker
652
653bool GCNDownwardRPTracker::reset(const MachineInstr &MI,
654 MachineBasicBlock::const_iterator End,
655 const LiveRegSet *LiveRegsCopy) {
656 MBBEnd = MI.getParent()->end();
657 assert((End == MBBEnd || End->getParent()->end() == MBBEnd) &&
658 "end unrelated to MI block");
659 NextMI = &MI;
660 NextMI = skipDebugInstructionsForward(It: NextMI, End);
661
662 // Do not use the MI to compute live registers when a set is provided.
663 // Otherwise the first non-debug instruction after the provided one (or the
664 // end of the block, if no such instruction exists) serves as the basis to
665 // compute a live register set.
666 if (LiveRegsCopy)
667 GCNRPTracker::reset(MRI: MI.getMF()->getRegInfo(), LiveRegs: *LiveRegsCopy);
668 else if (NextMI != MBBEnd)
669 GCNRPTracker::reset(MI: *NextMI, /*After=*/false);
670 else
671 GCNRPTracker::reset(MBB: *MI.getParent(), /*End=*/true);
672 return NextMI != End;
673}
674
675void GCNDownwardRPTrackerBase::retireVirtReg(Register Reg, SlotIndex SI) {
676 const LiveInterval &LI = LIS.getInterval(Reg);
677 if (LI.hasSubRanges()) {
678 auto It = LiveRegs.end();
679 for (const auto &S : LI.subranges()) {
680 if (!S.liveAt(index: SI)) {
681 if (It == LiveRegs.end()) {
682 It = LiveRegs.find(Val: Reg);
683 if (It == LiveRegs.end())
684 llvm_unreachable("register isn't live");
685 }
686 auto PrevMask = It->second;
687 It->second &= ~S.LaneMask;
688 CurPressure.inc(Reg, PrevMask, NewMask: It->second, MRI: *MRI);
689 }
690 }
691 if (It != LiveRegs.end() && It->second.none())
692 LiveRegs.erase(I: It);
693 } else if (!LI.liveAt(index: SI)) {
694 auto It = LiveRegs.find(Val: Reg);
695 if (It == LiveRegs.end())
696 llvm_unreachable("register isn't live");
697 CurPressure.inc(Reg, PrevMask: It->second, NewMask: LaneBitmask::getNone(), MRI: *MRI);
698 LiveRegs.erase(I: It);
699 }
700}
701
702void GCNDownwardRPTrackerBase::retireDeadUses(const MachineInstr &MI) {
703 assert(MRI && "call reset first");
704
705 if (MI.isPHI())
706 return;
707
708 // Mirror ECDefPressure handling from GCNUpwardRPTracker::recede().
709 GCNRegPressure ECDefPressure;
710 for (const MachineOperand &MO : MI.all_defs()) {
711 Register Reg = MO.getReg();
712 if (Reg.isVirtual() && MO.isEarlyClobber())
713 ECDefPressure.inc(Reg, PrevMask: LaneBitmask::getNone(), NewMask: getDefRegMask(MO, MRI: *MRI),
714 MRI: *MRI);
715 }
716
717 updateMaxPressure(RP: CurPressure + ECDefPressure);
718
719 SlotIndex SI = LIS.getInstructionIndex(Instr: MI).getDeadSlot();
720 assert(SI.isValid());
721 SmallSet<Register, 8> SeenRegs;
722 for (const MachineOperand &MO : MI.all_uses()) {
723 Register Reg = MO.getReg();
724 if (MO.readsReg() && Reg.isVirtual() && SeenRegs.insert(V: Reg).second)
725 retireVirtReg(Reg, SI);
726 }
727}
728
729void GCNDownwardRPTrackerBase::retireDeadDefs(const MachineInstr &MI,
730 SlotIndex SI) {
731 assert(MRI && "call reset first");
732
733 SmallSet<Register, 8> SeenRegs;
734 for (const MachineOperand &MO : MI.all_defs()) {
735 Register Reg = MO.getReg();
736 if (Reg.isVirtual() && SeenRegs.insert(V: Reg).second)
737 retireVirtReg(Reg, SI);
738 }
739
740 updateMaxPressure(RP: CurPressure);
741}
742
743void GCNDownwardRPTrackerBase::addDefs(const MachineInstr &MI) {
744 assert(MRI && "call reset first");
745
746 // Add new registers or mask bits.
747 for (const MachineOperand &MO : MI.all_defs()) {
748 Register Reg = MO.getReg();
749 if (!Reg.isVirtual())
750 continue;
751 auto &LiveMask = LiveRegs[Reg];
752 auto PrevMask = LiveMask;
753 LiveMask |= getDefRegMask(MO, MRI: *MRI);
754 CurPressure.inc(Reg, PrevMask, NewMask: LiveMask, MRI: *MRI);
755 }
756 updateMaxPressure(RP: CurPressure);
757}
758
759bool GCNDownwardRPTracker::advanceBeforeNext() {
760 if (!LastTrackedMI.isValid())
761 return NextMI == MBBEnd;
762 assert(NextMI == MBBEnd || !NextMI->isDebugInstr());
763
764 SlotIndex SI = NextMI == MBBEnd
765 ? LIS.getInstructionIndex(Instr: *LastTrackedMI).getDeadSlot()
766 : LIS.getInstructionIndex(Instr: *NextMI).getBaseIndex();
767 assert(SI.isValid());
768 GCNDownwardRPTrackerBase::retireDeadDefs(MI: *LastTrackedMI, SI);
769 LastTrackedMI = nullptr;
770 return NextMI == MBBEnd;
771}
772
773void GCNDownwardRPTracker::advanceToNext() {
774 LastTrackedMI = NextMI++;
775 NextMI = skipDebugInstructionsForward(It: NextMI, End: MBBEnd);
776 GCNDownwardRPTrackerBase::retireDeadUses(MI: *LastTrackedMI);
777 GCNDownwardRPTrackerBase::addDefs(MI: *LastTrackedMI);
778}
779
780bool GCNDownwardRPTracker::advance() {
781 if (NextMI == MBBEnd)
782 return false;
783
784 advanceBeforeNext();
785 advanceToNext();
786 return true;
787}
788
789bool GCNDownwardRPTracker::advance(MachineBasicBlock::const_iterator End) {
790 bool AnyAdvance = false;
791 while (NextMI != End && advance())
792 AnyAdvance = true;
793 return AnyAdvance;
794}
795
796bool GCNDownwardRPTracker::advance(MachineBasicBlock::const_iterator Begin,
797 MachineBasicBlock::const_iterator End,
798 const LiveRegSet *LiveRegsCopy) {
799 if (!reset(MI: *Begin, End, LiveRegsCopy))
800 return false;
801 return GCNDownwardRPTracker::advance(End);
802}
803
804void GCNManualDownwardRPTracker::advance(const MachineInstr &MI) {
805 GCNDownwardRPTrackerBase::retireDeadUses(MI);
806 GCNDownwardRPTrackerBase::addDefs(MI);
807 GCNDownwardRPTrackerBase::retireDeadDefs(
808 MI, SI: LIS.getInstructionIndex(Instr: MI).getDeadSlot());
809
810 // Speculative queries (bumpDownwardPressure) must know the last scheduled
811 // instruction to correctly estimate pressure change.
812 LastTrackedMI = &MI;
813}
814
815Printable llvm::reportMismatch(const GCNRPTracker::LiveRegSet &LISLR,
816 const GCNRPTracker::LiveRegSet &TrackedLR,
817 const TargetRegisterInfo *TRI, StringRef Pfx) {
818 return Printable([&LISLR, &TrackedLR, TRI, Pfx](raw_ostream &OS) {
819 for (auto const &P : TrackedLR) {
820 auto I = LISLR.find(Val: P.first);
821 if (I == LISLR.end()) {
822 OS << Pfx << printReg(Reg: P.first, TRI) << ":L" << PrintLaneMask(LaneMask: P.second)
823 << " isn't found in LIS reported set\n";
824 } else if (I->second != P.second) {
825 OS << Pfx << printReg(Reg: P.first, TRI)
826 << " masks doesn't match: LIS reported " << PrintLaneMask(LaneMask: I->second)
827 << ", tracked " << PrintLaneMask(LaneMask: P.second) << '\n';
828 }
829 }
830 for (auto const &P : LISLR) {
831 auto I = TrackedLR.find(Val: P.first);
832 if (I == TrackedLR.end()) {
833 OS << Pfx << printReg(Reg: P.first, TRI) << ":L" << PrintLaneMask(LaneMask: P.second)
834 << " isn't found in tracked set\n";
835 }
836 }
837 });
838}
839
840GCNRegPressure GCNDownwardRPTrackerBase::bumpDownwardPressure(
841 const MachineInstr *MI, const SIRegisterInfo *TRI) const {
842 assert(!MI->isDebugOrPseudoInstr() && "Expect a nondebug instruction.");
843
844 SlotIndex SlotIdx;
845 SlotIdx = LIS.getInstructionIndex(Instr: *MI).getRegSlot();
846
847 SlotIndex CurrIdx;
848 const MachineBasicBlock *MBB = MI->getParent();
849 MachineBasicBlock::const_iterator StartPos =
850 LastTrackedMI.isValid() ? std::next(x: LastTrackedMI) : MBB->begin();
851 MachineBasicBlock::const_iterator IdxPos =
852 skipDebugInstructionsForward(It: StartPos, End: MBB->end());
853 if (IdxPos == MBB->end()) {
854 CurrIdx = LIS.getMBBEndIdx(mbb: MBB);
855 } else {
856 CurrIdx = LIS.getInstructionIndex(Instr: *IdxPos).getRegSlot();
857 }
858
859 // Account for register pressure similar to RegPressureTracker::recede().
860 RegisterOperands RegOpers;
861 RegOpers.collect(MI: *MI, TRI: *TRI, MRI: *MRI, TrackLaneMasks: true, /*IgnoreDead=*/false);
862 RegOpers.adjustLaneLiveness(LIS, MRI: *MRI, Pos: SlotIdx);
863 GCNRegPressure TempPressure = CurPressure;
864 // Tracks the live mask reported by the use loop for redefined registers.
865 SmallDenseMap<Register, LaneBitmask, 8> PostUseMask;
866
867 for (const VRegMaskOrUnit &Use : RegOpers.Uses) {
868 if (!Use.VRegOrUnit.isVirtualReg())
869 continue;
870 Register Reg = Use.VRegOrUnit.asVirtualReg();
871 LaneBitmask LastUseMask = getLastUsedLanes(Reg, Pos: SlotIdx);
872 if (LastUseMask.none())
873 continue;
874 // The LastUseMask is queried from the liveness information of instruction
875 // which may be further down the schedule. Some lanes may actually not be
876 // last uses for the current position.
877 // FIXME: allow the caller to pass in the list of vreg uses that remain
878 // to be bottom-scheduled to avoid searching uses at each query.
879 LastUseMask =
880 findUseBetween(Reg, LastUseMask, PriorUseIdx: CurrIdx, NextUseIdx: SlotIdx, MRI: *MRI, TRI, LIS: &LIS);
881 if (LastUseMask.none())
882 continue;
883
884 auto It = LiveRegs.find(Val: Reg);
885 LaneBitmask LiveMask = It != LiveRegs.end() ? It->second : LaneBitmask(0);
886 LaneBitmask NewMask = LiveMask & ~LastUseMask;
887 PostUseMask[Reg] = NewMask;
888 TempPressure.inc(Reg, PrevMask: LiveMask, NewMask, MRI: *MRI);
889 }
890
891 // Generate liveness for defs.
892 for (const VRegMaskOrUnit &Def : RegOpers.Defs) {
893 if (!Def.VRegOrUnit.isVirtualReg())
894 continue;
895 Register Reg = Def.VRegOrUnit.asVirtualReg();
896 auto PostIt = PostUseMask.find(Val: Reg);
897 LaneBitmask LiveMask;
898 if (PostIt != PostUseMask.end()) {
899 LiveMask = PostIt->second;
900 } else {
901 auto It = LiveRegs.find(Val: Reg);
902 LiveMask = It != LiveRegs.end() ? It->second : LaneBitmask(0);
903 }
904
905 LaneBitmask NewMask = LiveMask | Def.LaneMask;
906 TempPressure.inc(Reg, PrevMask: LiveMask, NewMask, MRI: *MRI);
907 }
908
909 return TempPressure;
910}
911
912bool GCNUpwardRPTracker::isValid() const {
913 const auto &SI = LIS.getInstructionIndex(Instr: *LastTrackedMI).getBaseIndex();
914 const auto LISLR = llvm::getLiveRegs(SI, LIS, MRI: *MRI);
915 const auto &TrackedLR = LiveRegs;
916
917 if (!isEqual(S1: LISLR, S2: TrackedLR)) {
918 dbgs() << "\nGCNUpwardRPTracker error: Tracked and"
919 " LIS reported livesets mismatch:\n"
920 << print(LiveRegs: LISLR, MRI: *MRI);
921 reportMismatch(LISLR, TrackedLR, TRI: MRI->getTargetRegisterInfo());
922 return false;
923 }
924
925 auto LISPressure = getRegPressure(MRI: *MRI, LiveRegs: LISLR);
926 if (LISPressure != CurPressure) {
927 dbgs() << "GCNUpwardRPTracker error: Pressure sets different\nTracked: "
928 << print(RP: CurPressure) << "LIS rpt: " << print(RP: LISPressure);
929 return false;
930 }
931 return true;
932}
933
934Printable llvm::print(const GCNRPTracker::LiveRegSet &LiveRegs,
935 const MachineRegisterInfo &MRI) {
936 return Printable([&LiveRegs, &MRI](raw_ostream &OS) {
937 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
938 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
939 Register Reg = Register::index2VirtReg(Index: I);
940 auto It = LiveRegs.find(Val: Reg);
941 if (It != LiveRegs.end() && It->second.any())
942 OS << ' ' << printReg(Reg, TRI) << ':' << PrintLaneMask(LaneMask: It->second);
943 }
944 OS << '\n';
945 });
946}
947
948void GCNRegPressure::dump() const { dbgs() << print(RP: *this); }
949
950char llvm::GCNRegPressurePrinter::ID = 0;
951char &llvm::GCNRegPressurePrinterID = GCNRegPressurePrinter::ID;
952
953INITIALIZE_PASS(GCNRegPressurePrinter, "amdgpu-print-rp", "", true, true)
954
955// Return lanemask of Reg's subregs that are live-through at [Begin, End] and
956// are fully covered by Mask.
957static LaneBitmask
958getRegLiveThroughMask(const MachineRegisterInfo &MRI, const LiveIntervals &LIS,
959 Register Reg, SlotIndex Begin, SlotIndex End,
960 LaneBitmask Mask = LaneBitmask::getAll()) {
961
962 auto IsInOneSegment = [Begin, End](const LiveRange &LR) -> bool {
963 auto *Segment = LR.getSegmentContaining(Idx: Begin);
964 return Segment && Segment->contains(I: End);
965 };
966
967 LaneBitmask LiveThroughMask;
968 const LiveInterval &LI = LIS.getInterval(Reg);
969 if (LI.hasSubRanges()) {
970 for (auto &SR : LI.subranges()) {
971 if ((SR.LaneMask & Mask) == SR.LaneMask && IsInOneSegment(SR))
972 LiveThroughMask |= SR.LaneMask;
973 }
974 } else {
975 LaneBitmask RegMask = MRI.getMaxLaneMaskForVReg(Reg);
976 if ((RegMask & Mask) == RegMask && IsInOneSegment(LI))
977 LiveThroughMask = RegMask;
978 }
979
980 return LiveThroughMask;
981}
982
983bool GCNRegPressurePrinter::runOnMachineFunction(MachineFunction &MF) {
984 const MachineRegisterInfo &MRI = MF.getRegInfo();
985 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
986 LiveIntervals &LIS = getAnalysis<LiveIntervalsWrapperPass>().getLIS();
987 const bool UseDownwardTracker =
988 MF.getSubtarget<GCNSubtarget>().getCLOpts().print_rp_downward;
989
990 auto &OS = dbgs();
991
992// Leading spaces are important for YAML syntax.
993#define PFX " "
994
995 OS << "---\nname: " << MF.getName() << "\nbody: |\n";
996
997 auto printRP = [](const GCNRegPressure &RP) {
998 return Printable([&RP](raw_ostream &OS) {
999 OS << format(PFX " %-5d", Vals: RP.getSGPRNum())
1000 << format(Fmt: " %-5d", Vals: RP.getVGPRNum(UnifiedVGPRFile: false));
1001 });
1002 };
1003
1004 auto ReportLISMismatchIfAny = [&](const GCNRPTracker::LiveRegSet &TrackedLR,
1005 const GCNRPTracker::LiveRegSet &LISLR) {
1006 if (LISLR != TrackedLR) {
1007 OS << PFX " mis LIS: " << llvm::print(LiveRegs: LISLR, MRI)
1008 << reportMismatch(LISLR, TrackedLR, TRI, PFX " ");
1009 }
1010 };
1011
1012 // Register pressure before and at an instruction (in program order).
1013 SmallVector<std::pair<GCNRegPressure, GCNRegPressure>, 16> RP;
1014
1015 for (auto &MBB : MF) {
1016 RP.clear();
1017 RP.reserve(N: MBB.size());
1018
1019 OS << PFX;
1020 MBB.printName(os&: OS);
1021 OS << ":\n";
1022
1023 SlotIndex MBBStartSlot = LIS.getSlotIndexes()->getMBBStartIdx(mbb: &MBB);
1024 SlotIndex MBBLastSlot = LIS.getSlotIndexes()->getMBBLastIdx(MBB: &MBB);
1025
1026 GCNRPTracker::LiveRegSet LiveIn, LiveOut;
1027 GCNRegPressure RPAtMBBEnd;
1028
1029 if (UseDownwardTracker) {
1030 if (MBB.empty()) {
1031 LiveIn = LiveOut = getLiveRegs(SI: MBBStartSlot, LIS, MRI);
1032 RPAtMBBEnd = getRegPressure(MRI, LiveRegs&: LiveIn);
1033 } else {
1034 GCNDownwardRPTracker RPT(LIS);
1035 RPT.reset(MI: MBB.front(), End: MBB.end());
1036
1037 LiveIn = RPT.getLiveRegs();
1038
1039 while (!RPT.advanceBeforeNext()) {
1040 GCNRegPressure RPBeforeMI = RPT.getPressure();
1041 RPT.resetMaxPressure();
1042 RPT.advanceToNext();
1043 RP.emplace_back(Args&: RPBeforeMI, Args: RPT.getMaxPressure());
1044 }
1045
1046 LiveOut = RPT.getLiveRegs();
1047 RPAtMBBEnd = RPT.getPressure();
1048 }
1049 } else {
1050 GCNUpwardRPTracker RPT(LIS);
1051 RPT.reset(MRI, SI: MBBLastSlot);
1052
1053 LiveOut = RPT.getLiveRegs();
1054 RPAtMBBEnd = RPT.getPressure();
1055
1056 for (auto &MI : reverse(C&: MBB)) {
1057 RPT.resetMaxPressure();
1058 RPT.recede(MI);
1059 if (!MI.isDebugInstr())
1060 RP.emplace_back(Args: RPT.getPressure(), Args: RPT.getMaxPressure());
1061 }
1062
1063 LiveIn = RPT.getLiveRegs();
1064 }
1065
1066 OS << PFX " Live-in: " << llvm::print(LiveRegs: LiveIn, MRI);
1067 if (!UseDownwardTracker)
1068 ReportLISMismatchIfAny(LiveIn, getLiveRegs(SI: MBBStartSlot, LIS, MRI));
1069
1070 OS << PFX " SGPR VGPR\n";
1071 int I = 0;
1072 for (auto &MI : MBB) {
1073 if (!MI.isDebugInstr()) {
1074 auto &[RPBeforeInstr, RPAtInstr] =
1075 RP[UseDownwardTracker ? I : (RP.size() - 1 - I)];
1076 ++I;
1077 OS << printRP(RPBeforeInstr) << '\n' << printRP(RPAtInstr) << " ";
1078 } else
1079 OS << PFX " ";
1080 MI.print(OS);
1081 }
1082 OS << printRP(RPAtMBBEnd) << '\n';
1083
1084 OS << PFX " Live-out:" << llvm::print(LiveRegs: LiveOut, MRI);
1085 if (UseDownwardTracker)
1086 ReportLISMismatchIfAny(LiveOut, getLiveRegs(SI: MBBLastSlot, LIS, MRI));
1087
1088 GCNRPTracker::LiveRegSet LiveThrough;
1089 for (auto [Reg, Mask] : LiveIn) {
1090 LaneBitmask MaskIntersection = Mask & LiveOut.lookup(Val: Reg);
1091 if (MaskIntersection.any()) {
1092 LaneBitmask LTMask = getRegLiveThroughMask(
1093 MRI, LIS, Reg, Begin: MBBStartSlot, End: MBBLastSlot, Mask: MaskIntersection);
1094 if (LTMask.any())
1095 LiveThrough[Reg] = LTMask;
1096 }
1097 }
1098 OS << PFX " Live-thr:" << llvm::print(LiveRegs: LiveThrough, MRI);
1099 OS << printRP(getRegPressure(MRI, LiveRegs&: LiveThrough)) << '\n';
1100 }
1101 OS << "...\n";
1102 return false;
1103
1104#undef PFX
1105}
1106
1107#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1108LLVM_DUMP_METHOD void llvm::dumpMaxRegPressure(MachineFunction &MF,
1109 GCNRegPressure::RegKind Kind,
1110 LiveIntervals &LIS,
1111 const MachineLoopInfo *MLI) {
1112
1113 const MachineRegisterInfo &MRI = MF.getRegInfo();
1114 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
1115 auto &OS = dbgs();
1116 const char *RegName = GCNRegPressure::getName(Kind);
1117
1118 unsigned MaxNumRegs = 0;
1119 const MachineInstr *MaxPressureMI = nullptr;
1120 GCNUpwardRPTracker RPT(LIS);
1121 // The maximum pressure of the tracker pairs the VGPR and AVGPR components,
1122 // so take the maximum of Kind alone from each point.
1123 unsigned NumRegs = 0;
1124 RPT.setPressureCallback([&](const GCNRegPressure &RP) {
1125 NumRegs = std::max(NumRegs, RP.getNumRegs(Kind));
1126 });
1127 for (const MachineBasicBlock &MBB : MF) {
1128 RPT.reset(MRI, LIS.getSlotIndexes()->getMBBEndIdx(&MBB).getPrevSlot());
1129 NumRegs = RPT.getPressure().getNumRegs(Kind);
1130 for (const MachineInstr &MI : reverse(MBB)) {
1131 RPT.recede(MI);
1132 if (NumRegs > MaxNumRegs) {
1133 MaxNumRegs = NumRegs;
1134 MaxPressureMI = &MI;
1135 }
1136 }
1137 }
1138
1139 SlotIndex MISlot = LIS.getInstructionIndex(*MaxPressureMI);
1140
1141 // Max pressure can occur at either the early-clobber or register slot.
1142 // Choose the maximum liveset between both slots. This is ugly but this is
1143 // diagnostic code.
1144 SlotIndex ECSlot = MISlot.getRegSlot(true);
1145 SlotIndex RSlot = MISlot.getRegSlot(false);
1146 GCNRPTracker::LiveRegSet ECLiveSet = getLiveRegs(ECSlot, LIS, MRI, Kind);
1147 GCNRPTracker::LiveRegSet RLiveSet = getLiveRegs(RSlot, LIS, MRI, Kind);
1148 unsigned ECNumRegs = getRegPressure(MRI, ECLiveSet).getNumRegs(Kind);
1149 unsigned RNumRegs = getRegPressure(MRI, RLiveSet).getNumRegs(Kind);
1150 GCNRPTracker::LiveRegSet *LiveSet =
1151 ECNumRegs > RNumRegs ? &ECLiveSet : &RLiveSet;
1152 SlotIndex MaxPressureSlot = ECNumRegs > RNumRegs ? ECSlot : RSlot;
1153 assert(getRegPressure(MRI, *LiveSet).getNumRegs(Kind) == MaxNumRegs);
1154
1155 // Split live registers into single-def and multi-def sets.
1156 GCNRegPressure SDefPressure, MDefPressure;
1157 SmallVector<Register, 16> SDefRegs, MDefRegs;
1158 for (auto [Reg, LaneMask] : *LiveSet) {
1159 assert(GCNRegPressure::getRegKind(Reg, MRI) == Kind);
1160 LiveInterval &LI = LIS.getInterval(Reg);
1161 if (LI.getNumValNums() == 1 ||
1162 (LI.hasSubRanges() &&
1163 llvm::all_of(LI.subranges(), [](const LiveInterval::SubRange &SR) {
1164 return SR.getNumValNums() == 1;
1165 }))) {
1166 SDefPressure.inc(Reg, LaneBitmask::getNone(), LaneMask, MRI);
1167 SDefRegs.push_back(Reg);
1168 } else {
1169 MDefPressure.inc(Reg, LaneBitmask::getNone(), LaneMask, MRI);
1170 MDefRegs.push_back(Reg);
1171 }
1172 }
1173 unsigned SDefNumRegs = SDefPressure.getNumRegs(Kind);
1174 unsigned MDefNumRegs = MDefPressure.getNumRegs(Kind);
1175 assert(SDefNumRegs + MDefNumRegs == MaxNumRegs);
1176
1177 auto printLoc = [&](const MachineBasicBlock *MBB, SlotIndex SI) {
1178 return Printable([&, MBB, SI](raw_ostream &OS) {
1179 OS << SI << ':' << printMBBReference(*MBB);
1180 if (MLI)
1181 if (const MachineLoop *ML = MLI->getLoopFor(MBB))
1182 OS << " (LoopHdr " << printMBBReference(*ML->getHeader())
1183 << ", Depth " << ML->getLoopDepth() << ")";
1184 });
1185 };
1186
1187 auto PrintRegInfo = [&](Register Reg, LaneBitmask LiveMask) {
1188 GCNRegPressure RegPressure;
1189 RegPressure.inc(Reg, LaneBitmask::getNone(), LiveMask, MRI);
1190 OS << " " << printReg(Reg, TRI) << ':'
1191 << TRI->getRegClassName(MRI.getRegClass(Reg)) << ", LiveMask "
1192 << PrintLaneMask(LiveMask) << " (" << RegPressure.getNumRegs(Kind) << ' '
1193 << RegName << "s)\n";
1194
1195 // Use std::map to sort def/uses by SlotIndex.
1196 std::map<SlotIndex, const MachineInstr *> Instrs;
1197 for (const MachineInstr &MI : MRI.reg_nodbg_instructions(Reg)) {
1198 Instrs[LIS.getInstructionIndex(MI).getRegSlot()] = &MI;
1199 }
1200
1201 for (const auto &[SI, MI] : Instrs) {
1202 OS << " ";
1203 if (MI->definesRegister(Reg, TRI))
1204 OS << "def ";
1205 if (MI->readsRegister(Reg, TRI))
1206 OS << "use ";
1207 OS << printLoc(MI->getParent(), SI) << ": " << *MI;
1208 }
1209 };
1210
1211 OS << "\n*** Register pressure info (" << RegName << "s) for " << MF.getName()
1212 << " ***\n";
1213 OS << "Max pressure is " << MaxNumRegs << ' ' << RegName << "s at "
1214 << printLoc(MaxPressureMI->getParent(), MaxPressureSlot) << ": "
1215 << *MaxPressureMI;
1216
1217 OS << "\nLive registers with single definition (" << SDefNumRegs << ' '
1218 << RegName << "s):\n";
1219
1220 // Sort SDefRegs by number of uses (smallest first)
1221 llvm::sort(SDefRegs, [&](Register A, Register B) {
1222 return std::distance(MRI.use_nodbg_begin(A), MRI.use_nodbg_end()) <
1223 std::distance(MRI.use_nodbg_begin(B), MRI.use_nodbg_end());
1224 });
1225
1226 for (const Register Reg : SDefRegs) {
1227 PrintRegInfo(Reg, LiveSet->lookup(Reg));
1228 }
1229
1230 OS << "\nLive registers with multiple definitions (" << MDefNumRegs << ' '
1231 << RegName << "s):\n";
1232 for (const Register Reg : MDefRegs) {
1233 PrintRegInfo(Reg, LiveSet->lookup(Reg));
1234 }
1235}
1236#endif
1237
1238unsigned llvm::estimateGreedyVGPRPressure(
1239 MachineBasicBlock::const_iterator RegionBegin,
1240 MachineBasicBlock::const_iterator RegionEnd,
1241 const GCNRPTracker::LiveRegSet &LiveIns, const LiveIntervals &LIS,
1242 const MachineRegisterInfo &MRI, const SIRegisterInfo &TRI) {
1243
1244 SetVector<const LiveInterval *> IntervalSet;
1245 IntervalSet.reserve(Size: LiveIns.size());
1246
1247 auto checkAndCollect = [&](Register VReg) {
1248 if (!VReg.isVirtual() || !LIS.hasInterval(Reg: VReg))
1249 return;
1250
1251 const TargetRegisterClass *RC = MRI.getRegClass(Reg: VReg);
1252 if (!TRI.hasVGPRs(RC))
1253 return;
1254
1255 const LiveInterval &LI = LIS.getInterval(Reg: VReg);
1256 IntervalSet.insert(X: &LI);
1257 };
1258
1259 // Collect live-ins.
1260 for (const auto &[RegNum, LaneMask] : LiveIns) {
1261 checkAndCollect(Register(RegNum));
1262 }
1263
1264 // Collect defs in region.
1265 for (MachineBasicBlock::const_iterator I = RegionBegin; I != RegionEnd; ++I) {
1266 for (const MachineOperand &MO : I->operands()) {
1267 if (!MO.isReg() || !MO.isDef())
1268 continue;
1269 checkAndCollect(MO.getReg());
1270 }
1271 }
1272
1273 SmallVector<const LiveInterval *> Intervals = IntervalSet.takeVector();
1274 llvm::sort(C&: Intervals, Comp: [](const LiveInterval *LHS, const LiveInterval *RHS) {
1275 return LHS->beginIndex() < RHS->beginIndex();
1276 });
1277
1278 LiveIntervalUnion::Allocator Alloc;
1279 std::vector<LiveIntervalUnion> RegFile;
1280 unsigned MaxRegsUsed = 0;
1281
1282 // Simulate greedy register allocation, assuming an unlimited number of
1283 // physical registers.
1284 for (const LiveInterval *LI : Intervals) {
1285 const TargetRegisterClass *RC = MRI.getRegClass(Reg: LI->reg());
1286 unsigned Width =
1287 std::max<unsigned>(a: 1, b: TRI.getRegSizeInBits(RC: *RC).getFixedValue() / 32);
1288 unsigned Alignment =
1289 std::max<unsigned>(a: 1, b: TRI.getRegClassAlignmentNumBits(RC) / 32);
1290
1291 unsigned Start = 0;
1292 while (true) {
1293 unsigned End = Start + Width;
1294 if (RegFile.size() < End)
1295 RegFile.resize(new_size: End, x: LiveIntervalUnion(Alloc));
1296
1297 bool Fits = true;
1298 for (unsigned Idx = Start; Idx < End; Idx++) {
1299 LiveIntervalUnion::Query Q(*LI, RegFile[Idx]);
1300 if (Q.checkInterference()) {
1301 Start = alignTo(Value: Idx + 1, Align: Alignment);
1302 Fits = false;
1303 break;
1304 }
1305 }
1306
1307 if (Fits) {
1308 for (unsigned Idx = Start; Idx < End; Idx++)
1309 RegFile[Idx].unify(VirtReg: *LI, Range: *LI);
1310 MaxRegsUsed = std::max(a: MaxRegsUsed, b: End);
1311 break;
1312 }
1313 }
1314 }
1315
1316 return MaxRegsUsed;
1317}
1318