1//===- GCNRegPressure.cpp -------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the GCNRegPressure class.
11///
12//===----------------------------------------------------------------------===//
13
14#include "GCNRegPressure.h"
15#include "AMDGPU.h"
16#include "SIMachineFunctionInfo.h"
17#include "llvm/CodeGen/MachineBasicBlock.h"
18#include "llvm/CodeGen/MachineLoopInfo.h"
19#include "llvm/CodeGen/RegisterPressure.h"
20
21using namespace llvm;
22
23#define DEBUG_TYPE "machine-scheduler"
24
25bool llvm::isEqual(const GCNRPTracker::LiveRegSet &S1,
26 const GCNRPTracker::LiveRegSet &S2) {
27 if (S1.size() != S2.size())
28 return false;
29
30 for (const auto &P : S1) {
31 auto I = S2.find(Val: P.first);
32 if (I == S2.end() || I->second != P.second)
33 return false;
34 }
35 return true;
36}
37
38///////////////////////////////////////////////////////////////////////////////
39// GCNRegPressure
40
41unsigned GCNRegPressure::getRegKind(const TargetRegisterClass *RC,
42 const SIRegisterInfo *STI) {
43 return STI->isSGPRClass(RC)
44 ? SGPR
45 : (STI->isAGPRClass(RC)
46 ? AGPR
47 : (STI->isVectorSuperClass(RC) ? AVGPR : VGPR));
48}
49
50void GCNRegPressure::inc(unsigned Reg,
51 LaneBitmask PrevMask,
52 LaneBitmask NewMask,
53 const MachineRegisterInfo &MRI) {
54 unsigned NewNumCoveredRegs = SIRegisterInfo::getNumCoveredRegs(LM: NewMask);
55 unsigned PrevNumCoveredRegs = SIRegisterInfo::getNumCoveredRegs(LM: PrevMask);
56 if (NewNumCoveredRegs == PrevNumCoveredRegs)
57 return;
58
59 int Sign = 1;
60 if (NewMask < PrevMask) {
61 std::swap(a&: NewMask, b&: PrevMask);
62 std::swap(a&: NewNumCoveredRegs, b&: PrevNumCoveredRegs);
63 Sign = -1;
64 }
65 assert(PrevMask < NewMask && PrevNumCoveredRegs < NewNumCoveredRegs &&
66 "prev mask should always be lesser than new");
67
68 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
69 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
70 const SIRegisterInfo *STI = static_cast<const SIRegisterInfo *>(TRI);
71 unsigned RegKind = getRegKind(RC, STI);
72 if (TRI->getRegSizeInBits(RC: *RC) != 32) {
73 // Reg is from a tuple register class.
74 if (PrevMask.none()) {
75 unsigned TupleIdx = TOTAL_KINDS + RegKind;
76 Value[TupleIdx] += Sign * TRI->getRegClassWeight(RC).RegWeight;
77 }
78 // Pressure scales with number of new registers covered by the new mask.
79 // Note when true16 is enabled, we can no longer safely use the following
80 // approach to calculate the difference in the number of 32-bit registers
81 // between two masks:
82 //
83 // Sign *= SIRegisterInfo::getNumCoveredRegs(~PrevMask & NewMask);
84 //
85 // The issue is that the mask calculation `~PrevMask & NewMask` doesn't
86 // properly account for partial usage of a 32-bit register when dealing with
87 // 16-bit registers.
88 //
89 // Consider this example:
90 // Assume PrevMask = 0b0010 and NewMask = 0b1111. Here, the correct register
91 // usage difference should be 1, because even though PrevMask uses only half
92 // of a 32-bit register, it should still be counted as a full register use.
93 // However, the mask calculation yields `~PrevMask & NewMask = 0b1101`, and
94 // calling `getNumCoveredRegs` returns 2 instead of 1. This incorrect
95 // calculation can lead to integer overflow when Sign = -1.
96 Sign *= NewNumCoveredRegs - PrevNumCoveredRegs;
97 }
98 Value[RegKind] += Sign;
99}
100
101namespace {
102struct RegExcess {
103 unsigned SGPR = 0;
104 unsigned VGPR = 0;
105 unsigned ArchVGPR = 0;
106 unsigned AGPR = 0;
107
108 bool anyExcess() const { return SGPR || VGPR || ArchVGPR || AGPR; }
109 bool hasVectorRegisterExcess() const { return VGPR || ArchVGPR || AGPR; }
110
111 RegExcess(const MachineFunction &MF, const GCNRegPressure &RP)
112 : RegExcess(MF, RP, GCNRPTarget(MF, RP)) {}
113 RegExcess(const MachineFunction &MF, const GCNRegPressure &RP,
114 const GCNRPTarget &Target) {
115 unsigned MaxSGPRs = Target.getMaxSGPRs();
116 unsigned MaxVGPRs = Target.getMaxVGPRs();
117
118 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
119 SGPR = std::max(a: static_cast<int>(RP.getSGPRNum() - MaxSGPRs), b: 0);
120
121 // The number of virtual VGPRs required to handle excess SGPR
122 unsigned WaveSize = ST.getWavefrontSize();
123 unsigned VGPRForSGPRSpills = divideCeil(Numerator: SGPR, Denominator: WaveSize);
124
125 unsigned MaxArchVGPRs = ST.getAddressableNumArchVGPRs();
126
127 // Unified excess pressure conditions, accounting for VGPRs used for SGPR
128 // spills
129 VGPR = std::max(a: static_cast<int>(RP.getVGPRNum(UnifiedVGPRFile: ST.hasGFX90AInsts()) +
130 VGPRForSGPRSpills - MaxVGPRs),
131 b: 0);
132
133 unsigned ArchVGPRLimit = ST.hasGFX90AInsts() ? MaxArchVGPRs : MaxVGPRs;
134 // Arch VGPR excess pressure conditions, accounting for VGPRs used for SGPR
135 // spills
136 ArchVGPR = std::max(a: static_cast<int>(RP.getArchVGPRNum() +
137 VGPRForSGPRSpills - ArchVGPRLimit),
138 b: 0);
139
140 // AGPR excess pressure conditions
141 AGPR = std::max(a: static_cast<int>(RP.getAGPRNum() - ArchVGPRLimit), b: 0);
142 }
143};
144} // namespace
145
146bool GCNRegPressure::less(const MachineFunction &MF, const GCNRegPressure &O,
147 unsigned MaxOccupancy) const {
148 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
149 unsigned DynamicVGPRBlockSize =
150 MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize();
151
152 const auto SGPROcc = std::min(a: MaxOccupancy,
153 b: ST.getOccupancyWithNumSGPRs(SGPRs: getSGPRNum()));
154 const auto VGPROcc = std::min(
155 a: MaxOccupancy, b: ST.getOccupancyWithNumVGPRs(VGPRs: getVGPRNum(UnifiedVGPRFile: ST.hasGFX90AInsts()),
156 DynamicVGPRBlockSize));
157 const auto OtherSGPROcc = std::min(a: MaxOccupancy,
158 b: ST.getOccupancyWithNumSGPRs(SGPRs: O.getSGPRNum()));
159 const auto OtherVGPROcc =
160 std::min(a: MaxOccupancy,
161 b: ST.getOccupancyWithNumVGPRs(VGPRs: O.getVGPRNum(UnifiedVGPRFile: ST.hasGFX90AInsts()),
162 DynamicVGPRBlockSize));
163
164 const auto Occ = std::min(a: SGPROcc, b: VGPROcc);
165 const auto OtherOcc = std::min(a: OtherSGPROcc, b: OtherVGPROcc);
166
167 // Give first precedence to the better occupancy.
168 if (Occ != OtherOcc)
169 return Occ > OtherOcc;
170
171 unsigned MaxVGPRs = ST.getMaxNumVGPRs(MF);
172
173 RegExcess Excess(MF, *this);
174 RegExcess OtherExcess(MF, O);
175
176 unsigned MaxArchVGPRs = ST.getAddressableNumArchVGPRs();
177
178 bool ExcessRP = Excess.anyExcess();
179 bool OtherExcessRP = OtherExcess.anyExcess();
180
181 // Give second precedence to the reduced number of spills to hold the register
182 // pressure.
183 if (ExcessRP || OtherExcessRP) {
184 // The difference in excess VGPR pressure, after including VGPRs used for
185 // SGPR spills
186 int VGPRDiff =
187 ((OtherExcess.VGPR + OtherExcess.ArchVGPR + OtherExcess.AGPR) -
188 (Excess.VGPR + Excess.ArchVGPR + Excess.AGPR));
189
190 int SGPRDiff = OtherExcess.SGPR - Excess.SGPR;
191
192 if (VGPRDiff != 0)
193 return VGPRDiff > 0;
194 if (SGPRDiff != 0) {
195 unsigned PureExcessVGPR =
196 std::max(a: static_cast<int>(getVGPRNum(UnifiedVGPRFile: ST.hasGFX90AInsts()) - MaxVGPRs),
197 b: 0) +
198 std::max(a: static_cast<int>(getVGPRNum(UnifiedVGPRFile: false) - MaxArchVGPRs), b: 0);
199 unsigned OtherPureExcessVGPR =
200 std::max(
201 a: static_cast<int>(O.getVGPRNum(UnifiedVGPRFile: ST.hasGFX90AInsts()) - MaxVGPRs),
202 b: 0) +
203 std::max(a: static_cast<int>(O.getVGPRNum(UnifiedVGPRFile: false) - MaxArchVGPRs), b: 0);
204
205 // If we have a special case where there is a tie in excess VGPR, but one
206 // of the pressures has VGPR usage from SGPR spills, prefer the pressure
207 // with SGPR spills.
208 if (PureExcessVGPR != OtherPureExcessVGPR)
209 return SGPRDiff < 0;
210 // If both pressures have the same excess pressure before and after
211 // accounting for SGPR spills, prefer fewer SGPR spills.
212 return SGPRDiff > 0;
213 }
214 }
215
216 bool SGPRImportant = SGPROcc < VGPROcc;
217 const bool OtherSGPRImportant = OtherSGPROcc < OtherVGPROcc;
218
219 // If both pressures disagree on what is more important compare vgprs.
220 if (SGPRImportant != OtherSGPRImportant) {
221 SGPRImportant = false;
222 }
223
224 // Give third precedence to lower register tuple pressure.
225 bool SGPRFirst = SGPRImportant;
226 for (int I = 2; I > 0; --I, SGPRFirst = !SGPRFirst) {
227 if (SGPRFirst) {
228 auto SW = getSGPRTuplesWeight();
229 auto OtherSW = O.getSGPRTuplesWeight();
230 if (SW != OtherSW)
231 return SW < OtherSW;
232 } else {
233 auto VW = getVGPRTuplesWeight();
234 auto OtherVW = O.getVGPRTuplesWeight();
235 if (VW != OtherVW)
236 return VW < OtherVW;
237 }
238 }
239
240 // Give final precedence to lower general RP.
241 return SGPRImportant ? (getSGPRNum() < O.getSGPRNum()):
242 (getVGPRNum(UnifiedVGPRFile: ST.hasGFX90AInsts()) <
243 O.getVGPRNum(UnifiedVGPRFile: ST.hasGFX90AInsts()));
244}
245
246Printable llvm::print(const GCNRegPressure &RP, const GCNSubtarget *ST,
247 unsigned DynamicVGPRBlockSize) {
248 return Printable([&RP, ST, DynamicVGPRBlockSize](raw_ostream &OS) {
249 OS << "VGPRs: " << RP.getArchVGPRNum() << ' '
250 << "AGPRs: " << RP.getAGPRNum();
251 if (ST)
252 OS << "(O"
253 << ST->getOccupancyWithNumVGPRs(VGPRs: RP.getVGPRNum(UnifiedVGPRFile: ST->hasGFX90AInsts()),
254 DynamicVGPRBlockSize)
255 << ')';
256 OS << ", SGPRs: " << RP.getSGPRNum();
257 if (ST)
258 OS << "(O" << ST->getOccupancyWithNumSGPRs(SGPRs: RP.getSGPRNum()) << ')';
259 OS << ", LVGPR WT: " << RP.getVGPRTuplesWeight()
260 << ", LSGPR WT: " << RP.getSGPRTuplesWeight();
261 if (ST)
262 OS << " -> Occ: " << RP.getOccupancy(ST: *ST, DynamicVGPRBlockSize);
263 OS << '\n';
264 });
265}
266
267static LaneBitmask getDefRegMask(const MachineOperand &MO,
268 const MachineRegisterInfo &MRI) {
269 assert(MO.isDef() && MO.isReg() && MO.getReg().isVirtual());
270
271 // We don't rely on read-undef flag because in case of tentative schedule
272 // tracking it isn't set correctly yet. This works correctly however since
273 // use mask has been tracked before using LIS.
274 return MO.getSubReg() == 0 ?
275 MRI.getMaxLaneMaskForVReg(Reg: MO.getReg()) :
276 MRI.getTargetRegisterInfo()->getSubRegIndexLaneMask(SubIdx: MO.getSubReg());
277}
278
279static void
280collectVirtualRegUses(SmallVectorImpl<VRegMaskOrUnit> &VRegMaskOrUnits,
281 const MachineInstr &MI, const LiveIntervals &LIS,
282 const MachineRegisterInfo &MRI) {
283
284 auto &TRI = *MRI.getTargetRegisterInfo();
285 for (const auto &MO : MI.operands()) {
286 if (!MO.isReg() || !MO.getReg().isVirtual())
287 continue;
288 if (!MO.isUse() || !MO.readsReg())
289 continue;
290
291 Register Reg = MO.getReg();
292 auto I = llvm::find_if(Range&: VRegMaskOrUnits, P: [Reg](const VRegMaskOrUnit &RM) {
293 return RM.VRegOrUnit.asVirtualReg() == Reg;
294 });
295
296 auto &P = I == VRegMaskOrUnits.end()
297 ? VRegMaskOrUnits.emplace_back(Args: VirtRegOrUnit(Reg),
298 Args: LaneBitmask::getNone())
299 : *I;
300
301 P.LaneMask |= MO.getSubReg() ? TRI.getSubRegIndexLaneMask(SubIdx: MO.getSubReg())
302 : MRI.getMaxLaneMaskForVReg(Reg);
303 }
304
305 SlotIndex InstrSI;
306 for (auto &P : VRegMaskOrUnits) {
307 auto &LI = LIS.getInterval(Reg: P.VRegOrUnit.asVirtualReg());
308 if (!LI.hasSubRanges())
309 continue;
310
311 // For a tentative schedule LIS isn't updated yet but livemask should
312 // remain the same on any schedule. Subreg defs can be reordered but they
313 // all must dominate uses anyway.
314 if (!InstrSI)
315 InstrSI = LIS.getInstructionIndex(Instr: MI).getBaseIndex();
316
317 P.LaneMask = getLiveLaneMask(LI, SI: InstrSI, MRI, LaneMaskFilter: P.LaneMask);
318 }
319}
320
321/// Mostly copy/paste from CodeGen/RegisterPressure.cpp
322static LaneBitmask getLanesWithProperty(
323 const LiveIntervals &LIS, const MachineRegisterInfo &MRI,
324 bool TrackLaneMasks, Register Reg, SlotIndex Pos,
325 function_ref<bool(const LiveRange &LR, SlotIndex Pos)> Property) {
326 assert(Reg.isVirtual());
327 const LiveInterval &LI = LIS.getInterval(Reg);
328 LaneBitmask Result;
329 if (TrackLaneMasks && LI.hasSubRanges()) {
330 for (const LiveInterval::SubRange &SR : LI.subranges()) {
331 if (Property(SR, Pos))
332 Result |= SR.LaneMask;
333 }
334 } else if (Property(LI, Pos)) {
335 Result =
336 TrackLaneMasks ? MRI.getMaxLaneMaskForVReg(Reg) : LaneBitmask::getAll();
337 }
338
339 return Result;
340}
341
342/// Mostly copy/paste from CodeGen/RegisterPressure.cpp
343/// Helper to find a vreg use between two indices {PriorUseIdx, NextUseIdx}.
344/// The query starts with a lane bitmask which gets lanes/bits removed for every
345/// use we find.
346static LaneBitmask findUseBetween(unsigned Reg, LaneBitmask LastUseMask,
347 SlotIndex PriorUseIdx, SlotIndex NextUseIdx,
348 const MachineRegisterInfo &MRI,
349 const SIRegisterInfo *TRI,
350 const LiveIntervals *LIS,
351 bool Upward = false) {
352 for (const MachineOperand &MO : MRI.use_nodbg_operands(Reg)) {
353 if (MO.isUndef())
354 continue;
355 const MachineInstr *MI = MO.getParent();
356 SlotIndex InstSlot = LIS->getInstructionIndex(Instr: *MI).getRegSlot();
357 bool InRange = Upward ? (InstSlot > PriorUseIdx && InstSlot <= NextUseIdx)
358 : (InstSlot >= PriorUseIdx && InstSlot < NextUseIdx);
359 if (!InRange)
360 continue;
361
362 unsigned SubRegIdx = MO.getSubReg();
363 LaneBitmask UseMask = TRI->getSubRegIndexLaneMask(SubIdx: SubRegIdx);
364 LastUseMask &= ~UseMask;
365 if (LastUseMask.none())
366 return LaneBitmask::getNone();
367 }
368 return LastUseMask;
369}
370
371////////////////////////////////////////////////////////////////////////////////
372// GCNRPTarget
373
374GCNRPTarget::GCNRPTarget(const MachineFunction &MF, const GCNRegPressure &RP)
375 : GCNRPTarget(RP, MF) {
376 const Function &F = MF.getFunction();
377 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
378 setTarget(NumSGPRs: ST.getMaxNumSGPRs(F), NumVGPRs: ST.getMaxNumVGPRs(F));
379}
380
381GCNRPTarget::GCNRPTarget(unsigned NumSGPRs, unsigned NumVGPRs,
382 const MachineFunction &MF, const GCNRegPressure &RP)
383 : GCNRPTarget(RP, MF) {
384 setTarget(NumSGPRs, NumVGPRs);
385}
386
387GCNRPTarget::GCNRPTarget(unsigned Occupancy, const MachineFunction &MF,
388 const GCNRegPressure &RP)
389 : GCNRPTarget(RP, MF) {
390 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
391 unsigned DynamicVGPRBlockSize =
392 MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize();
393 setTarget(NumSGPRs: ST.getMaxNumSGPRs(WavesPerEU: Occupancy, /*Addressable=*/false),
394 NumVGPRs: ST.getMaxNumVGPRs(WavesPerEU: Occupancy, DynamicVGPRBlockSize));
395}
396
397void GCNRPTarget::setTarget(unsigned NumSGPRs, unsigned NumVGPRs) {
398 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
399 MaxSGPRs = std::min(a: ST.getAddressableNumSGPRs(), b: NumSGPRs);
400 MaxVGPRs = std::min(a: ST.getAddressableNumArchVGPRs(), b: NumVGPRs);
401 if (UnifiedRF) {
402 unsigned DynamicVGPRBlockSize =
403 MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize();
404 MaxUnifiedVGPRs =
405 std::min(a: ST.getAddressableNumVGPRs(DynamicVGPRBlockSize), b: NumVGPRs);
406 } else {
407 MaxUnifiedVGPRs = 0;
408 }
409}
410
411bool GCNRPTarget::isSaveBeneficial(Register Reg) const {
412 const MachineRegisterInfo &MRI = MF.getRegInfo();
413 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
414 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
415 const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo *>(TRI);
416
417 RegExcess Excess(MF, RP, *this);
418
419 if (SRI->isSGPRClass(RC))
420 return Excess.SGPR;
421
422 if (SRI->isAGPRClass(RC))
423 return (UnifiedRF && Excess.VGPR) || Excess.AGPR;
424
425 return (UnifiedRF && Excess.VGPR) || Excess.ArchVGPR;
426}
427
428bool GCNRPTarget::isSaveBeneficial(const GCNRegPressure &SaveRP) const {
429 RegExcess Excess(MF, RP, *this);
430 if (SaveRP.getSGPRNum() != 0 && Excess.SGPR != 0)
431 return true;
432 if (SaveRP.getArchVGPRNum() != 0 && Excess.ArchVGPR != 0)
433 return true;
434 if (SaveRP.getAGPRNum() != 0 && Excess.AGPR != 0)
435 return true;
436 if (UnifiedRF && Excess.VGPR != 0)
437 return SaveRP.getArchVGPRNum() != 0 || SaveRP.getAGPRNum() != 0;
438 return false;
439}
440
441unsigned GCNRPTarget::getNumRegsBenefit(const GCNRegPressure &SaveRP) const {
442 RegExcess Excess(MF, RP, *this);
443 const unsigned NumVGPRAboveAddrLimit =
444 std::min(a: Excess.ArchVGPR, b: SaveRP.getArchVGPRNum()) +
445 std::min(a: Excess.AGPR, b: SaveRP.getAGPRNum());
446 unsigned NumRegsSaved =
447 std::min(a: Excess.SGPR, b: SaveRP.getSGPRNum()) + NumVGPRAboveAddrLimit;
448
449 if (UnifiedRF && Excess.VGPR) {
450 // We have already accounted for excess pressure above addressive limits for
451 // the individual VGPR classes. However for targets with unified RFs there
452 // is also a unified VGPR pressure (ArchVGPR + AGPR combination) limit to
453 // honor that may be more restrictive that the per-VGPR-class limits. We
454 // must also be careful not to double-count VGPR saves that may contribute
455 // to lowering pressure both above the addressable limit in their respective
456 // class as well as in the unified VGPR limit.
457 const unsigned VGPRSave = SaveRP.getArchVGPRNum() + SaveRP.getAGPRNum();
458 if (NumVGPRAboveAddrLimit < VGPRSave)
459 NumRegsSaved += std::min(a: Excess.VGPR, b: VGPRSave - NumVGPRAboveAddrLimit);
460 }
461
462 return NumRegsSaved;
463}
464
465bool GCNRPTarget::satisfied(const GCNRegPressure &TestRP) const {
466 if (TestRP.getSGPRNum() > MaxSGPRs || TestRP.getVGPRNum(UnifiedVGPRFile: false) > MaxVGPRs)
467 return false;
468 if (UnifiedRF && TestRP.getVGPRNum(UnifiedVGPRFile: true) > MaxUnifiedVGPRs)
469 return false;
470 return true;
471}
472
473bool GCNRPTarget::hasVectorRegisterExcess() const {
474 RegExcess Excess(MF, RP, *this);
475 return Excess.hasVectorRegisterExcess();
476}
477
478///////////////////////////////////////////////////////////////////////////////
479// GCNRPTracker
480
481LaneBitmask llvm::getLiveLaneMask(unsigned Reg, SlotIndex SI,
482 const LiveIntervals &LIS,
483 const MachineRegisterInfo &MRI,
484 LaneBitmask LaneMaskFilter) {
485 return getLiveLaneMask(LI: LIS.getInterval(Reg), SI, MRI, LaneMaskFilter);
486}
487
488LaneBitmask llvm::getLiveLaneMask(const LiveInterval &LI, SlotIndex SI,
489 const MachineRegisterInfo &MRI,
490 LaneBitmask LaneMaskFilter) {
491 LaneBitmask LiveMask;
492 if (LI.hasSubRanges()) {
493 for (const auto &S : LI.subranges())
494 if ((S.LaneMask & LaneMaskFilter).any() && S.liveAt(index: SI)) {
495 LiveMask |= S.LaneMask;
496 assert(LiveMask == (LiveMask & MRI.getMaxLaneMaskForVReg(LI.reg())));
497 }
498 } else if (LI.liveAt(index: SI)) {
499 LiveMask = MRI.getMaxLaneMaskForVReg(Reg: LI.reg());
500 }
501 LiveMask &= LaneMaskFilter;
502 return LiveMask;
503}
504
505GCNRPTracker::LiveRegSet llvm::getLiveRegs(SlotIndex SI,
506 const LiveIntervals &LIS,
507 const MachineRegisterInfo &MRI,
508 GCNRegPressure::RegKind RegKind) {
509 GCNRPTracker::LiveRegSet LiveRegs;
510 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
511 auto Reg = Register::index2VirtReg(Index: I);
512 if (RegKind != GCNRegPressure::TOTAL_KINDS &&
513 GCNRegPressure::getRegKind(Reg, MRI) != RegKind)
514 continue;
515 if (!LIS.hasInterval(Reg))
516 continue;
517 auto LiveMask = getLiveLaneMask(Reg, SI, LIS, MRI);
518 if (LiveMask.any())
519 LiveRegs[Reg] = LiveMask;
520 }
521 return LiveRegs;
522}
523
524void GCNRPTracker::reset(const MachineInstr &MI, bool After) {
525 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
526 if (!MI.isDebugInstr()) {
527 SlotIndex SI = LIS.getInstructionIndex(Instr: MI);
528 if (After)
529 SI = SI.getDeadSlot();
530 reset(MRI, SI);
531 return;
532 }
533
534 // Look for the first valid index after the provided debug MI.
535 MachineBasicBlock::const_iterator It = MI.getIterator(),
536 MBBEnd = MI.getParent()->end();
537 MachineBasicBlock::const_iterator NonDbgMI =
538 skipDebugInstructionsForward(It, End: MBBEnd);
539 if (NonDbgMI == MBBEnd) {
540 // There are no non-debug instruction between MI and the end of the
541 // block, so we reset the tracker at the end of the block.
542 reset(MBB: *MI.getParent(), /*End=*/true);
543 return;
544 }
545 // MI is a debug instruction so register pressure before or after it is
546 // identical. Since we moved forward to finding a non-debug instruction
547 // in the block, we reset the tracker before that instruction i.e., at its
548 // base index.
549 reset(MRI, SI: LIS.getInstructionIndex(Instr: *NonDbgMI));
550}
551
552void GCNRPTracker::reset(const MachineBasicBlock &MBB, bool End) {
553 SlotIndex SI = End ? LIS.getSlotIndexes()->getMBBLastIdx(MBB: &MBB)
554 : LIS.getMBBStartIdx(mbb: &MBB);
555 reset(MRI: MBB.getParent()->getRegInfo(), SI);
556}
557
558void GCNRPTracker::reset(const MachineRegisterInfo &MRI, SlotIndex SI) {
559 this->MRI = &MRI;
560 LastTrackedMI = nullptr;
561 LiveRegs = llvm::getLiveRegs(SI, LIS, MRI);
562 MaxPressure = CurPressure = getRegPressure(MRI, LiveRegs);
563}
564
565void GCNRPTracker::reset(const MachineRegisterInfo &MRI,
566 const LiveRegSet &LiveRegs) {
567 this->MRI = &MRI;
568 LastTrackedMI = nullptr;
569 if (&this->LiveRegs != &LiveRegs)
570 this->LiveRegs = LiveRegs;
571 MaxPressure = CurPressure = getRegPressure(MRI, LiveRegs);
572}
573
574/// Mostly copy/paste from CodeGen/RegisterPressure.cpp
575LaneBitmask GCNRPTracker::getLastUsedLanes(Register Reg, SlotIndex Pos) const {
576 return getLanesWithProperty(
577 LIS, MRI: *MRI, TrackLaneMasks: true, Reg, Pos: Pos.getBaseIndex(),
578 Property: [](const LiveRange &LR, SlotIndex Pos) {
579 const LiveRange::Segment *S = LR.getSegmentContaining(Idx: Pos);
580 return S != nullptr && S->end == Pos.getRegSlot();
581 });
582}
583
584////////////////////////////////////////////////////////////////////////////////
585// GCNUpwardRPTracker
586
587void GCNUpwardRPTracker::recede(const MachineInstr &MI) {
588 assert(MRI && "call reset first");
589
590 LastTrackedMI = &MI;
591
592 if (MI.isDebugInstr())
593 return;
594
595 // Kill all defs.
596 GCNRegPressure DefPressure, ECDefPressure;
597 bool HasECDefs = false;
598 for (const MachineOperand &MO : MI.all_defs()) {
599 if (!MO.getReg().isVirtual())
600 continue;
601
602 Register Reg = MO.getReg();
603 LaneBitmask DefMask = getDefRegMask(MO, MRI: *MRI);
604
605 // Treat a def as fully live at the moment of definition: keep a record.
606 if (MO.isEarlyClobber()) {
607 ECDefPressure.inc(Reg, PrevMask: LaneBitmask::getNone(), NewMask: DefMask, MRI: *MRI);
608 HasECDefs = true;
609 } else
610 DefPressure.inc(Reg, PrevMask: LaneBitmask::getNone(), NewMask: DefMask, MRI: *MRI);
611
612 auto I = LiveRegs.find(Val: Reg);
613 if (I == LiveRegs.end())
614 continue;
615
616 LaneBitmask &LiveMask = I->second;
617 LaneBitmask PrevMask = LiveMask;
618 LiveMask &= ~DefMask;
619 CurPressure.inc(Reg, PrevMask, NewMask: LiveMask, MRI: *MRI);
620 if (LiveMask.none())
621 LiveRegs.erase(I);
622 }
623
624 // Update MaxPressure with defs pressure.
625 DefPressure += CurPressure;
626 if (HasECDefs)
627 DefPressure += ECDefPressure;
628 MaxPressure = max(P1: DefPressure, P2: MaxPressure);
629
630 // Make uses alive.
631 SmallVector<VRegMaskOrUnit, 8> RegUses;
632 collectVirtualRegUses(VRegMaskOrUnits&: RegUses, MI, LIS, MRI: *MRI);
633 for (const VRegMaskOrUnit &U : RegUses) {
634 LaneBitmask &LiveMask = LiveRegs[U.VRegOrUnit.asVirtualReg()];
635 LaneBitmask PrevMask = LiveMask;
636 LiveMask |= U.LaneMask;
637 CurPressure.inc(Reg: U.VRegOrUnit.asVirtualReg(), PrevMask, NewMask: LiveMask, MRI: *MRI);
638 }
639
640 // Update MaxPressure with uses plus early-clobber defs pressure.
641 MaxPressure = HasECDefs ? max(P1: CurPressure + ECDefPressure, P2: MaxPressure)
642 : max(P1: CurPressure, P2: MaxPressure);
643
644 assert(CurPressure == getRegPressure(*MRI, LiveRegs));
645}
646
647////////////////////////////////////////////////////////////////////////////////
648// GCNDownwardRPTracker
649
650bool GCNDownwardRPTracker::reset(const MachineInstr &MI,
651 MachineBasicBlock::const_iterator End,
652 const LiveRegSet *LiveRegsCopy) {
653 MBBEnd = MI.getParent()->end();
654 assert(End == MBBEnd ||
655 End->getParent()->end() == MBBEnd && "end unrelated to MI block");
656 NextMI = &MI;
657 NextMI = skipDebugInstructionsForward(It: NextMI, End);
658
659 // Do not use the MI to compute live registers when a set is provided.
660 // Otherwise the first non-debug instruction after the provided one (or the
661 // end of the block, if no such instruction exists) serves as the basis to
662 // compute a live register set.
663 if (LiveRegsCopy)
664 GCNRPTracker::reset(MRI: MI.getMF()->getRegInfo(), LiveRegs: *LiveRegsCopy);
665 else if (NextMI != MBBEnd)
666 GCNRPTracker::reset(MI: *NextMI, /*After=*/false);
667 else
668 GCNRPTracker::reset(MBB: *MI.getParent(), /*End=*/true);
669 return NextMI != End;
670}
671
672bool GCNDownwardRPTracker::advanceBeforeNext(MachineInstr *MI,
673 bool UseInternalIterator) {
674 assert(MRI && "call reset first");
675 SlotIndex SI;
676 const MachineInstr *CurrMI;
677 if (UseInternalIterator) {
678 if (!LastTrackedMI)
679 return NextMI == MBBEnd;
680
681 assert(NextMI == MBBEnd || !NextMI->isDebugInstr());
682 CurrMI = LastTrackedMI;
683
684 SI = NextMI == MBBEnd
685 ? LIS.getInstructionIndex(Instr: *LastTrackedMI).getDeadSlot()
686 : LIS.getInstructionIndex(Instr: *NextMI).getBaseIndex();
687 } else { //! UseInternalIterator
688 SI = LIS.getInstructionIndex(Instr: *MI).getBaseIndex();
689 CurrMI = MI;
690 }
691
692 assert(SI.isValid());
693
694 // Remove dead registers or mask bits.
695 SmallSet<Register, 8> SeenRegs;
696 for (auto &MO : CurrMI->operands()) {
697 if (!MO.isReg() || !MO.getReg().isVirtual())
698 continue;
699 if (MO.isUse() && !MO.readsReg())
700 continue;
701 if (!UseInternalIterator && MO.isDef())
702 continue;
703 if (!SeenRegs.insert(V: MO.getReg()).second)
704 continue;
705 const LiveInterval &LI = LIS.getInterval(Reg: MO.getReg());
706 if (LI.hasSubRanges()) {
707 auto It = LiveRegs.end();
708 for (const auto &S : LI.subranges()) {
709 if (!S.liveAt(index: SI)) {
710 if (It == LiveRegs.end()) {
711 It = LiveRegs.find(Val: MO.getReg());
712 if (It == LiveRegs.end())
713 llvm_unreachable("register isn't live");
714 }
715 auto PrevMask = It->second;
716 It->second &= ~S.LaneMask;
717 CurPressure.inc(Reg: MO.getReg(), PrevMask, NewMask: It->second, MRI: *MRI);
718 }
719 }
720 if (It != LiveRegs.end() && It->second.none())
721 LiveRegs.erase(I: It);
722 } else if (!LI.liveAt(index: SI)) {
723 auto It = LiveRegs.find(Val: MO.getReg());
724 if (It == LiveRegs.end())
725 llvm_unreachable("register isn't live");
726 CurPressure.inc(Reg: MO.getReg(), PrevMask: It->second, NewMask: LaneBitmask::getNone(), MRI: *MRI);
727 LiveRegs.erase(I: It);
728 }
729 }
730
731 MaxPressure = max(P1: MaxPressure, P2: CurPressure);
732
733 LastTrackedMI = nullptr;
734
735 return UseInternalIterator && (NextMI == MBBEnd);
736}
737
738void GCNDownwardRPTracker::advanceToNext(MachineInstr *MI,
739 bool UseInternalIterator) {
740 if (UseInternalIterator) {
741 LastTrackedMI = &*NextMI++;
742 NextMI = skipDebugInstructionsForward(It: NextMI, End: MBBEnd);
743 } else {
744 LastTrackedMI = MI;
745 }
746
747 const MachineInstr *CurrMI = LastTrackedMI;
748
749 // Add new registers or mask bits.
750 for (const auto &MO : CurrMI->all_defs()) {
751 Register Reg = MO.getReg();
752 if (!Reg.isVirtual())
753 continue;
754 auto &LiveMask = LiveRegs[Reg];
755 auto PrevMask = LiveMask;
756 LiveMask |= getDefRegMask(MO, MRI: *MRI);
757 CurPressure.inc(Reg, PrevMask, NewMask: LiveMask, MRI: *MRI);
758 }
759
760 MaxPressure = max(P1: MaxPressure, P2: CurPressure);
761}
762
763bool GCNDownwardRPTracker::advance(MachineInstr *MI, bool UseInternalIterator) {
764 if (UseInternalIterator && NextMI == MBBEnd)
765 return false;
766
767 advanceBeforeNext(MI, UseInternalIterator);
768 advanceToNext(MI, UseInternalIterator);
769 if (!UseInternalIterator) {
770 const MachineInstr *SavedLastTrackedMI = LastTrackedMI;
771 // We must remove any dead def lanes from the current RP
772 advanceBeforeNext(MI, UseInternalIterator: true);
773 // Restore LastTrackedMI set by advanceToNext, otherwise
774 // speculative queries (bumpDownwardPressure) don't
775 // know the last scheduled instruction and fail to
776 // correctly estimate pressure change.
777 LastTrackedMI = SavedLastTrackedMI;
778 }
779 return true;
780}
781
782bool GCNDownwardRPTracker::advance(MachineBasicBlock::const_iterator End) {
783 bool AnyAdvance = false;
784 while (NextMI != End && advance())
785 AnyAdvance = true;
786 return AnyAdvance;
787}
788
789bool GCNDownwardRPTracker::advance(MachineBasicBlock::const_iterator Begin,
790 MachineBasicBlock::const_iterator End,
791 const LiveRegSet *LiveRegsCopy) {
792 if (!reset(MI: *Begin, End, LiveRegsCopy))
793 return false;
794 return advance(End);
795}
796
797Printable llvm::reportMismatch(const GCNRPTracker::LiveRegSet &LISLR,
798 const GCNRPTracker::LiveRegSet &TrackedLR,
799 const TargetRegisterInfo *TRI, StringRef Pfx) {
800 return Printable([&LISLR, &TrackedLR, TRI, Pfx](raw_ostream &OS) {
801 for (auto const &P : TrackedLR) {
802 auto I = LISLR.find(Val: P.first);
803 if (I == LISLR.end()) {
804 OS << Pfx << printReg(Reg: P.first, TRI) << ":L" << PrintLaneMask(LaneMask: P.second)
805 << " isn't found in LIS reported set\n";
806 } else if (I->second != P.second) {
807 OS << Pfx << printReg(Reg: P.first, TRI)
808 << " masks doesn't match: LIS reported " << PrintLaneMask(LaneMask: I->second)
809 << ", tracked " << PrintLaneMask(LaneMask: P.second) << '\n';
810 }
811 }
812 for (auto const &P : LISLR) {
813 auto I = TrackedLR.find(Val: P.first);
814 if (I == TrackedLR.end()) {
815 OS << Pfx << printReg(Reg: P.first, TRI) << ":L" << PrintLaneMask(LaneMask: P.second)
816 << " isn't found in tracked set\n";
817 }
818 }
819 });
820}
821
822GCNRegPressure
823GCNDownwardRPTracker::bumpDownwardPressure(const MachineInstr *MI,
824 const SIRegisterInfo *TRI) const {
825 assert(!MI->isDebugOrPseudoInstr() && "Expect a nondebug instruction.");
826
827 SlotIndex SlotIdx;
828 SlotIdx = LIS.getInstructionIndex(Instr: *MI).getRegSlot();
829
830 SlotIndex CurrIdx;
831 const MachineBasicBlock *MBB = MI->getParent();
832 MachineBasicBlock::const_iterator StartPos =
833 LastTrackedMI ? std::next(x: LastTrackedMI->getIterator()) : MBB->begin();
834 MachineBasicBlock::const_iterator IdxPos =
835 skipDebugInstructionsForward(It: StartPos, End: MBB->end());
836 if (IdxPos == MBB->end()) {
837 CurrIdx = LIS.getMBBEndIdx(mbb: MBB);
838 } else {
839 CurrIdx = LIS.getInstructionIndex(Instr: *IdxPos).getRegSlot();
840 }
841
842 // Account for register pressure similar to RegPressureTracker::recede().
843 RegisterOperands RegOpers;
844 RegOpers.collect(MI: *MI, TRI: *TRI, MRI: *MRI, TrackLaneMasks: true, /*IgnoreDead=*/false);
845 RegOpers.adjustLaneLiveness(LIS, MRI: *MRI, Pos: SlotIdx);
846 GCNRegPressure TempPressure = CurPressure;
847
848 for (const VRegMaskOrUnit &Use : RegOpers.Uses) {
849 if (!Use.VRegOrUnit.isVirtualReg())
850 continue;
851 Register Reg = Use.VRegOrUnit.asVirtualReg();
852 LaneBitmask LastUseMask = getLastUsedLanes(Reg, Pos: SlotIdx);
853 if (LastUseMask.none())
854 continue;
855 // The LastUseMask is queried from the liveness information of instruction
856 // which may be further down the schedule. Some lanes may actually not be
857 // last uses for the current position.
858 // FIXME: allow the caller to pass in the list of vreg uses that remain
859 // to be bottom-scheduled to avoid searching uses at each query.
860 LastUseMask =
861 findUseBetween(Reg, LastUseMask, PriorUseIdx: CurrIdx, NextUseIdx: SlotIdx, MRI: *MRI, TRI, LIS: &LIS);
862 if (LastUseMask.none())
863 continue;
864
865 auto It = LiveRegs.find(Val: Reg);
866 LaneBitmask LiveMask = It != LiveRegs.end() ? It->second : LaneBitmask(0);
867 LaneBitmask NewMask = LiveMask & ~LastUseMask;
868 TempPressure.inc(Reg, PrevMask: LiveMask, NewMask, MRI: *MRI);
869 }
870
871 // Generate liveness for defs.
872 for (const VRegMaskOrUnit &Def : RegOpers.Defs) {
873 if (!Def.VRegOrUnit.isVirtualReg())
874 continue;
875 Register Reg = Def.VRegOrUnit.asVirtualReg();
876 auto It = LiveRegs.find(Val: Reg);
877 LaneBitmask LiveMask = It != LiveRegs.end() ? It->second : LaneBitmask(0);
878 LaneBitmask NewMask = LiveMask | Def.LaneMask;
879 TempPressure.inc(Reg, PrevMask: LiveMask, NewMask, MRI: *MRI);
880 }
881
882 return TempPressure;
883}
884
885bool GCNUpwardRPTracker::isValid() const {
886 const auto &SI = LIS.getInstructionIndex(Instr: *LastTrackedMI).getBaseIndex();
887 const auto LISLR = llvm::getLiveRegs(SI, LIS, MRI: *MRI);
888 const auto &TrackedLR = LiveRegs;
889
890 if (!isEqual(S1: LISLR, S2: TrackedLR)) {
891 dbgs() << "\nGCNUpwardRPTracker error: Tracked and"
892 " LIS reported livesets mismatch:\n"
893 << print(LiveRegs: LISLR, MRI: *MRI);
894 reportMismatch(LISLR, TrackedLR, TRI: MRI->getTargetRegisterInfo());
895 return false;
896 }
897
898 auto LISPressure = getRegPressure(MRI: *MRI, LiveRegs: LISLR);
899 if (LISPressure != CurPressure) {
900 dbgs() << "GCNUpwardRPTracker error: Pressure sets different\nTracked: "
901 << print(RP: CurPressure) << "LIS rpt: " << print(RP: LISPressure);
902 return false;
903 }
904 return true;
905}
906
907Printable llvm::print(const GCNRPTracker::LiveRegSet &LiveRegs,
908 const MachineRegisterInfo &MRI) {
909 return Printable([&LiveRegs, &MRI](raw_ostream &OS) {
910 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
911 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
912 Register Reg = Register::index2VirtReg(Index: I);
913 auto It = LiveRegs.find(Val: Reg);
914 if (It != LiveRegs.end() && It->second.any())
915 OS << ' ' << printReg(Reg, TRI) << ':' << PrintLaneMask(LaneMask: It->second);
916 }
917 OS << '\n';
918 });
919}
920
921void GCNRegPressure::dump() const { dbgs() << print(RP: *this); }
922
923static cl::opt<bool> UseDownwardTracker(
924 "amdgpu-print-rp-downward",
925 cl::desc("Use GCNDownwardRPTracker for GCNRegPressurePrinter pass"),
926 cl::init(Val: false), cl::Hidden);
927
928char llvm::GCNRegPressurePrinter::ID = 0;
929char &llvm::GCNRegPressurePrinterID = GCNRegPressurePrinter::ID;
930
931INITIALIZE_PASS(GCNRegPressurePrinter, "amdgpu-print-rp", "", true, true)
932
933// Return lanemask of Reg's subregs that are live-through at [Begin, End] and
934// are fully covered by Mask.
935static LaneBitmask
936getRegLiveThroughMask(const MachineRegisterInfo &MRI, const LiveIntervals &LIS,
937 Register Reg, SlotIndex Begin, SlotIndex End,
938 LaneBitmask Mask = LaneBitmask::getAll()) {
939
940 auto IsInOneSegment = [Begin, End](const LiveRange &LR) -> bool {
941 auto *Segment = LR.getSegmentContaining(Idx: Begin);
942 return Segment && Segment->contains(I: End);
943 };
944
945 LaneBitmask LiveThroughMask;
946 const LiveInterval &LI = LIS.getInterval(Reg);
947 if (LI.hasSubRanges()) {
948 for (auto &SR : LI.subranges()) {
949 if ((SR.LaneMask & Mask) == SR.LaneMask && IsInOneSegment(SR))
950 LiveThroughMask |= SR.LaneMask;
951 }
952 } else {
953 LaneBitmask RegMask = MRI.getMaxLaneMaskForVReg(Reg);
954 if ((RegMask & Mask) == RegMask && IsInOneSegment(LI))
955 LiveThroughMask = RegMask;
956 }
957
958 return LiveThroughMask;
959}
960
961bool GCNRegPressurePrinter::runOnMachineFunction(MachineFunction &MF) {
962 const MachineRegisterInfo &MRI = MF.getRegInfo();
963 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
964 const LiveIntervals &LIS = getAnalysis<LiveIntervalsWrapperPass>().getLIS();
965
966 auto &OS = dbgs();
967
968// Leading spaces are important for YAML syntax.
969#define PFX " "
970
971 OS << "---\nname: " << MF.getName() << "\nbody: |\n";
972
973 auto printRP = [](const GCNRegPressure &RP) {
974 return Printable([&RP](raw_ostream &OS) {
975 OS << format(PFX " %-5d", Vals: RP.getSGPRNum())
976 << format(Fmt: " %-5d", Vals: RP.getVGPRNum(UnifiedVGPRFile: false));
977 });
978 };
979
980 auto ReportLISMismatchIfAny = [&](const GCNRPTracker::LiveRegSet &TrackedLR,
981 const GCNRPTracker::LiveRegSet &LISLR) {
982 if (LISLR != TrackedLR) {
983 OS << PFX " mis LIS: " << llvm::print(LiveRegs: LISLR, MRI)
984 << reportMismatch(LISLR, TrackedLR, TRI, PFX " ");
985 }
986 };
987
988 // Register pressure before and at an instruction (in program order).
989 SmallVector<std::pair<GCNRegPressure, GCNRegPressure>, 16> RP;
990
991 for (auto &MBB : MF) {
992 RP.clear();
993 RP.reserve(N: MBB.size());
994
995 OS << PFX;
996 MBB.printName(os&: OS);
997 OS << ":\n";
998
999 SlotIndex MBBStartSlot = LIS.getSlotIndexes()->getMBBStartIdx(mbb: &MBB);
1000 SlotIndex MBBLastSlot = LIS.getSlotIndexes()->getMBBLastIdx(MBB: &MBB);
1001
1002 GCNRPTracker::LiveRegSet LiveIn, LiveOut;
1003 GCNRegPressure RPAtMBBEnd;
1004
1005 if (UseDownwardTracker) {
1006 if (MBB.empty()) {
1007 LiveIn = LiveOut = getLiveRegs(SI: MBBStartSlot, LIS, MRI);
1008 RPAtMBBEnd = getRegPressure(MRI, LiveRegs&: LiveIn);
1009 } else {
1010 GCNDownwardRPTracker RPT(LIS);
1011 RPT.reset(MI: MBB.front(), End: MBB.end());
1012
1013 LiveIn = RPT.getLiveRegs();
1014
1015 while (!RPT.advanceBeforeNext()) {
1016 GCNRegPressure RPBeforeMI = RPT.getPressure();
1017 RPT.advanceToNext();
1018 RP.emplace_back(Args&: RPBeforeMI, Args: RPT.getPressure());
1019 }
1020
1021 LiveOut = RPT.getLiveRegs();
1022 RPAtMBBEnd = RPT.getPressure();
1023 }
1024 } else {
1025 GCNUpwardRPTracker RPT(LIS);
1026 RPT.reset(MRI, SI: MBBLastSlot);
1027
1028 LiveOut = RPT.getLiveRegs();
1029 RPAtMBBEnd = RPT.getPressure();
1030
1031 for (auto &MI : reverse(C&: MBB)) {
1032 RPT.resetMaxPressure();
1033 RPT.recede(MI);
1034 if (!MI.isDebugInstr())
1035 RP.emplace_back(Args: RPT.getPressure(), Args: RPT.getMaxPressure());
1036 }
1037
1038 LiveIn = RPT.getLiveRegs();
1039 }
1040
1041 OS << PFX " Live-in: " << llvm::print(LiveRegs: LiveIn, MRI);
1042 if (!UseDownwardTracker)
1043 ReportLISMismatchIfAny(LiveIn, getLiveRegs(SI: MBBStartSlot, LIS, MRI));
1044
1045 OS << PFX " SGPR VGPR\n";
1046 int I = 0;
1047 for (auto &MI : MBB) {
1048 if (!MI.isDebugInstr()) {
1049 auto &[RPBeforeInstr, RPAtInstr] =
1050 RP[UseDownwardTracker ? I : (RP.size() - 1 - I)];
1051 ++I;
1052 OS << printRP(RPBeforeInstr) << '\n' << printRP(RPAtInstr) << " ";
1053 } else
1054 OS << PFX " ";
1055 MI.print(OS);
1056 }
1057 OS << printRP(RPAtMBBEnd) << '\n';
1058
1059 OS << PFX " Live-out:" << llvm::print(LiveRegs: LiveOut, MRI);
1060 if (UseDownwardTracker)
1061 ReportLISMismatchIfAny(LiveOut, getLiveRegs(SI: MBBLastSlot, LIS, MRI));
1062
1063 GCNRPTracker::LiveRegSet LiveThrough;
1064 for (auto [Reg, Mask] : LiveIn) {
1065 LaneBitmask MaskIntersection = Mask & LiveOut.lookup(Val: Reg);
1066 if (MaskIntersection.any()) {
1067 LaneBitmask LTMask = getRegLiveThroughMask(
1068 MRI, LIS, Reg, Begin: MBBStartSlot, End: MBBLastSlot, Mask: MaskIntersection);
1069 if (LTMask.any())
1070 LiveThrough[Reg] = LTMask;
1071 }
1072 }
1073 OS << PFX " Live-thr:" << llvm::print(LiveRegs: LiveThrough, MRI);
1074 OS << printRP(getRegPressure(MRI, LiveRegs&: LiveThrough)) << '\n';
1075 }
1076 OS << "...\n";
1077 return false;
1078
1079#undef PFX
1080}
1081
1082#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1083LLVM_DUMP_METHOD void llvm::dumpMaxRegPressure(MachineFunction &MF,
1084 GCNRegPressure::RegKind Kind,
1085 LiveIntervals &LIS,
1086 const MachineLoopInfo *MLI) {
1087
1088 const MachineRegisterInfo &MRI = MF.getRegInfo();
1089 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
1090 auto &OS = dbgs();
1091 const char *RegName = GCNRegPressure::getName(Kind);
1092
1093 unsigned MaxNumRegs = 0;
1094 const MachineInstr *MaxPressureMI = nullptr;
1095 GCNUpwardRPTracker RPT(LIS);
1096 for (const MachineBasicBlock &MBB : MF) {
1097 RPT.reset(MRI, LIS.getSlotIndexes()->getMBBEndIdx(&MBB).getPrevSlot());
1098 for (const MachineInstr &MI : reverse(MBB)) {
1099 RPT.recede(MI);
1100 unsigned NumRegs = RPT.getMaxPressure().getNumRegs(Kind);
1101 if (NumRegs > MaxNumRegs) {
1102 MaxNumRegs = NumRegs;
1103 MaxPressureMI = &MI;
1104 }
1105 }
1106 }
1107
1108 SlotIndex MISlot = LIS.getInstructionIndex(*MaxPressureMI);
1109
1110 // Max pressure can occur at either the early-clobber or register slot.
1111 // Choose the maximum liveset between both slots. This is ugly but this is
1112 // diagnostic code.
1113 SlotIndex ECSlot = MISlot.getRegSlot(true);
1114 SlotIndex RSlot = MISlot.getRegSlot(false);
1115 GCNRPTracker::LiveRegSet ECLiveSet = getLiveRegs(ECSlot, LIS, MRI, Kind);
1116 GCNRPTracker::LiveRegSet RLiveSet = getLiveRegs(RSlot, LIS, MRI, Kind);
1117 unsigned ECNumRegs = getRegPressure(MRI, ECLiveSet).getNumRegs(Kind);
1118 unsigned RNumRegs = getRegPressure(MRI, RLiveSet).getNumRegs(Kind);
1119 GCNRPTracker::LiveRegSet *LiveSet =
1120 ECNumRegs > RNumRegs ? &ECLiveSet : &RLiveSet;
1121 SlotIndex MaxPressureSlot = ECNumRegs > RNumRegs ? ECSlot : RSlot;
1122 assert(getRegPressure(MRI, *LiveSet).getNumRegs(Kind) == MaxNumRegs);
1123
1124 // Split live registers into single-def and multi-def sets.
1125 GCNRegPressure SDefPressure, MDefPressure;
1126 SmallVector<Register, 16> SDefRegs, MDefRegs;
1127 for (auto [Reg, LaneMask] : *LiveSet) {
1128 assert(GCNRegPressure::getRegKind(Reg, MRI) == Kind);
1129 LiveInterval &LI = LIS.getInterval(Reg);
1130 if (LI.getNumValNums() == 1 ||
1131 (LI.hasSubRanges() &&
1132 llvm::all_of(LI.subranges(), [](const LiveInterval::SubRange &SR) {
1133 return SR.getNumValNums() == 1;
1134 }))) {
1135 SDefPressure.inc(Reg, LaneBitmask::getNone(), LaneMask, MRI);
1136 SDefRegs.push_back(Reg);
1137 } else {
1138 MDefPressure.inc(Reg, LaneBitmask::getNone(), LaneMask, MRI);
1139 MDefRegs.push_back(Reg);
1140 }
1141 }
1142 unsigned SDefNumRegs = SDefPressure.getNumRegs(Kind);
1143 unsigned MDefNumRegs = MDefPressure.getNumRegs(Kind);
1144 assert(SDefNumRegs + MDefNumRegs == MaxNumRegs);
1145
1146 auto printLoc = [&](const MachineBasicBlock *MBB, SlotIndex SI) {
1147 return Printable([&, MBB, SI](raw_ostream &OS) {
1148 OS << SI << ':' << printMBBReference(*MBB);
1149 if (MLI)
1150 if (const MachineLoop *ML = MLI->getLoopFor(MBB))
1151 OS << " (LoopHdr " << printMBBReference(*ML->getHeader())
1152 << ", Depth " << ML->getLoopDepth() << ")";
1153 });
1154 };
1155
1156 auto PrintRegInfo = [&](Register Reg, LaneBitmask LiveMask) {
1157 GCNRegPressure RegPressure;
1158 RegPressure.inc(Reg, LaneBitmask::getNone(), LiveMask, MRI);
1159 OS << " " << printReg(Reg, TRI) << ':'
1160 << TRI->getRegClassName(MRI.getRegClass(Reg)) << ", LiveMask "
1161 << PrintLaneMask(LiveMask) << " (" << RegPressure.getNumRegs(Kind) << ' '
1162 << RegName << "s)\n";
1163
1164 // Use std::map to sort def/uses by SlotIndex.
1165 std::map<SlotIndex, const MachineInstr *> Instrs;
1166 for (const MachineInstr &MI : MRI.reg_nodbg_instructions(Reg)) {
1167 Instrs[LIS.getInstructionIndex(MI).getRegSlot()] = &MI;
1168 }
1169
1170 for (const auto &[SI, MI] : Instrs) {
1171 OS << " ";
1172 if (MI->definesRegister(Reg, TRI))
1173 OS << "def ";
1174 if (MI->readsRegister(Reg, TRI))
1175 OS << "use ";
1176 OS << printLoc(MI->getParent(), SI) << ": " << *MI;
1177 }
1178 };
1179
1180 OS << "\n*** Register pressure info (" << RegName << "s) for " << MF.getName()
1181 << " ***\n";
1182 OS << "Max pressure is " << MaxNumRegs << ' ' << RegName << "s at "
1183 << printLoc(MaxPressureMI->getParent(), MaxPressureSlot) << ": "
1184 << *MaxPressureMI;
1185
1186 OS << "\nLive registers with single definition (" << SDefNumRegs << ' '
1187 << RegName << "s):\n";
1188
1189 // Sort SDefRegs by number of uses (smallest first)
1190 llvm::sort(SDefRegs, [&](Register A, Register B) {
1191 return std::distance(MRI.use_nodbg_begin(A), MRI.use_nodbg_end()) <
1192 std::distance(MRI.use_nodbg_begin(B), MRI.use_nodbg_end());
1193 });
1194
1195 for (const Register Reg : SDefRegs) {
1196 PrintRegInfo(Reg, LiveSet->lookup(Reg));
1197 }
1198
1199 OS << "\nLive registers with multiple definitions (" << MDefNumRegs << ' '
1200 << RegName << "s):\n";
1201 for (const Register Reg : MDefRegs) {
1202 PrintRegInfo(Reg, LiveSet->lookup(Reg));
1203 }
1204}
1205#endif
1206