1//===- GCNRegPressure.h -----------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file defines the GCNRegPressure class, which tracks registry pressure
11/// by bookkeeping number of SGPR/VGPRs used, weights for large SGPR/VGPRs. It
12/// also implements a compare function, which compares different register
13/// pressures, and declares one with max occupancy as winner.
14///
15//===----------------------------------------------------------------------===//
16
17#ifndef LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H
18#define LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H
19
20#include "GCNSubtarget.h"
21#include "llvm/CodeGen/LiveIntervals.h"
22#include "llvm/CodeGen/RegisterPressure.h"
23#include <algorithm>
24#include <array>
25#include <functional>
26
27namespace llvm {
28
29class MachineRegisterInfo;
30class raw_ostream;
31class SlotIndex;
32
33struct GCNRegPressure {
34 enum RegKind { SGPR, VGPR, AGPR, AVGPR, TOTAL_KINDS };
35
36 static constexpr const char *getName(RegKind Kind) {
37 const char *Names[] = {"SGPR", "VGPR", "AGPR", "AVGPR"};
38 assert(Kind < TOTAL_KINDS);
39 return Names[Kind];
40 }
41
42 GCNRegPressure() {
43 clear();
44 }
45
46 bool empty() const {
47 return !Value[SGPR] && !Value[VGPR] && !Value[AGPR] && !Value[AVGPR];
48 }
49
50 void clear() { Value.fill(u: 0); }
51
52 unsigned getNumRegs(RegKind Kind) const {
53 assert(Kind < TOTAL_KINDS);
54 return Value[Kind];
55 }
56
57 /// \returns the SGPR32 pressure
58 unsigned getSGPRNum() const { return Value[SGPR]; }
59 /// \returns the aggregated ArchVGPR32, AccVGPR32, and Pseudo AVGPR pressure
60 /// dependent upon \p UnifiedVGPRFile
61 unsigned getVGPRNum(bool UnifiedVGPRFile) const {
62 if (UnifiedVGPRFile) {
63 return Value[AGPR]
64 ? getUnifiedVGPRNum(NumArchVGPRs: Value[VGPR], NumAGPRs: Value[AGPR], NumAVGPRs: Value[AVGPR])
65 : Value[VGPR] + Value[AVGPR];
66 }
67 // AVGPR assignment priority is based on the width of the register. Account
68 // AVGPR pressure as VGPR.
69 return std::max(a: Value[VGPR] + Value[AVGPR], b: Value[AGPR]);
70 }
71
72 /// Returns the aggregated VGPR pressure, assuming \p NumArchVGPRs ArchVGPRs
73 /// \p NumAGPRs AGPRS, and \p NumAVGPRs AVGPRs for a target with a unified
74 /// VGPR file.
75 inline static unsigned getUnifiedVGPRNum(unsigned NumArchVGPRs,
76 unsigned NumAGPRs,
77 unsigned NumAVGPRs) {
78
79 // Assume AVGPRs will be assigned as VGPRs.
80 return alignTo(Value: NumArchVGPRs + NumAVGPRs,
81 Align: AMDGPU::IsaInfo::getArchVGPRAllocGranule()) +
82 NumAGPRs;
83 }
84
85 /// Returns the aggregated VGPR pressure for \p NumArchVGPRs ArchVGPRs and
86 /// \p NumAGPRs AGPRs, with AVGPRs already distributed across the banks.
87 static unsigned getBankedUnifiedVGPRNum(unsigned NumArchVGPRs,
88 unsigned NumAGPRs) {
89 return NumAGPRs ? getUnifiedVGPRNum(NumArchVGPRs, NumAGPRs, NumAVGPRs: 0)
90 : NumArchVGPRs;
91 }
92
93 /// \returns the ArchVGPR32 pressure, plus the AVGPRS which we assume will be
94 /// allocated as VGPR
95 unsigned getArchVGPRNum() const { return Value[VGPR] + Value[AVGPR]; }
96 /// \returns the AccVGPR32 pressure
97 unsigned getAGPRNum() const { return Value[AGPR]; }
98 /// \returns the AVGPR32 pressure
99 unsigned getAVGPRNum() const { return Value[AVGPR]; }
100
101 /// \returns the ArchVGPR32 and AccVGPR32 pressure after distributing the
102 /// bank-agnostic AVGPR pressure across the two banks. Follow the register
103 /// allocator, first charge AVGPRs to the ArchVGPR bank up to \p ArchLimit,
104 /// then to the AccVGPR bank up to \p AccLimit. AVGPRs that fit in neither
105 /// bank are charged to the ArchVGPR bank so that they show up as excess.
106 std::pair<unsigned, unsigned> getBankedVGPRNum(unsigned ArchLimit,
107 unsigned AccLimit) const {
108 unsigned Arch = Value[VGPR];
109 unsigned Acc = Value[AGPR];
110 unsigned Flex = Value[AVGPR];
111 unsigned ToArch = std::min(a: Flex, b: Arch < ArchLimit ? ArchLimit - Arch : 0);
112 Flex -= ToArch;
113 unsigned ToAcc = std::min(a: Flex, b: Acc < AccLimit ? AccLimit - Acc : 0);
114 Flex -= ToAcc;
115 return {Arch + ToArch + Flex, Acc + ToAcc};
116 }
117
118 unsigned getVGPRTuplesWeight() const {
119 return std::max(a: Value[TOTAL_KINDS + VGPR] + Value[TOTAL_KINDS + AVGPR],
120 b: Value[TOTAL_KINDS + AGPR]);
121 }
122 unsigned getSGPRTuplesWeight() const { return Value[TOTAL_KINDS + SGPR]; }
123
124 unsigned getOccupancy(const GCNSubtarget &ST,
125 unsigned DynamicVGPRBlockSize) const {
126 return std::min(a: ST.getOccupancyWithNumSGPRs(SGPRs: getSGPRNum()),
127 b: ST.getOccupancyWithNumVGPRs(VGPRs: getVGPRNum(UnifiedVGPRFile: ST.hasGFX90AInsts()),
128 DynamicVGPRBlockSize));
129 }
130
131 unsigned getVGPRSpills(MachineFunction &MF, unsigned ArchVGPRThreshold,
132 unsigned AGPRThreshold, unsigned CombinedThreshold) {
133 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
134 if (!ST.hasGFX90AInsts())
135 return 0;
136
137 unsigned ArchPressure = getArchVGPRNum();
138 unsigned AGPRPressure = getAGPRNum();
139
140 unsigned ArchSpill = ArchPressure > ArchVGPRThreshold
141 ? (ArchPressure - ArchVGPRThreshold)
142 : 0;
143 unsigned AGPRSpill =
144 AGPRPressure > AGPRThreshold ? (AGPRPressure - AGPRThreshold) : 0;
145
146 unsigned UnifiedPressure = getVGPRNum(/*UnifiedVGPRFile=*/UnifiedVGPRFile: true);
147 unsigned UnifiedSpill = UnifiedPressure > CombinedThreshold
148 ? (UnifiedPressure - CombinedThreshold)
149 : 0;
150
151 return std::max(a: UnifiedSpill, b: ArchSpill + AGPRSpill);
152 }
153
154 void inc(unsigned Reg,
155 LaneBitmask PrevMask,
156 LaneBitmask NewMask,
157 const MachineRegisterInfo &MRI);
158
159 /// Compares \p this GCNRegpressure to \p O, returning true if \p this is
160 /// less. Since GCNRegpressure contains different types of pressures, and due
161 /// to target-specific pecularities (e.g. we care about occupancy rather than
162 /// raw register usage), we determine if \p this GCNRegPressure is less than
163 /// \p O based on the following tiered comparisons (in order order of
164 /// precedence):
165 /// 1. Better occupancy
166 /// 2. Less spilling (first preference to VGPR spills, then to SGPR spills)
167 /// 3. Less tuple register pressure (first preference to VGPR tuples if we
168 /// determine that SGPR pressure is not important)
169 /// 4. Less raw register pressure (first preference to VGPR tuples if we
170 /// determine that SGPR pressure is not important)
171 bool less(const MachineFunction &MF, const GCNRegPressure &O,
172 unsigned MaxOccupancy = std::numeric_limits<unsigned>::max()) const;
173
174 bool operator==(const GCNRegPressure &O) const { return Value == O.Value; }
175
176 bool operator!=(const GCNRegPressure &O) const {
177 return !(*this == O);
178 }
179
180 GCNRegPressure &operator+=(const GCNRegPressure &RHS) {
181 for (unsigned I = 0; I < ValueArraySize; ++I)
182 Value[I] += RHS.Value[I];
183 return *this;
184 }
185
186 GCNRegPressure &operator-=(const GCNRegPressure &RHS) {
187 for (unsigned I = 0; I < ValueArraySize; ++I)
188 Value[I] -= RHS.Value[I];
189 return *this;
190 }
191
192 void dump() const;
193
194 static RegKind getRegKind(unsigned Reg, const MachineRegisterInfo &MRI) {
195 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
196 const SIRegisterInfo *STI = static_cast<const SIRegisterInfo *>(TRI);
197 return (RegKind)getRegKind(RC: MRI.getRegClass(Reg), STI);
198 }
199
200private:
201 static constexpr unsigned ValueArraySize = TOTAL_KINDS * 2;
202
203 /// Pressure for all register kinds (first all regular registers kinds, then
204 /// all tuple register kinds).
205 std::array<unsigned, ValueArraySize> Value;
206
207 static unsigned getRegKind(const TargetRegisterClass *RC,
208 const SIRegisterInfo *STI);
209
210 friend GCNRegPressure maxPerRegFile(const GCNRegPressure &P1,
211 const GCNRegPressure &P2);
212
213 friend Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST,
214 unsigned DynamicVGPRBlockSize);
215};
216
217/// \returns the maximum of \p P1 and \p P2 for each register file. The VGPR and
218/// AVGPR components are queried as one sum, so both come from the operand with
219/// the larger sum. The AGPR maximum stays separate also for a unified VGPR
220/// file: ArchVGPRs and AGPRs use separate ranges of it, so the unified pressure
221/// is the sum of both peaks even when they are at different points.
222inline GCNRegPressure maxPerRegFile(const GCNRegPressure &P1,
223 const GCNRegPressure &P2) {
224 GCNRegPressure Res;
225 // Value holds the register counts, then the tuple weights, in the same
226 // RegKind order.
227 for (unsigned I = 0; I < GCNRegPressure::ValueArraySize; ++I) {
228 switch (I % GCNRegPressure::TOTAL_KINDS) {
229 case GCNRegPressure::VGPR: {
230 unsigned AV = I - GCNRegPressure::VGPR + GCNRegPressure::AVGPR;
231 const GCNRegPressure &From =
232 P1.Value[I] + P1.Value[AV] >= P2.Value[I] + P2.Value[AV] ? P1 : P2;
233 Res.Value[I] = From.Value[I];
234 Res.Value[AV] = From.Value[AV];
235 break;
236 }
237 case GCNRegPressure::AVGPR:
238 // Already set with VGPR, which comes first in RegKind.
239 break;
240 default:
241 Res.Value[I] = std::max(a: P1.Value[I], b: P2.Value[I]);
242 }
243 }
244 return Res;
245}
246
247inline GCNRegPressure operator+(const GCNRegPressure &P1,
248 const GCNRegPressure &P2) {
249 GCNRegPressure Sum = P1;
250 Sum += P2;
251 return Sum;
252}
253
254inline GCNRegPressure operator-(const GCNRegPressure &P1,
255 const GCNRegPressure &P2) {
256 GCNRegPressure Diff = P1;
257 Diff -= P2;
258 return Diff;
259}
260
261////////////////////////////////////////////////////////////////////////////////
262// GCNRPTarget
263
264/// Models a register pressure target, allowing to evaluate and track register
265/// savings against that target from a starting \ref GCNRegPressure.
266class GCNRPTarget {
267public:
268 /// Sets up the target such that the register pressure starting at \p RP does
269 /// not show register spilling on function \p MF (w.r.t. the function's
270 /// mininum target occupancy).
271 GCNRPTarget(const MachineFunction &MF, const GCNRegPressure &RP);
272
273 /// Sets up the target such that the register pressure starting at \p RP does
274 /// not use more than \p NumSGPRs SGPRs and \p NumVGPRs VGPRs on function \p
275 /// MF.
276 GCNRPTarget(unsigned NumSGPRs, unsigned NumVGPRs, const MachineFunction &MF,
277 const GCNRegPressure &RP);
278
279 /// Sets up the target such that the register pressure starting at \p RP does
280 /// not prevent achieving an occupancy of at least \p Occupancy on function
281 /// \p MF.
282 GCNRPTarget(unsigned Occupancy, const MachineFunction &MF,
283 const GCNRegPressure &RP);
284
285 /// Changes the target (same semantics as constructor).
286 void setTarget(unsigned NumSGPRs, unsigned NumVGPRs);
287
288 const GCNRegPressure &getCurrentRP() const { return RP; }
289
290 void setRP(const GCNRegPressure &NewRP) { RP = NewRP; }
291
292 /// Determines whether saving virtual register \p Reg will be beneficial
293 /// towards achieving the RP target.
294 bool isSaveBeneficial(Register Reg) const;
295
296 /// Returns whether the benefit that saving \p SaveRP represents will be
297 /// beneficial towards achieving the RP target.
298 bool isSaveBeneficial(const GCNRegPressure &SaveRP) const;
299
300 /// Returns the benefit towards achieving the RP target that saving \p SaveRP
301 /// represents, in total number of registers saved across all classes.
302 unsigned getNumRegsBenefit(const GCNRegPressure &SaveRP) const;
303
304 /// Saves a total pressure of \p SaveRP.
305 void saveRP(const GCNRegPressure &SaveRP) {
306 assert(!RP.less(MF, SaveRP) && "saving beyond current RP");
307 RP -= SaveRP;
308 }
309
310 /// Whether \p TestRP is at or below the defined pressure target.
311 bool satisfied(const GCNRegPressure &TestRP) const;
312 /// Whether the current RP is at or below the defined pressure target.
313 bool satisfied() const { return satisfied(TestRP: RP); }
314 bool hasVectorRegisterExcess() const;
315
316 unsigned getMaxSGPRs() const { return MaxSGPRs; }
317 unsigned getMaxVGPRs() const {
318 return UnifiedRF ? MaxUnifiedVGPRs : MaxVGPRs;
319 }
320
321#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
322 friend raw_ostream &operator<<(raw_ostream &OS, const GCNRPTarget &Target) {
323 OS << "Actual/Target: " << Target.RP.getSGPRNum() << '/' << Target.MaxSGPRs
324 << " SGPRs, " << Target.RP.getArchVGPRNum() << '/' << Target.MaxVGPRs
325 << " ArchVGPRs, " << Target.RP.getAGPRNum() << '/' << Target.MaxVGPRs
326 << " AGPRs";
327
328 if (Target.MaxUnifiedVGPRs) {
329 OS << ", " << Target.RP.getVGPRNum(true) << '/' << Target.MaxUnifiedVGPRs
330 << " VGPRs (unified)";
331 }
332 return OS;
333 }
334#endif
335
336private:
337 const MachineFunction &MF;
338 const bool UnifiedRF;
339
340 /// Current register pressure.
341 GCNRegPressure RP;
342
343 /// Target number of SGPRs.
344 unsigned MaxSGPRs = 0;
345 /// Target number of ArchVGPRs and AGPRs.
346 unsigned MaxVGPRs = 0;
347 /// Target number of overall VGPRs for subtargets with unified RFs. Always 0
348 /// for subtargets with non-unified RFs.
349 unsigned MaxUnifiedVGPRs = 0;
350
351 GCNRPTarget(const GCNRegPressure &RP, const MachineFunction &MF)
352 : MF(MF), UnifiedRF(MF.getSubtarget<GCNSubtarget>().hasGFX90AInsts()),
353 RP(RP) {}
354};
355
356///////////////////////////////////////////////////////////////////////////////
357// GCNRPTracker
358
359class GCNRPTracker {
360public:
361 using LiveRegSet = DenseMap<unsigned, LaneBitmask>;
362
363protected:
364 LiveIntervals &LIS;
365 LiveRegSet LiveRegs;
366 GCNRegPressure CurPressure, MaxPressure;
367 MachineBasicBlock::const_iterator LastTrackedMI;
368 mutable const MachineRegisterInfo *MRI = nullptr;
369 std::function<void(const GCNRegPressure &)> PressureCallback;
370
371 GCNRPTracker(LiveIntervals &LIS_) : LIS(LIS_) {}
372
373 /// Folds the pressure \p RP of one program point into MaxPressure and passes
374 /// it to the pressure callback.
375 void updateMaxPressure(const GCNRegPressure &RP) {
376 MaxPressure = maxPerRegFile(P1: MaxPressure, P2: RP);
377 if (PressureCallback)
378 PressureCallback(RP);
379 }
380
381 /// Resets tracker before or \p After the provided \p MI, which can be a debug
382 /// instruction.
383 void reset(const MachineInstr &MI, bool After);
384
385 /// Resets tracker at the start or \p End of the \p MBB.
386 void reset(const MachineBasicBlock &MBB, bool End);
387
388 /// Resets tracker at the specified slot index \p SI.
389 void reset(const MachineRegisterInfo &MRI, SlotIndex SI);
390
391 LaneBitmask getLastUsedLanes(Register Reg, SlotIndex Pos) const;
392
393public:
394 /// Resets tracker with the provided \p LiveRegs.
395 void reset(const MachineRegisterInfo &MRI, const LiveRegSet &LiveRegs);
396
397 // live regs for the current state
398 const decltype(LiveRegs) &getLiveRegs() const { return LiveRegs; }
399 const MachineInstr *getLastTrackedMI() const {
400 return LastTrackedMI.isValid() ? &*LastTrackedMI : nullptr;
401 }
402
403 void clearMaxPressure() { MaxPressure.clear(); }
404
405 const GCNRegPressure &getMaxPressure() const { return MaxPressure; }
406
407 void resetMaxPressure() { MaxPressure = CurPressure; }
408
409 /// Sets a function that the tracker calls with the pressure of every program
410 /// point that it folds into the maximum pressure. A user can compute there a
411 /// maximum that maxPerRegFile() does not keep, e.g. of one register kind.
412 void setPressureCallback(std::function<void(const GCNRegPressure &)> CB) {
413 PressureCallback = std::move(CB);
414 }
415
416 GCNRegPressure getPressure() const { return CurPressure; }
417
418 decltype(LiveRegs) moveLiveRegs() {
419 return std::move(LiveRegs);
420 }
421};
422
423GCNRPTracker::LiveRegSet
424getLiveRegs(SlotIndex SI, const LiveIntervals &LIS,
425 const MachineRegisterInfo &MRI,
426 GCNRegPressure::RegKind RegKind = GCNRegPressure::TOTAL_KINDS);
427
428////////////////////////////////////////////////////////////////////////////////
429// GCNUpwardRPTracker
430
431class GCNUpwardRPTracker : public GCNRPTracker {
432public:
433 GCNUpwardRPTracker(LiveIntervals &LIS_) : GCNRPTracker(LIS_) {}
434
435 using GCNRPTracker::reset;
436
437 /// Resets tracker to the point just after \p MI (in program order), which can
438 /// be a debug instruction.
439 void reset(const MachineInstr &MI) { reset(MI, /*After=*/After: true); }
440
441 /// Move to the state of RP just before the \p MI.
442 void recede(const MachineInstr &MI);
443
444 /// \p returns whether the tracker's state after receding MI corresponds
445 /// to reported by LIS.
446 bool isValid() const;
447
448 GCNRegPressure getMaxPressureAndReset() {
449 GCNRegPressure RP = MaxPressure;
450 resetMaxPressure();
451 return RP;
452 }
453};
454
455////////////////////////////////////////////////////////////////////////////////
456// GCNDownwardRPTracker
457
458/// Common behavior for downward RP trackers. Unconstructible directly.
459class GCNDownwardRPTrackerBase : public GCNRPTracker {
460public:
461 /// Mostly copy/paste from CodeGen/RegisterPressure.cpp
462 /// Calculate the impact \p MI will have on CurPressure and \return the
463 /// speculated pressure. In order to support RP Speculation, this does not
464 /// rely on the implicit program ordering in the LiveIntervals.
465 GCNRegPressure bumpDownwardPressure(const MachineInstr *MI,
466 const SIRegisterInfo *TRI) const;
467
468protected:
469 GCNDownwardRPTrackerBase(LiveIntervals &LIS) : GCNRPTracker(LIS) {}
470
471 /// Retires the lanes of \p MI's used registers that die at \p MI, mirroring
472 /// RegPressureTracker::advance(). Early-clobber defs of \p MI are accounted
473 /// in \ref MaxPressure as live together with all of its uses. The uses of a
474 /// PHI are not retired since they do not die there.
475 void retireDeadUses(const MachineInstr &MI);
476
477 /// Retires the lanes defined by \p MI that are dead at \p SI, according to
478 /// LIS.
479 void retireDeadDefs(const MachineInstr &MI, SlotIndex SI);
480
481 /// Adds all lanes (dead or not) defined by \p MI's register operands to \ref
482 /// CurPressure.
483 void addDefs(const MachineInstr &MI);
484
485private:
486 /// Drop the lanes of \p Reg that are no longer live at \p SI, decreasing
487 /// CurPressure accordingly. \p Reg must be a virtual register that is
488 /// currently tracked as live.
489 void retireVirtReg(Register Reg, SlotIndex SI);
490};
491
492/// Downward RP tracker that relies on program order to advance down the
493/// instruction stream.
494class GCNDownwardRPTracker : public GCNDownwardRPTrackerBase {
495private:
496 /// Next MI to advance over.
497 MachineBasicBlock::const_iterator NextMI;
498 /// End iterator for basic block being advanced in.
499 MachineBasicBlock::const_iterator MBBEnd;
500
501public:
502 GCNDownwardRPTracker(LiveIntervals &LIS) : GCNDownwardRPTrackerBase(LIS) {}
503
504 using GCNDownwardRPTrackerBase::addDefs;
505 using GCNRPTracker::reset;
506
507 /// Returns the next MI that will be advanced over.
508 MachineBasicBlock::const_iterator getNext() const { return NextMI; }
509
510 /// \p return MaxPressure and clear it.
511 GCNRegPressure moveMaxPressure() {
512 auto Res = MaxPressure;
513 MaxPressure.clear();
514 return Res;
515 }
516
517 /// Reset tracker to the point before the \p MI (filling \ref LiveRegs upon
518 /// this point using \p LiveRegsCopy if provided or LIS otherwise) point using
519 /// LIS. \p End must be between the MI and the end of its parent block
520 /// (inclusive). \p returns false if the range [MI, End) is empty except debug
521 /// values.
522 bool reset(const MachineInstr &MI, MachineBasicBlock::const_iterator End,
523 const LiveRegSet *LiveRegsCopy = nullptr);
524
525 /// Move to the state right before \ref NextMI or the end of the MBB. This
526 /// retires defs of \ref LastTrackedMI that are dead after the instruction.
527 /// Its dead uses were already retired by \ref advanceToNext. \p returns
528 /// false if it reached the end of the block.
529 bool advanceBeforeNext();
530
531 /// Move to the state at \ref NextMI. This retires the uses that die at \p
532 /// NextMI, then adds the register lanes it defines to \ref CurPressure. Has
533 /// to be called after \ref advanceBeforeNext.
534 void advanceToNext();
535
536 /// Advance to the state at the next MI in program order. \p returns false if
537 /// it reached end of block.
538 bool advance();
539
540 /// Advance in program order until before \p End. Returns whether it actually
541 /// had to advance to reach \p End.
542 bool advance(MachineBasicBlock::const_iterator End);
543
544 /// Reset tracker to \p Begin (filling \ref LiveRegs upon this point using \p
545 /// LiveRegsCopy if provided or LIS otherwise) and advance to \p End, which
546 /// must be between \p Begin and the end of its parent block (inclusive). \p
547 /// returns false if the range [Begin, End) is empty except debug values.
548 bool advance(MachineBasicBlock::const_iterator Begin,
549 MachineBasicBlock::const_iterator End,
550 const LiveRegSet *LiveRegsCopy = nullptr);
551};
552
553/// Downward RP tracker that relies on the user to provide the next instruction
554/// down the instruction stream. It still uses LIS for liveness queries.
555class GCNManualDownwardRPTracker : public GCNDownwardRPTrackerBase {
556public:
557 GCNManualDownwardRPTracker(LiveIntervals &LIS)
558 : GCNDownwardRPTrackerBase(LIS) {}
559
560 using GCNRPTracker::reset;
561
562 /// Advances to the state after \p MI.
563 void advance(const MachineInstr &MI);
564};
565
566/// \returns the LaneMask of live lanes of \p Reg at position \p SI. Only the
567/// active lanes of \p LaneMaskFilter will be set in the return value. This is
568/// used, for example, to limit the live lanes to a specific subreg when
569/// calculating use masks.
570LaneBitmask getLiveLaneMask(unsigned Reg, SlotIndex SI,
571 const LiveIntervals &LIS,
572 const MachineRegisterInfo &MRI,
573 LaneBitmask LaneMaskFilter = LaneBitmask::getAll());
574
575LaneBitmask getLiveLaneMask(const LiveInterval &LI, SlotIndex SI,
576 const MachineRegisterInfo &MRI,
577 LaneBitmask LaneMaskFilter = LaneBitmask::getAll());
578
579/// creates a map MachineInstr -> LiveRegSet
580/// R - range of iterators on instructions
581/// After - upon entry or exit of every instruction
582/// Note: there is no entry in the map for instructions with empty live reg set
583/// Complexity = O(NumVirtRegs * averageLiveRangeSegmentsPerReg * lg(R))
584template <typename Range>
585DenseMap<MachineInstr*, GCNRPTracker::LiveRegSet>
586getLiveRegMap(Range &&R, bool After, LiveIntervals &LIS) {
587 std::vector<SlotIndex> Indexes;
588 Indexes.reserve(n: llvm::size(R));
589 auto &SII = *LIS.getSlotIndexes();
590 for (MachineInstr *I : R) {
591 auto SI = SII.getInstructionIndex(MI: *I);
592 Indexes.push_back(x: After ? SI.getDeadSlot() : SI.getBaseIndex());
593 }
594 llvm::sort(C&: Indexes);
595
596 auto &MRI = (*R.begin())->getMF()->getRegInfo();
597 DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> LiveRegMap;
598 SmallVector<SlotIndex, 32> LiveIdxs, SRLiveIdxs;
599 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
600 auto Reg = Register::index2VirtReg(Index: I);
601 if (!LIS.hasInterval(Reg))
602 continue;
603 auto &LI = LIS.getInterval(Reg);
604 LiveIdxs.clear();
605 if (!LI.findIndexesLiveAt(R&: Indexes, O: std::back_inserter(x&: LiveIdxs)))
606 continue;
607 if (!LI.hasSubRanges()) {
608 for (auto SI : LiveIdxs)
609 LiveRegMap[SII.getInstructionFromIndex(index: SI)][Reg] =
610 MRI.getMaxLaneMaskForVReg(Reg);
611 } else
612 for (const auto &S : LI.subranges()) {
613 // constrain search for subranges by indexes live at main range
614 SRLiveIdxs.clear();
615 S.findIndexesLiveAt(R&: LiveIdxs, O: std::back_inserter(x&: SRLiveIdxs));
616 for (auto SI : SRLiveIdxs)
617 LiveRegMap[SII.getInstructionFromIndex(index: SI)][Reg] |= S.LaneMask;
618 }
619 }
620 return LiveRegMap;
621}
622
623inline GCNRPTracker::LiveRegSet getLiveRegsAfter(const MachineInstr &MI,
624 const LiveIntervals &LIS) {
625 return getLiveRegs(SI: LIS.getInstructionIndex(Instr: MI).getDeadSlot(), LIS,
626 MRI: MI.getMF()->getRegInfo());
627}
628
629inline GCNRPTracker::LiveRegSet getLiveRegsBefore(const MachineInstr &MI,
630 const LiveIntervals &LIS) {
631 return getLiveRegs(SI: LIS.getInstructionIndex(Instr: MI).getBaseIndex(), LIS,
632 MRI: MI.getMF()->getRegInfo());
633}
634
635template <typename Range>
636GCNRegPressure getRegPressure(const MachineRegisterInfo &MRI,
637 Range &&LiveRegs) {
638 GCNRegPressure Res;
639 for (const auto &RM : LiveRegs)
640 Res.inc(Reg: RM.first, PrevMask: LaneBitmask::getNone(), NewMask: RM.second, MRI);
641 return Res;
642}
643
644bool isEqual(const GCNRPTracker::LiveRegSet &S1,
645 const GCNRPTracker::LiveRegSet &S2);
646
647Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST = nullptr,
648 unsigned DynamicVGPRBlockSize = 0);
649
650Printable print(const GCNRPTracker::LiveRegSet &LiveRegs,
651 const MachineRegisterInfo &MRI);
652
653Printable reportMismatch(const GCNRPTracker::LiveRegSet &LISLR,
654 const GCNRPTracker::LiveRegSet &TrackedL,
655 const TargetRegisterInfo *TRI, StringRef Pfx = " ");
656
657struct GCNRegPressurePrinter : public MachineFunctionPass {
658 static char ID;
659
660public:
661 GCNRegPressurePrinter() : MachineFunctionPass(ID) {}
662
663 bool runOnMachineFunction(MachineFunction &MF) override;
664
665 void getAnalysisUsage(AnalysisUsage &AU) const override {
666 AU.addRequired<LiveIntervalsWrapperPass>();
667 AU.setPreservesAll();
668 MachineFunctionPass::getAnalysisUsage(AU);
669 }
670};
671
672LLVM_ABI void dumpMaxRegPressure(MachineFunction &MF,
673 GCNRegPressure::RegKind Kind,
674 LiveIntervals &LIS,
675 const MachineLoopInfo *MLI);
676
677/// Estimate VGPR pressure using greedy, non-splitting register allocation
678/// simulation, accounting for live interval interference.
679/// \param RegionBegin Start iterator of the region
680/// \param RegionEnd End iterator of the region
681/// \param LiveIns Live-in registers for the region
682/// \returns estimated VGPR pressure
683unsigned estimateGreedyVGPRPressure(
684 MachineBasicBlock::const_iterator RegionBegin,
685 MachineBasicBlock::const_iterator RegionEnd,
686 const GCNRPTracker::LiveRegSet &LiveIns, const LiveIntervals &LIS,
687 const MachineRegisterInfo &MRI, const SIRegisterInfo &TRI);
688
689} // end namespace llvm
690
691#endif // LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H
692