| 1 | //===- GCNRegPressure.h -----------------------------------------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | /// |
| 9 | /// \file |
| 10 | /// This file defines the GCNRegPressure class, which tracks registry pressure |
| 11 | /// by bookkeeping number of SGPR/VGPRs used, weights for large SGPR/VGPRs. It |
| 12 | /// also implements a compare function, which compares different register |
| 13 | /// pressures, and declares one with max occupancy as winner. |
| 14 | /// |
| 15 | //===----------------------------------------------------------------------===// |
| 16 | |
| 17 | #ifndef LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H |
| 18 | #define LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H |
| 19 | |
| 20 | #include "GCNSubtarget.h" |
| 21 | #include "llvm/CodeGen/LiveIntervals.h" |
| 22 | #include "llvm/CodeGen/RegisterPressure.h" |
| 23 | #include <algorithm> |
| 24 | #include <array> |
| 25 | #include <functional> |
| 26 | |
| 27 | namespace llvm { |
| 28 | |
| 29 | class MachineRegisterInfo; |
| 30 | class raw_ostream; |
| 31 | class SlotIndex; |
| 32 | |
| 33 | struct GCNRegPressure { |
| 34 | enum RegKind { SGPR, VGPR, AGPR, AVGPR, TOTAL_KINDS }; |
| 35 | |
| 36 | static constexpr const char *getName(RegKind Kind) { |
| 37 | const char *Names[] = {"SGPR" , "VGPR" , "AGPR" , "AVGPR" }; |
| 38 | assert(Kind < TOTAL_KINDS); |
| 39 | return Names[Kind]; |
| 40 | } |
| 41 | |
| 42 | GCNRegPressure() { |
| 43 | clear(); |
| 44 | } |
| 45 | |
| 46 | bool empty() const { |
| 47 | return !Value[SGPR] && !Value[VGPR] && !Value[AGPR] && !Value[AVGPR]; |
| 48 | } |
| 49 | |
| 50 | void clear() { Value.fill(u: 0); } |
| 51 | |
| 52 | unsigned getNumRegs(RegKind Kind) const { |
| 53 | assert(Kind < TOTAL_KINDS); |
| 54 | return Value[Kind]; |
| 55 | } |
| 56 | |
| 57 | /// \returns the SGPR32 pressure |
| 58 | unsigned getSGPRNum() const { return Value[SGPR]; } |
| 59 | /// \returns the aggregated ArchVGPR32, AccVGPR32, and Pseudo AVGPR pressure |
| 60 | /// dependent upon \p UnifiedVGPRFile |
| 61 | unsigned getVGPRNum(bool UnifiedVGPRFile) const { |
| 62 | if (UnifiedVGPRFile) { |
| 63 | return Value[AGPR] |
| 64 | ? getUnifiedVGPRNum(NumArchVGPRs: Value[VGPR], NumAGPRs: Value[AGPR], NumAVGPRs: Value[AVGPR]) |
| 65 | : Value[VGPR] + Value[AVGPR]; |
| 66 | } |
| 67 | // AVGPR assignment priority is based on the width of the register. Account |
| 68 | // AVGPR pressure as VGPR. |
| 69 | return std::max(a: Value[VGPR] + Value[AVGPR], b: Value[AGPR]); |
| 70 | } |
| 71 | |
| 72 | /// Returns the aggregated VGPR pressure, assuming \p NumArchVGPRs ArchVGPRs |
| 73 | /// \p NumAGPRs AGPRS, and \p NumAVGPRs AVGPRs for a target with a unified |
| 74 | /// VGPR file. |
| 75 | inline static unsigned getUnifiedVGPRNum(unsigned NumArchVGPRs, |
| 76 | unsigned NumAGPRs, |
| 77 | unsigned NumAVGPRs) { |
| 78 | |
| 79 | // Assume AVGPRs will be assigned as VGPRs. |
| 80 | return alignTo(Value: NumArchVGPRs + NumAVGPRs, |
| 81 | Align: AMDGPU::IsaInfo::getArchVGPRAllocGranule()) + |
| 82 | NumAGPRs; |
| 83 | } |
| 84 | |
| 85 | /// Returns the aggregated VGPR pressure for \p NumArchVGPRs ArchVGPRs and |
| 86 | /// \p NumAGPRs AGPRs, with AVGPRs already distributed across the banks. |
| 87 | static unsigned getBankedUnifiedVGPRNum(unsigned NumArchVGPRs, |
| 88 | unsigned NumAGPRs) { |
| 89 | return NumAGPRs ? getUnifiedVGPRNum(NumArchVGPRs, NumAGPRs, NumAVGPRs: 0) |
| 90 | : NumArchVGPRs; |
| 91 | } |
| 92 | |
| 93 | /// \returns the ArchVGPR32 pressure, plus the AVGPRS which we assume will be |
| 94 | /// allocated as VGPR |
| 95 | unsigned getArchVGPRNum() const { return Value[VGPR] + Value[AVGPR]; } |
| 96 | /// \returns the AccVGPR32 pressure |
| 97 | unsigned getAGPRNum() const { return Value[AGPR]; } |
| 98 | /// \returns the AVGPR32 pressure |
| 99 | unsigned getAVGPRNum() const { return Value[AVGPR]; } |
| 100 | |
| 101 | /// \returns the ArchVGPR32 and AccVGPR32 pressure after distributing the |
| 102 | /// bank-agnostic AVGPR pressure across the two banks. Follow the register |
| 103 | /// allocator, first charge AVGPRs to the ArchVGPR bank up to \p ArchLimit, |
| 104 | /// then to the AccVGPR bank up to \p AccLimit. AVGPRs that fit in neither |
| 105 | /// bank are charged to the ArchVGPR bank so that they show up as excess. |
| 106 | std::pair<unsigned, unsigned> getBankedVGPRNum(unsigned ArchLimit, |
| 107 | unsigned AccLimit) const { |
| 108 | unsigned Arch = Value[VGPR]; |
| 109 | unsigned Acc = Value[AGPR]; |
| 110 | unsigned Flex = Value[AVGPR]; |
| 111 | unsigned ToArch = std::min(a: Flex, b: Arch < ArchLimit ? ArchLimit - Arch : 0); |
| 112 | Flex -= ToArch; |
| 113 | unsigned ToAcc = std::min(a: Flex, b: Acc < AccLimit ? AccLimit - Acc : 0); |
| 114 | Flex -= ToAcc; |
| 115 | return {Arch + ToArch + Flex, Acc + ToAcc}; |
| 116 | } |
| 117 | |
| 118 | unsigned getVGPRTuplesWeight() const { |
| 119 | return std::max(a: Value[TOTAL_KINDS + VGPR] + Value[TOTAL_KINDS + AVGPR], |
| 120 | b: Value[TOTAL_KINDS + AGPR]); |
| 121 | } |
| 122 | unsigned getSGPRTuplesWeight() const { return Value[TOTAL_KINDS + SGPR]; } |
| 123 | |
| 124 | unsigned getOccupancy(const GCNSubtarget &ST, |
| 125 | unsigned DynamicVGPRBlockSize) const { |
| 126 | return std::min(a: ST.getOccupancyWithNumSGPRs(SGPRs: getSGPRNum()), |
| 127 | b: ST.getOccupancyWithNumVGPRs(VGPRs: getVGPRNum(UnifiedVGPRFile: ST.hasGFX90AInsts()), |
| 128 | DynamicVGPRBlockSize)); |
| 129 | } |
| 130 | |
| 131 | unsigned getVGPRSpills(MachineFunction &MF, unsigned ArchVGPRThreshold, |
| 132 | unsigned AGPRThreshold, unsigned CombinedThreshold) { |
| 133 | const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); |
| 134 | if (!ST.hasGFX90AInsts()) |
| 135 | return 0; |
| 136 | |
| 137 | unsigned ArchPressure = getArchVGPRNum(); |
| 138 | unsigned AGPRPressure = getAGPRNum(); |
| 139 | |
| 140 | unsigned ArchSpill = ArchPressure > ArchVGPRThreshold |
| 141 | ? (ArchPressure - ArchVGPRThreshold) |
| 142 | : 0; |
| 143 | unsigned AGPRSpill = |
| 144 | AGPRPressure > AGPRThreshold ? (AGPRPressure - AGPRThreshold) : 0; |
| 145 | |
| 146 | unsigned UnifiedPressure = getVGPRNum(/*UnifiedVGPRFile=*/UnifiedVGPRFile: true); |
| 147 | unsigned UnifiedSpill = UnifiedPressure > CombinedThreshold |
| 148 | ? (UnifiedPressure - CombinedThreshold) |
| 149 | : 0; |
| 150 | |
| 151 | return std::max(a: UnifiedSpill, b: ArchSpill + AGPRSpill); |
| 152 | } |
| 153 | |
| 154 | void inc(unsigned Reg, |
| 155 | LaneBitmask PrevMask, |
| 156 | LaneBitmask NewMask, |
| 157 | const MachineRegisterInfo &MRI); |
| 158 | |
| 159 | /// Compares \p this GCNRegpressure to \p O, returning true if \p this is |
| 160 | /// less. Since GCNRegpressure contains different types of pressures, and due |
| 161 | /// to target-specific pecularities (e.g. we care about occupancy rather than |
| 162 | /// raw register usage), we determine if \p this GCNRegPressure is less than |
| 163 | /// \p O based on the following tiered comparisons (in order order of |
| 164 | /// precedence): |
| 165 | /// 1. Better occupancy |
| 166 | /// 2. Less spilling (first preference to VGPR spills, then to SGPR spills) |
| 167 | /// 3. Less tuple register pressure (first preference to VGPR tuples if we |
| 168 | /// determine that SGPR pressure is not important) |
| 169 | /// 4. Less raw register pressure (first preference to VGPR tuples if we |
| 170 | /// determine that SGPR pressure is not important) |
| 171 | bool less(const MachineFunction &MF, const GCNRegPressure &O, |
| 172 | unsigned MaxOccupancy = std::numeric_limits<unsigned>::max()) const; |
| 173 | |
| 174 | bool operator==(const GCNRegPressure &O) const { return Value == O.Value; } |
| 175 | |
| 176 | bool operator!=(const GCNRegPressure &O) const { |
| 177 | return !(*this == O); |
| 178 | } |
| 179 | |
| 180 | GCNRegPressure &operator+=(const GCNRegPressure &RHS) { |
| 181 | for (unsigned I = 0; I < ValueArraySize; ++I) |
| 182 | Value[I] += RHS.Value[I]; |
| 183 | return *this; |
| 184 | } |
| 185 | |
| 186 | GCNRegPressure &operator-=(const GCNRegPressure &RHS) { |
| 187 | for (unsigned I = 0; I < ValueArraySize; ++I) |
| 188 | Value[I] -= RHS.Value[I]; |
| 189 | return *this; |
| 190 | } |
| 191 | |
| 192 | void dump() const; |
| 193 | |
| 194 | static RegKind getRegKind(unsigned Reg, const MachineRegisterInfo &MRI) { |
| 195 | const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo(); |
| 196 | const SIRegisterInfo *STI = static_cast<const SIRegisterInfo *>(TRI); |
| 197 | return (RegKind)getRegKind(RC: MRI.getRegClass(Reg), STI); |
| 198 | } |
| 199 | |
| 200 | private: |
| 201 | static constexpr unsigned ValueArraySize = TOTAL_KINDS * 2; |
| 202 | |
| 203 | /// Pressure for all register kinds (first all regular registers kinds, then |
| 204 | /// all tuple register kinds). |
| 205 | std::array<unsigned, ValueArraySize> Value; |
| 206 | |
| 207 | static unsigned getRegKind(const TargetRegisterClass *RC, |
| 208 | const SIRegisterInfo *STI); |
| 209 | |
| 210 | friend GCNRegPressure maxPerRegFile(const GCNRegPressure &P1, |
| 211 | const GCNRegPressure &P2); |
| 212 | |
| 213 | friend Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST, |
| 214 | unsigned DynamicVGPRBlockSize); |
| 215 | }; |
| 216 | |
| 217 | /// \returns the maximum of \p P1 and \p P2 for each register file. The VGPR and |
| 218 | /// AVGPR components are queried as one sum, so both come from the operand with |
| 219 | /// the larger sum. The AGPR maximum stays separate also for a unified VGPR |
| 220 | /// file: ArchVGPRs and AGPRs use separate ranges of it, so the unified pressure |
| 221 | /// is the sum of both peaks even when they are at different points. |
| 222 | inline GCNRegPressure maxPerRegFile(const GCNRegPressure &P1, |
| 223 | const GCNRegPressure &P2) { |
| 224 | GCNRegPressure Res; |
| 225 | // Value holds the register counts, then the tuple weights, in the same |
| 226 | // RegKind order. |
| 227 | for (unsigned I = 0; I < GCNRegPressure::ValueArraySize; ++I) { |
| 228 | switch (I % GCNRegPressure::TOTAL_KINDS) { |
| 229 | case GCNRegPressure::VGPR: { |
| 230 | unsigned AV = I - GCNRegPressure::VGPR + GCNRegPressure::AVGPR; |
| 231 | const GCNRegPressure &From = |
| 232 | P1.Value[I] + P1.Value[AV] >= P2.Value[I] + P2.Value[AV] ? P1 : P2; |
| 233 | Res.Value[I] = From.Value[I]; |
| 234 | Res.Value[AV] = From.Value[AV]; |
| 235 | break; |
| 236 | } |
| 237 | case GCNRegPressure::AVGPR: |
| 238 | // Already set with VGPR, which comes first in RegKind. |
| 239 | break; |
| 240 | default: |
| 241 | Res.Value[I] = std::max(a: P1.Value[I], b: P2.Value[I]); |
| 242 | } |
| 243 | } |
| 244 | return Res; |
| 245 | } |
| 246 | |
| 247 | inline GCNRegPressure operator+(const GCNRegPressure &P1, |
| 248 | const GCNRegPressure &P2) { |
| 249 | GCNRegPressure Sum = P1; |
| 250 | Sum += P2; |
| 251 | return Sum; |
| 252 | } |
| 253 | |
| 254 | inline GCNRegPressure operator-(const GCNRegPressure &P1, |
| 255 | const GCNRegPressure &P2) { |
| 256 | GCNRegPressure Diff = P1; |
| 257 | Diff -= P2; |
| 258 | return Diff; |
| 259 | } |
| 260 | |
| 261 | //////////////////////////////////////////////////////////////////////////////// |
| 262 | // GCNRPTarget |
| 263 | |
| 264 | /// Models a register pressure target, allowing to evaluate and track register |
| 265 | /// savings against that target from a starting \ref GCNRegPressure. |
| 266 | class GCNRPTarget { |
| 267 | public: |
| 268 | /// Sets up the target such that the register pressure starting at \p RP does |
| 269 | /// not show register spilling on function \p MF (w.r.t. the function's |
| 270 | /// mininum target occupancy). |
| 271 | GCNRPTarget(const MachineFunction &MF, const GCNRegPressure &RP); |
| 272 | |
| 273 | /// Sets up the target such that the register pressure starting at \p RP does |
| 274 | /// not use more than \p NumSGPRs SGPRs and \p NumVGPRs VGPRs on function \p |
| 275 | /// MF. |
| 276 | GCNRPTarget(unsigned NumSGPRs, unsigned NumVGPRs, const MachineFunction &MF, |
| 277 | const GCNRegPressure &RP); |
| 278 | |
| 279 | /// Sets up the target such that the register pressure starting at \p RP does |
| 280 | /// not prevent achieving an occupancy of at least \p Occupancy on function |
| 281 | /// \p MF. |
| 282 | GCNRPTarget(unsigned Occupancy, const MachineFunction &MF, |
| 283 | const GCNRegPressure &RP); |
| 284 | |
| 285 | /// Changes the target (same semantics as constructor). |
| 286 | void setTarget(unsigned NumSGPRs, unsigned NumVGPRs); |
| 287 | |
| 288 | const GCNRegPressure &getCurrentRP() const { return RP; } |
| 289 | |
| 290 | void setRP(const GCNRegPressure &NewRP) { RP = NewRP; } |
| 291 | |
| 292 | /// Determines whether saving virtual register \p Reg will be beneficial |
| 293 | /// towards achieving the RP target. |
| 294 | bool isSaveBeneficial(Register Reg) const; |
| 295 | |
| 296 | /// Returns whether the benefit that saving \p SaveRP represents will be |
| 297 | /// beneficial towards achieving the RP target. |
| 298 | bool isSaveBeneficial(const GCNRegPressure &SaveRP) const; |
| 299 | |
| 300 | /// Returns the benefit towards achieving the RP target that saving \p SaveRP |
| 301 | /// represents, in total number of registers saved across all classes. |
| 302 | unsigned getNumRegsBenefit(const GCNRegPressure &SaveRP) const; |
| 303 | |
| 304 | /// Saves a total pressure of \p SaveRP. |
| 305 | void saveRP(const GCNRegPressure &SaveRP) { |
| 306 | assert(!RP.less(MF, SaveRP) && "saving beyond current RP" ); |
| 307 | RP -= SaveRP; |
| 308 | } |
| 309 | |
| 310 | /// Whether \p TestRP is at or below the defined pressure target. |
| 311 | bool satisfied(const GCNRegPressure &TestRP) const; |
| 312 | /// Whether the current RP is at or below the defined pressure target. |
| 313 | bool satisfied() const { return satisfied(TestRP: RP); } |
| 314 | bool hasVectorRegisterExcess() const; |
| 315 | |
| 316 | unsigned getMaxSGPRs() const { return MaxSGPRs; } |
| 317 | unsigned getMaxVGPRs() const { |
| 318 | return UnifiedRF ? MaxUnifiedVGPRs : MaxVGPRs; |
| 319 | } |
| 320 | |
| 321 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) |
| 322 | friend raw_ostream &operator<<(raw_ostream &OS, const GCNRPTarget &Target) { |
| 323 | OS << "Actual/Target: " << Target.RP.getSGPRNum() << '/' << Target.MaxSGPRs |
| 324 | << " SGPRs, " << Target.RP.getArchVGPRNum() << '/' << Target.MaxVGPRs |
| 325 | << " ArchVGPRs, " << Target.RP.getAGPRNum() << '/' << Target.MaxVGPRs |
| 326 | << " AGPRs" ; |
| 327 | |
| 328 | if (Target.MaxUnifiedVGPRs) { |
| 329 | OS << ", " << Target.RP.getVGPRNum(true) << '/' << Target.MaxUnifiedVGPRs |
| 330 | << " VGPRs (unified)" ; |
| 331 | } |
| 332 | return OS; |
| 333 | } |
| 334 | #endif |
| 335 | |
| 336 | private: |
| 337 | const MachineFunction &MF; |
| 338 | const bool UnifiedRF; |
| 339 | |
| 340 | /// Current register pressure. |
| 341 | GCNRegPressure RP; |
| 342 | |
| 343 | /// Target number of SGPRs. |
| 344 | unsigned MaxSGPRs = 0; |
| 345 | /// Target number of ArchVGPRs and AGPRs. |
| 346 | unsigned MaxVGPRs = 0; |
| 347 | /// Target number of overall VGPRs for subtargets with unified RFs. Always 0 |
| 348 | /// for subtargets with non-unified RFs. |
| 349 | unsigned MaxUnifiedVGPRs = 0; |
| 350 | |
| 351 | GCNRPTarget(const GCNRegPressure &RP, const MachineFunction &MF) |
| 352 | : MF(MF), UnifiedRF(MF.getSubtarget<GCNSubtarget>().hasGFX90AInsts()), |
| 353 | RP(RP) {} |
| 354 | }; |
| 355 | |
| 356 | /////////////////////////////////////////////////////////////////////////////// |
| 357 | // GCNRPTracker |
| 358 | |
| 359 | class GCNRPTracker { |
| 360 | public: |
| 361 | using LiveRegSet = DenseMap<unsigned, LaneBitmask>; |
| 362 | |
| 363 | protected: |
| 364 | LiveIntervals &LIS; |
| 365 | LiveRegSet LiveRegs; |
| 366 | GCNRegPressure CurPressure, MaxPressure; |
| 367 | MachineBasicBlock::const_iterator LastTrackedMI; |
| 368 | mutable const MachineRegisterInfo *MRI = nullptr; |
| 369 | std::function<void(const GCNRegPressure &)> PressureCallback; |
| 370 | |
| 371 | GCNRPTracker(LiveIntervals &LIS_) : LIS(LIS_) {} |
| 372 | |
| 373 | /// Folds the pressure \p RP of one program point into MaxPressure and passes |
| 374 | /// it to the pressure callback. |
| 375 | void updateMaxPressure(const GCNRegPressure &RP) { |
| 376 | MaxPressure = maxPerRegFile(P1: MaxPressure, P2: RP); |
| 377 | if (PressureCallback) |
| 378 | PressureCallback(RP); |
| 379 | } |
| 380 | |
| 381 | /// Resets tracker before or \p After the provided \p MI, which can be a debug |
| 382 | /// instruction. |
| 383 | void reset(const MachineInstr &MI, bool After); |
| 384 | |
| 385 | /// Resets tracker at the start or \p End of the \p MBB. |
| 386 | void reset(const MachineBasicBlock &MBB, bool End); |
| 387 | |
| 388 | /// Resets tracker at the specified slot index \p SI. |
| 389 | void reset(const MachineRegisterInfo &MRI, SlotIndex SI); |
| 390 | |
| 391 | LaneBitmask getLastUsedLanes(Register Reg, SlotIndex Pos) const; |
| 392 | |
| 393 | public: |
| 394 | /// Resets tracker with the provided \p LiveRegs. |
| 395 | void reset(const MachineRegisterInfo &MRI, const LiveRegSet &LiveRegs); |
| 396 | |
| 397 | // live regs for the current state |
| 398 | const decltype(LiveRegs) &getLiveRegs() const { return LiveRegs; } |
| 399 | const MachineInstr *getLastTrackedMI() const { |
| 400 | return LastTrackedMI.isValid() ? &*LastTrackedMI : nullptr; |
| 401 | } |
| 402 | |
| 403 | void clearMaxPressure() { MaxPressure.clear(); } |
| 404 | |
| 405 | const GCNRegPressure &getMaxPressure() const { return MaxPressure; } |
| 406 | |
| 407 | void resetMaxPressure() { MaxPressure = CurPressure; } |
| 408 | |
| 409 | /// Sets a function that the tracker calls with the pressure of every program |
| 410 | /// point that it folds into the maximum pressure. A user can compute there a |
| 411 | /// maximum that maxPerRegFile() does not keep, e.g. of one register kind. |
| 412 | void setPressureCallback(std::function<void(const GCNRegPressure &)> CB) { |
| 413 | PressureCallback = std::move(CB); |
| 414 | } |
| 415 | |
| 416 | GCNRegPressure getPressure() const { return CurPressure; } |
| 417 | |
| 418 | decltype(LiveRegs) moveLiveRegs() { |
| 419 | return std::move(LiveRegs); |
| 420 | } |
| 421 | }; |
| 422 | |
| 423 | GCNRPTracker::LiveRegSet |
| 424 | getLiveRegs(SlotIndex SI, const LiveIntervals &LIS, |
| 425 | const MachineRegisterInfo &MRI, |
| 426 | GCNRegPressure::RegKind RegKind = GCNRegPressure::TOTAL_KINDS); |
| 427 | |
| 428 | //////////////////////////////////////////////////////////////////////////////// |
| 429 | // GCNUpwardRPTracker |
| 430 | |
| 431 | class GCNUpwardRPTracker : public GCNRPTracker { |
| 432 | public: |
| 433 | GCNUpwardRPTracker(LiveIntervals &LIS_) : GCNRPTracker(LIS_) {} |
| 434 | |
| 435 | using GCNRPTracker::reset; |
| 436 | |
| 437 | /// Resets tracker to the point just after \p MI (in program order), which can |
| 438 | /// be a debug instruction. |
| 439 | void reset(const MachineInstr &MI) { reset(MI, /*After=*/After: true); } |
| 440 | |
| 441 | /// Move to the state of RP just before the \p MI. |
| 442 | void recede(const MachineInstr &MI); |
| 443 | |
| 444 | /// \p returns whether the tracker's state after receding MI corresponds |
| 445 | /// to reported by LIS. |
| 446 | bool isValid() const; |
| 447 | |
| 448 | GCNRegPressure getMaxPressureAndReset() { |
| 449 | GCNRegPressure RP = MaxPressure; |
| 450 | resetMaxPressure(); |
| 451 | return RP; |
| 452 | } |
| 453 | }; |
| 454 | |
| 455 | //////////////////////////////////////////////////////////////////////////////// |
| 456 | // GCNDownwardRPTracker |
| 457 | |
| 458 | /// Common behavior for downward RP trackers. Unconstructible directly. |
| 459 | class GCNDownwardRPTrackerBase : public GCNRPTracker { |
| 460 | public: |
| 461 | /// Mostly copy/paste from CodeGen/RegisterPressure.cpp |
| 462 | /// Calculate the impact \p MI will have on CurPressure and \return the |
| 463 | /// speculated pressure. In order to support RP Speculation, this does not |
| 464 | /// rely on the implicit program ordering in the LiveIntervals. |
| 465 | GCNRegPressure bumpDownwardPressure(const MachineInstr *MI, |
| 466 | const SIRegisterInfo *TRI) const; |
| 467 | |
| 468 | protected: |
| 469 | GCNDownwardRPTrackerBase(LiveIntervals &LIS) : GCNRPTracker(LIS) {} |
| 470 | |
| 471 | /// Retires the lanes of \p MI's used registers that die at \p MI, mirroring |
| 472 | /// RegPressureTracker::advance(). Early-clobber defs of \p MI are accounted |
| 473 | /// in \ref MaxPressure as live together with all of its uses. The uses of a |
| 474 | /// PHI are not retired since they do not die there. |
| 475 | void retireDeadUses(const MachineInstr &MI); |
| 476 | |
| 477 | /// Retires the lanes defined by \p MI that are dead at \p SI, according to |
| 478 | /// LIS. |
| 479 | void retireDeadDefs(const MachineInstr &MI, SlotIndex SI); |
| 480 | |
| 481 | /// Adds all lanes (dead or not) defined by \p MI's register operands to \ref |
| 482 | /// CurPressure. |
| 483 | void addDefs(const MachineInstr &MI); |
| 484 | |
| 485 | private: |
| 486 | /// Drop the lanes of \p Reg that are no longer live at \p SI, decreasing |
| 487 | /// CurPressure accordingly. \p Reg must be a virtual register that is |
| 488 | /// currently tracked as live. |
| 489 | void retireVirtReg(Register Reg, SlotIndex SI); |
| 490 | }; |
| 491 | |
| 492 | /// Downward RP tracker that relies on program order to advance down the |
| 493 | /// instruction stream. |
| 494 | class GCNDownwardRPTracker : public GCNDownwardRPTrackerBase { |
| 495 | private: |
| 496 | /// Next MI to advance over. |
| 497 | MachineBasicBlock::const_iterator NextMI; |
| 498 | /// End iterator for basic block being advanced in. |
| 499 | MachineBasicBlock::const_iterator MBBEnd; |
| 500 | |
| 501 | public: |
| 502 | GCNDownwardRPTracker(LiveIntervals &LIS) : GCNDownwardRPTrackerBase(LIS) {} |
| 503 | |
| 504 | using GCNDownwardRPTrackerBase::addDefs; |
| 505 | using GCNRPTracker::reset; |
| 506 | |
| 507 | /// Returns the next MI that will be advanced over. |
| 508 | MachineBasicBlock::const_iterator getNext() const { return NextMI; } |
| 509 | |
| 510 | /// \p return MaxPressure and clear it. |
| 511 | GCNRegPressure moveMaxPressure() { |
| 512 | auto Res = MaxPressure; |
| 513 | MaxPressure.clear(); |
| 514 | return Res; |
| 515 | } |
| 516 | |
| 517 | /// Reset tracker to the point before the \p MI (filling \ref LiveRegs upon |
| 518 | /// this point using \p LiveRegsCopy if provided or LIS otherwise) point using |
| 519 | /// LIS. \p End must be between the MI and the end of its parent block |
| 520 | /// (inclusive). \p returns false if the range [MI, End) is empty except debug |
| 521 | /// values. |
| 522 | bool reset(const MachineInstr &MI, MachineBasicBlock::const_iterator End, |
| 523 | const LiveRegSet *LiveRegsCopy = nullptr); |
| 524 | |
| 525 | /// Move to the state right before \ref NextMI or the end of the MBB. This |
| 526 | /// retires defs of \ref LastTrackedMI that are dead after the instruction. |
| 527 | /// Its dead uses were already retired by \ref advanceToNext. \p returns |
| 528 | /// false if it reached the end of the block. |
| 529 | bool advanceBeforeNext(); |
| 530 | |
| 531 | /// Move to the state at \ref NextMI. This retires the uses that die at \p |
| 532 | /// NextMI, then adds the register lanes it defines to \ref CurPressure. Has |
| 533 | /// to be called after \ref advanceBeforeNext. |
| 534 | void advanceToNext(); |
| 535 | |
| 536 | /// Advance to the state at the next MI in program order. \p returns false if |
| 537 | /// it reached end of block. |
| 538 | bool advance(); |
| 539 | |
| 540 | /// Advance in program order until before \p End. Returns whether it actually |
| 541 | /// had to advance to reach \p End. |
| 542 | bool advance(MachineBasicBlock::const_iterator End); |
| 543 | |
| 544 | /// Reset tracker to \p Begin (filling \ref LiveRegs upon this point using \p |
| 545 | /// LiveRegsCopy if provided or LIS otherwise) and advance to \p End, which |
| 546 | /// must be between \p Begin and the end of its parent block (inclusive). \p |
| 547 | /// returns false if the range [Begin, End) is empty except debug values. |
| 548 | bool advance(MachineBasicBlock::const_iterator Begin, |
| 549 | MachineBasicBlock::const_iterator End, |
| 550 | const LiveRegSet *LiveRegsCopy = nullptr); |
| 551 | }; |
| 552 | |
| 553 | /// Downward RP tracker that relies on the user to provide the next instruction |
| 554 | /// down the instruction stream. It still uses LIS for liveness queries. |
| 555 | class GCNManualDownwardRPTracker : public GCNDownwardRPTrackerBase { |
| 556 | public: |
| 557 | GCNManualDownwardRPTracker(LiveIntervals &LIS) |
| 558 | : GCNDownwardRPTrackerBase(LIS) {} |
| 559 | |
| 560 | using GCNRPTracker::reset; |
| 561 | |
| 562 | /// Advances to the state after \p MI. |
| 563 | void advance(const MachineInstr &MI); |
| 564 | }; |
| 565 | |
| 566 | /// \returns the LaneMask of live lanes of \p Reg at position \p SI. Only the |
| 567 | /// active lanes of \p LaneMaskFilter will be set in the return value. This is |
| 568 | /// used, for example, to limit the live lanes to a specific subreg when |
| 569 | /// calculating use masks. |
| 570 | LaneBitmask getLiveLaneMask(unsigned Reg, SlotIndex SI, |
| 571 | const LiveIntervals &LIS, |
| 572 | const MachineRegisterInfo &MRI, |
| 573 | LaneBitmask LaneMaskFilter = LaneBitmask::getAll()); |
| 574 | |
| 575 | LaneBitmask getLiveLaneMask(const LiveInterval &LI, SlotIndex SI, |
| 576 | const MachineRegisterInfo &MRI, |
| 577 | LaneBitmask LaneMaskFilter = LaneBitmask::getAll()); |
| 578 | |
| 579 | /// creates a map MachineInstr -> LiveRegSet |
| 580 | /// R - range of iterators on instructions |
| 581 | /// After - upon entry or exit of every instruction |
| 582 | /// Note: there is no entry in the map for instructions with empty live reg set |
| 583 | /// Complexity = O(NumVirtRegs * averageLiveRangeSegmentsPerReg * lg(R)) |
| 584 | template <typename Range> |
| 585 | DenseMap<MachineInstr*, GCNRPTracker::LiveRegSet> |
| 586 | getLiveRegMap(Range &&R, bool After, LiveIntervals &LIS) { |
| 587 | std::vector<SlotIndex> Indexes; |
| 588 | Indexes.reserve(n: llvm::size(R)); |
| 589 | auto &SII = *LIS.getSlotIndexes(); |
| 590 | for (MachineInstr *I : R) { |
| 591 | auto SI = SII.getInstructionIndex(MI: *I); |
| 592 | Indexes.push_back(x: After ? SI.getDeadSlot() : SI.getBaseIndex()); |
| 593 | } |
| 594 | llvm::sort(C&: Indexes); |
| 595 | |
| 596 | auto &MRI = (*R.begin())->getMF()->getRegInfo(); |
| 597 | DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> LiveRegMap; |
| 598 | SmallVector<SlotIndex, 32> LiveIdxs, SRLiveIdxs; |
| 599 | for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) { |
| 600 | auto Reg = Register::index2VirtReg(Index: I); |
| 601 | if (!LIS.hasInterval(Reg)) |
| 602 | continue; |
| 603 | auto &LI = LIS.getInterval(Reg); |
| 604 | LiveIdxs.clear(); |
| 605 | if (!LI.findIndexesLiveAt(R&: Indexes, O: std::back_inserter(x&: LiveIdxs))) |
| 606 | continue; |
| 607 | if (!LI.hasSubRanges()) { |
| 608 | for (auto SI : LiveIdxs) |
| 609 | LiveRegMap[SII.getInstructionFromIndex(index: SI)][Reg] = |
| 610 | MRI.getMaxLaneMaskForVReg(Reg); |
| 611 | } else |
| 612 | for (const auto &S : LI.subranges()) { |
| 613 | // constrain search for subranges by indexes live at main range |
| 614 | SRLiveIdxs.clear(); |
| 615 | S.findIndexesLiveAt(R&: LiveIdxs, O: std::back_inserter(x&: SRLiveIdxs)); |
| 616 | for (auto SI : SRLiveIdxs) |
| 617 | LiveRegMap[SII.getInstructionFromIndex(index: SI)][Reg] |= S.LaneMask; |
| 618 | } |
| 619 | } |
| 620 | return LiveRegMap; |
| 621 | } |
| 622 | |
| 623 | inline GCNRPTracker::LiveRegSet getLiveRegsAfter(const MachineInstr &MI, |
| 624 | const LiveIntervals &LIS) { |
| 625 | return getLiveRegs(SI: LIS.getInstructionIndex(Instr: MI).getDeadSlot(), LIS, |
| 626 | MRI: MI.getMF()->getRegInfo()); |
| 627 | } |
| 628 | |
| 629 | inline GCNRPTracker::LiveRegSet getLiveRegsBefore(const MachineInstr &MI, |
| 630 | const LiveIntervals &LIS) { |
| 631 | return getLiveRegs(SI: LIS.getInstructionIndex(Instr: MI).getBaseIndex(), LIS, |
| 632 | MRI: MI.getMF()->getRegInfo()); |
| 633 | } |
| 634 | |
| 635 | template <typename Range> |
| 636 | GCNRegPressure getRegPressure(const MachineRegisterInfo &MRI, |
| 637 | Range &&LiveRegs) { |
| 638 | GCNRegPressure Res; |
| 639 | for (const auto &RM : LiveRegs) |
| 640 | Res.inc(Reg: RM.first, PrevMask: LaneBitmask::getNone(), NewMask: RM.second, MRI); |
| 641 | return Res; |
| 642 | } |
| 643 | |
| 644 | bool isEqual(const GCNRPTracker::LiveRegSet &S1, |
| 645 | const GCNRPTracker::LiveRegSet &S2); |
| 646 | |
| 647 | Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST = nullptr, |
| 648 | unsigned DynamicVGPRBlockSize = 0); |
| 649 | |
| 650 | Printable print(const GCNRPTracker::LiveRegSet &LiveRegs, |
| 651 | const MachineRegisterInfo &MRI); |
| 652 | |
| 653 | Printable reportMismatch(const GCNRPTracker::LiveRegSet &LISLR, |
| 654 | const GCNRPTracker::LiveRegSet &TrackedL, |
| 655 | const TargetRegisterInfo *TRI, StringRef Pfx = " " ); |
| 656 | |
| 657 | struct GCNRegPressurePrinter : public MachineFunctionPass { |
| 658 | static char ID; |
| 659 | |
| 660 | public: |
| 661 | GCNRegPressurePrinter() : MachineFunctionPass(ID) {} |
| 662 | |
| 663 | bool runOnMachineFunction(MachineFunction &MF) override; |
| 664 | |
| 665 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
| 666 | AU.addRequired<LiveIntervalsWrapperPass>(); |
| 667 | AU.setPreservesAll(); |
| 668 | MachineFunctionPass::getAnalysisUsage(AU); |
| 669 | } |
| 670 | }; |
| 671 | |
| 672 | LLVM_ABI void dumpMaxRegPressure(MachineFunction &MF, |
| 673 | GCNRegPressure::RegKind Kind, |
| 674 | LiveIntervals &LIS, |
| 675 | const MachineLoopInfo *MLI); |
| 676 | |
| 677 | /// Estimate VGPR pressure using greedy, non-splitting register allocation |
| 678 | /// simulation, accounting for live interval interference. |
| 679 | /// \param RegionBegin Start iterator of the region |
| 680 | /// \param RegionEnd End iterator of the region |
| 681 | /// \param LiveIns Live-in registers for the region |
| 682 | /// \returns estimated VGPR pressure |
| 683 | unsigned estimateGreedyVGPRPressure( |
| 684 | MachineBasicBlock::const_iterator RegionBegin, |
| 685 | MachineBasicBlock::const_iterator RegionEnd, |
| 686 | const GCNRPTracker::LiveRegSet &LiveIns, const LiveIntervals &LIS, |
| 687 | const MachineRegisterInfo &MRI, const SIRegisterInfo &TRI); |
| 688 | |
| 689 | } // end namespace llvm |
| 690 | |
| 691 | #endif // LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H |
| 692 | |