| 1 | //===- AMDGPUWaitcntUtils.cpp ---------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "AMDGPUWaitcntUtils.h" |
| 10 | #include "GCNSubtarget.h" |
| 11 | #include "MCTargetDesc/AMDGPUMCTargetDesc.h" |
| 12 | #include "Utils/AMDGPUBaseInfo.h" |
| 13 | #include "llvm/IR/Function.h" |
| 14 | |
| 15 | namespace llvm::AMDGPU { |
| 16 | |
| 17 | bool isExpertSchedulingMode(const GCNSubtarget &ST, const Function &F) { |
| 18 | return ST.hasExpertSchedulingMode() && |
| 19 | valueOr(X: ST.getCLOpts().expert_scheduling_mode, |
| 20 | Default: F.getFnAttribute(Kind: "amdgpu-expert-scheduling-mode" ) |
| 21 | .getValueAsBool()); |
| 22 | } |
| 23 | |
| 24 | iota_range<InstCounterType> inst_counter_types(InstCounterType MaxCounter) { |
| 25 | return enum_seq(Begin: LOAD_CNT, End: MaxCounter); |
| 26 | } |
| 27 | |
| 28 | StringLiteral getInstCounterName(InstCounterType T) { |
| 29 | switch (T) { |
| 30 | case LOAD_CNT: |
| 31 | return "LOAD_CNT" ; |
| 32 | case DS_CNT: |
| 33 | return "DS_CNT" ; |
| 34 | case EXP_CNT: |
| 35 | return "EXP_CNT" ; |
| 36 | case STORE_CNT: |
| 37 | return "STORE_CNT" ; |
| 38 | case SAMPLE_CNT: |
| 39 | return "SAMPLE_CNT" ; |
| 40 | case BVH_CNT: |
| 41 | return "BVH_CNT" ; |
| 42 | case KM_CNT: |
| 43 | return "KM_CNT" ; |
| 44 | case X_CNT: |
| 45 | return "X_CNT" ; |
| 46 | case ASYNC_CNT: |
| 47 | return "ASYNC_CNT" ; |
| 48 | case TENSOR_CNT: |
| 49 | return "TENSOR_CNT" ; |
| 50 | case VA_VDST_RD: |
| 51 | return "VA_VDST_RD" ; |
| 52 | case VA_VDST_WR: |
| 53 | return "VA_VDST_WR" ; |
| 54 | case VM_VSRC: |
| 55 | return "VM_VSRC" ; |
| 56 | case NUM_INST_CNTS: |
| 57 | return "NUM_INST_CNTS" ; |
| 58 | } |
| 59 | llvm_unreachable("Unhandled InstCounterType" ); |
| 60 | } |
| 61 | |
| 62 | HardwareLimits::HardwareLimits(const IsaVersion &IV) { |
| 63 | bool HasExtendedWaitCounts = IV.Major >= 12; |
| 64 | if (HasExtendedWaitCounts) { |
| 65 | LoadcntMax = getLoadcntBitMask(Version: IV); |
| 66 | DscntMax = getDscntBitMask(Version: IV); |
| 67 | } else { |
| 68 | LoadcntMax = getVmcntBitMask(Version: IV); |
| 69 | DscntMax = getLgkmcntBitMask(Version: IV); |
| 70 | } |
| 71 | ExpcntMax = getExpcntBitMask(Version: IV); |
| 72 | StorecntMax = getStorecntBitMask(Version: IV); |
| 73 | SamplecntMax = getSamplecntBitMask(Version: IV); |
| 74 | BvhcntMax = getBvhcntBitMask(Version: IV); |
| 75 | KmcntMax = getKmcntBitMask(Version: IV); |
| 76 | XcntMax = getXcntBitMask(Version: IV); |
| 77 | AsyncMax = getAsynccntBitMask(Version: IV); |
| 78 | VaVdstMax = DepCtr::getVaVdstBitMask(); |
| 79 | VmVsrcMax = DepCtr::getVmVsrcBitMask(); |
| 80 | } |
| 81 | |
| 82 | unsigned HardwareLimits::get(InstCounterType T) const { |
| 83 | switch (T) { |
| 84 | case AMDGPU::LOAD_CNT: |
| 85 | return LoadcntMax; |
| 86 | case AMDGPU::DS_CNT: |
| 87 | return DscntMax; |
| 88 | case AMDGPU::EXP_CNT: |
| 89 | return ExpcntMax; |
| 90 | case AMDGPU::STORE_CNT: |
| 91 | return StorecntMax; |
| 92 | case AMDGPU::SAMPLE_CNT: |
| 93 | return SamplecntMax; |
| 94 | case AMDGPU::BVH_CNT: |
| 95 | return BvhcntMax; |
| 96 | case AMDGPU::KM_CNT: |
| 97 | return KmcntMax; |
| 98 | case AMDGPU::X_CNT: |
| 99 | return XcntMax; |
| 100 | case AMDGPU::VA_VDST_RD: |
| 101 | case AMDGPU::VA_VDST_WR: |
| 102 | return VaVdstMax; |
| 103 | case AMDGPU::VM_VSRC: |
| 104 | return VmVsrcMax; |
| 105 | default: |
| 106 | return 0; |
| 107 | } |
| 108 | } |
| 109 | |
| 110 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) |
| 111 | void Waitcnt::dump() const { dbgs() << *this << '\n'; } |
| 112 | #endif |
| 113 | |
| 114 | Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded) { |
| 115 | Waitcnt Decoded; |
| 116 | Decoded.set(T: LOAD_CNT, Val: decodeVmcnt(Version, Waitcnt: Encoded)); |
| 117 | Decoded.set(T: EXP_CNT, Val: decodeExpcnt(Version, Waitcnt: Encoded)); |
| 118 | Decoded.set(T: DS_CNT, Val: decodeLgkmcnt(Version, Waitcnt: Encoded)); |
| 119 | return Decoded; |
| 120 | } |
| 121 | |
| 122 | unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded) { |
| 123 | return encodeWaitcnt(Version, Vmcnt: Decoded.get(T: LOAD_CNT), Expcnt: Decoded.get(T: EXP_CNT), |
| 124 | Lgkmcnt: Decoded.get(T: DS_CNT)); |
| 125 | } |
| 126 | |
| 127 | Waitcnt decodeLoadcntDscnt(const IsaVersion &Version, unsigned LoadcntDscnt) { |
| 128 | Waitcnt Decoded; |
| 129 | Decoded.set(T: LOAD_CNT, Val: decodeLoadcnt(Version, Waitcnt: LoadcntDscnt)); |
| 130 | Decoded.set(T: DS_CNT, Val: decodeDscnt(Version, Waitcnt: LoadcntDscnt)); |
| 131 | return Decoded; |
| 132 | } |
| 133 | |
| 134 | Waitcnt decodeStorecntDscnt(const IsaVersion &Version, unsigned StorecntDscnt) { |
| 135 | Waitcnt Decoded; |
| 136 | Decoded.set(T: STORE_CNT, Val: decodeStorecnt(Version, Waitcnt: StorecntDscnt)); |
| 137 | Decoded.set(T: DS_CNT, Val: decodeDscnt(Version, Waitcnt: StorecntDscnt)); |
| 138 | return Decoded; |
| 139 | } |
| 140 | |
| 141 | unsigned encodeLoadcntDscnt(const IsaVersion &Version, const Waitcnt &Decoded) { |
| 142 | return encodeLoadcntDscnt(Version, Loadcnt: Decoded.get(T: LOAD_CNT), |
| 143 | Dscnt: Decoded.get(T: DS_CNT)); |
| 144 | } |
| 145 | |
| 146 | unsigned encodeStorecntDscnt(const IsaVersion &Version, |
| 147 | const Waitcnt &Decoded) { |
| 148 | return encodeStorecntDscnt(Version, Storecnt: Decoded.get(T: STORE_CNT), |
| 149 | Dscnt: Decoded.get(T: DS_CNT)); |
| 150 | } |
| 151 | |
| 152 | std::optional<AMDGPU::InstCounterType> counterTypeForInstr(unsigned Opcode) { |
| 153 | switch (Opcode) { |
| 154 | case AMDGPU::S_WAIT_LOADCNT: |
| 155 | return AMDGPU::LOAD_CNT; |
| 156 | case AMDGPU::S_WAIT_EXPCNT: |
| 157 | return AMDGPU::EXP_CNT; |
| 158 | case AMDGPU::S_WAIT_STORECNT: |
| 159 | return AMDGPU::STORE_CNT; |
| 160 | case AMDGPU::S_WAIT_SAMPLECNT: |
| 161 | return AMDGPU::SAMPLE_CNT; |
| 162 | case AMDGPU::S_WAIT_BVHCNT: |
| 163 | return AMDGPU::BVH_CNT; |
| 164 | case AMDGPU::S_WAIT_DSCNT: |
| 165 | return AMDGPU::DS_CNT; |
| 166 | case AMDGPU::S_WAIT_KMCNT: |
| 167 | return AMDGPU::KM_CNT; |
| 168 | case AMDGPU::S_WAIT_XCNT: |
| 169 | return AMDGPU::X_CNT; |
| 170 | case AMDGPU::S_WAIT_ASYNCCNT: |
| 171 | return AMDGPU::ASYNC_CNT; |
| 172 | case AMDGPU::S_WAIT_TENSORCNT: |
| 173 | return AMDGPU::TENSOR_CNT; |
| 174 | default: |
| 175 | return {}; |
| 176 | } |
| 177 | } |
| 178 | |
| 179 | } // namespace llvm::AMDGPU |
| 180 | |