1//===- AMDGPUWaitcntUtils.cpp ---------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AMDGPUWaitcntUtils.h"
10#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
11#include "Utils/AMDGPUBaseInfo.h"
12
13namespace llvm::AMDGPU {
14
15iota_range<InstCounterType> inst_counter_types(InstCounterType MaxCounter) {
16 return enum_seq(Begin: LOAD_CNT, End: MaxCounter);
17}
18
19StringLiteral getInstCounterName(InstCounterType T) {
20 switch (T) {
21 case LOAD_CNT:
22 return "LOAD_CNT";
23 case DS_CNT:
24 return "DS_CNT";
25 case EXP_CNT:
26 return "EXP_CNT";
27 case STORE_CNT:
28 return "STORE_CNT";
29 case SAMPLE_CNT:
30 return "SAMPLE_CNT";
31 case BVH_CNT:
32 return "BVH_CNT";
33 case KM_CNT:
34 return "KM_CNT";
35 case X_CNT:
36 return "X_CNT";
37 case ASYNC_CNT:
38 return "ASYNC_CNT";
39 case TENSOR_CNT:
40 return "TENSOR_CNT";
41 case VA_VDST_RD:
42 return "VA_VDST_RD";
43 case VA_VDST_WR:
44 return "VA_VDST_WR";
45 case VM_VSRC:
46 return "VM_VSRC";
47 case NUM_INST_CNTS:
48 return "NUM_INST_CNTS";
49 }
50 llvm_unreachable("Unhandled InstCounterType");
51}
52
53HardwareLimits::HardwareLimits(const IsaVersion &IV) {
54 bool HasExtendedWaitCounts = IV.Major >= 12;
55 if (HasExtendedWaitCounts) {
56 LoadcntMax = getLoadcntBitMask(Version: IV);
57 DscntMax = getDscntBitMask(Version: IV);
58 } else {
59 LoadcntMax = getVmcntBitMask(Version: IV);
60 DscntMax = getLgkmcntBitMask(Version: IV);
61 }
62 ExpcntMax = getExpcntBitMask(Version: IV);
63 StorecntMax = getStorecntBitMask(Version: IV);
64 SamplecntMax = getSamplecntBitMask(Version: IV);
65 BvhcntMax = getBvhcntBitMask(Version: IV);
66 KmcntMax = getKmcntBitMask(Version: IV);
67 XcntMax = getXcntBitMask(Version: IV);
68 AsyncMax = getAsynccntBitMask(Version: IV);
69 VaVdstMax = DepCtr::getVaVdstBitMask();
70 VmVsrcMax = DepCtr::getVmVsrcBitMask();
71}
72
73unsigned HardwareLimits::get(InstCounterType T) const {
74 switch (T) {
75 case AMDGPU::LOAD_CNT:
76 return LoadcntMax;
77 case AMDGPU::DS_CNT:
78 return DscntMax;
79 case AMDGPU::EXP_CNT:
80 return ExpcntMax;
81 case AMDGPU::STORE_CNT:
82 return StorecntMax;
83 case AMDGPU::SAMPLE_CNT:
84 return SamplecntMax;
85 case AMDGPU::BVH_CNT:
86 return BvhcntMax;
87 case AMDGPU::KM_CNT:
88 return KmcntMax;
89 case AMDGPU::X_CNT:
90 return XcntMax;
91 case AMDGPU::VA_VDST_RD:
92 case AMDGPU::VA_VDST_WR:
93 return VaVdstMax;
94 case AMDGPU::VM_VSRC:
95 return VmVsrcMax;
96 default:
97 return 0;
98 }
99}
100
101#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
102void Waitcnt::dump() const { dbgs() << *this << '\n'; }
103#endif
104
105Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded) {
106 Waitcnt Decoded;
107 Decoded.set(T: LOAD_CNT, Val: decodeVmcnt(Version, Waitcnt: Encoded));
108 Decoded.set(T: EXP_CNT, Val: decodeExpcnt(Version, Waitcnt: Encoded));
109 Decoded.set(T: DS_CNT, Val: decodeLgkmcnt(Version, Waitcnt: Encoded));
110 return Decoded;
111}
112
113unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded) {
114 return encodeWaitcnt(Version, Vmcnt: Decoded.get(T: LOAD_CNT), Expcnt: Decoded.get(T: EXP_CNT),
115 Lgkmcnt: Decoded.get(T: DS_CNT));
116}
117
118Waitcnt decodeLoadcntDscnt(const IsaVersion &Version, unsigned LoadcntDscnt) {
119 Waitcnt Decoded;
120 Decoded.set(T: LOAD_CNT, Val: decodeLoadcnt(Version, Waitcnt: LoadcntDscnt));
121 Decoded.set(T: DS_CNT, Val: decodeDscnt(Version, Waitcnt: LoadcntDscnt));
122 return Decoded;
123}
124
125Waitcnt decodeStorecntDscnt(const IsaVersion &Version, unsigned StorecntDscnt) {
126 Waitcnt Decoded;
127 Decoded.set(T: STORE_CNT, Val: decodeStorecnt(Version, Waitcnt: StorecntDscnt));
128 Decoded.set(T: DS_CNT, Val: decodeDscnt(Version, Waitcnt: StorecntDscnt));
129 return Decoded;
130}
131
132unsigned encodeLoadcntDscnt(const IsaVersion &Version, const Waitcnt &Decoded) {
133 return encodeLoadcntDscnt(Version, Loadcnt: Decoded.get(T: LOAD_CNT),
134 Dscnt: Decoded.get(T: DS_CNT));
135}
136
137unsigned encodeStorecntDscnt(const IsaVersion &Version,
138 const Waitcnt &Decoded) {
139 return encodeStorecntDscnt(Version, Storecnt: Decoded.get(T: STORE_CNT),
140 Dscnt: Decoded.get(T: DS_CNT));
141}
142
143std::optional<AMDGPU::InstCounterType> counterTypeForInstr(unsigned Opcode) {
144 switch (Opcode) {
145 case AMDGPU::S_WAIT_LOADCNT:
146 return AMDGPU::LOAD_CNT;
147 case AMDGPU::S_WAIT_EXPCNT:
148 return AMDGPU::EXP_CNT;
149 case AMDGPU::S_WAIT_STORECNT:
150 return AMDGPU::STORE_CNT;
151 case AMDGPU::S_WAIT_SAMPLECNT:
152 return AMDGPU::SAMPLE_CNT;
153 case AMDGPU::S_WAIT_BVHCNT:
154 return AMDGPU::BVH_CNT;
155 case AMDGPU::S_WAIT_DSCNT:
156 return AMDGPU::DS_CNT;
157 case AMDGPU::S_WAIT_KMCNT:
158 return AMDGPU::KM_CNT;
159 case AMDGPU::S_WAIT_XCNT:
160 return AMDGPU::X_CNT;
161 case AMDGPU::S_WAIT_ASYNCCNT:
162 return AMDGPU::ASYNC_CNT;
163 case AMDGPU::S_WAIT_TENSORCNT:
164 return AMDGPU::TENSOR_CNT;
165 default:
166 return {};
167 }
168}
169
170} // namespace llvm::AMDGPU
171