1//===- AMDGPUWaitcntUtils.h -------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUWAITCNTUTILS_H
10#define LLVM_LIB_TARGET_AMDGPU_AMDGPUWAITCNTUTILS_H
11
12#include "llvm/ADT/Sequence.h"
13#include "llvm/ADT/StringExtras.h"
14#include "llvm/Support/Debug.h"
15#include "llvm/Support/raw_ostream.h"
16#include "llvm/TargetParser/AMDGPUTargetParser.h"
17
18namespace llvm {
19
20class Function;
21class GCNSubtarget;
22
23namespace AMDGPU {
24
25/// Return whether expert scheduling mode is available and enabled for \p F.
26bool isExpertSchedulingMode(const GCNSubtarget &ST, const Function &F);
27
28enum InstCounterType {
29 LOAD_CNT = 0, // VMcnt prior to gfx12.
30 DS_CNT, // LKGMcnt prior to gfx12.
31 EXP_CNT, //
32 STORE_CNT, // VScnt in gfx10/gfx11.
33 NUM_NORMAL_INST_CNTS,
34 SAMPLE_CNT = NUM_NORMAL_INST_CNTS, // gfx12+ only.
35 BVH_CNT, // gfx12+ only.
36 KM_CNT, // gfx12+ only.
37 X_CNT, // gfx1250.
38 ASYNC_CNT, // gfx1250.
39 TENSOR_CNT, // gfx1250.
40 NUM_EXTENDED_INST_CNTS,
41 VA_VDST_RD = NUM_EXTENDED_INST_CNTS, // gfx12+ expert mode only.
42 VA_VDST_WR, // gfx12+ expert mode only.
43 VM_VSRC, // gfx12+ expert mode only.
44 NUM_EXPERT_INST_CNTS,
45 NUM_INST_CNTS = NUM_EXPERT_INST_CNTS
46};
47
48StringLiteral getInstCounterName(InstCounterType T);
49
50// Return an iterator over all counters between LOAD_CNT (the first counter)
51// and \c MaxCounter (exclusive, default value yields an enumeration over
52// all counters).
53iota_range<InstCounterType>
54inst_counter_types(InstCounterType MaxCounter = NUM_INST_CNTS);
55
56/// Represents the hardware counter limits for different wait count types.
57struct HardwareLimits {
58 unsigned LoadcntMax; // Corresponds to Vmcnt prior to gfx12.
59 unsigned ExpcntMax;
60 unsigned DscntMax; // Corresponds to LGKMcnt prior to gfx12.
61 unsigned StorecntMax; // Corresponds to VScnt in gfx10/gfx11.
62 unsigned SamplecntMax; // gfx12+ only.
63 unsigned BvhcntMax; // gfx12+ only.
64 unsigned KmcntMax; // gfx12+ only.
65 unsigned XcntMax; // gfx1250.
66 unsigned AsyncMax; // gfx1250.
67 unsigned VaVdstMax; // gfx12+ expert mode only.
68 unsigned VmVsrcMax; // gfx12+ expert mode only.
69
70 HardwareLimits() = default;
71
72 /// Initializes hardware limits from ISA version.
73 HardwareLimits(const IsaVersion &IV);
74
75 unsigned get(InstCounterType T) const;
76};
77
78} // namespace AMDGPU
79
80template <> struct enum_iteration_traits<AMDGPU::InstCounterType> {
81 static constexpr bool is_iterable = true;
82};
83
84namespace AMDGPU {
85
86/// Represents the counter values to wait for in an s_waitcnt instruction.
87///
88/// Large values (including the maximum possible integer) can be used to
89/// represent "don't care" waits.
90class Waitcnt {
91 std::array<unsigned, NUM_INST_CNTS> Cnt;
92
93public:
94 unsigned get(InstCounterType T) const { return Cnt[T]; }
95 void set(InstCounterType T, unsigned Val) { Cnt[T] = Val; }
96
97 Waitcnt() { fill(Range&: Cnt, Value: ~0u); }
98 // Pre-gfx12 constructor.
99 Waitcnt(unsigned VmCnt, unsigned ExpCnt, unsigned LgkmCnt, unsigned VsCnt)
100 : Waitcnt() {
101 Cnt[LOAD_CNT] = VmCnt;
102 Cnt[EXP_CNT] = ExpCnt;
103 Cnt[DS_CNT] = LgkmCnt;
104 Cnt[STORE_CNT] = VsCnt;
105 }
106
107 // gfx12+ constructor.
108 Waitcnt(unsigned LoadCnt, unsigned ExpCnt, unsigned DsCnt, unsigned StoreCnt,
109 unsigned SampleCnt, unsigned BvhCnt, unsigned KmCnt, unsigned XCnt,
110 unsigned AsyncCnt, unsigned TensorCnt, unsigned VaVdstRd,
111 unsigned VaVdstWr, unsigned VmVsrc)
112 : Waitcnt() {
113 Cnt[LOAD_CNT] = LoadCnt;
114 Cnt[DS_CNT] = DsCnt;
115 Cnt[EXP_CNT] = ExpCnt;
116 Cnt[STORE_CNT] = StoreCnt;
117 Cnt[SAMPLE_CNT] = SampleCnt;
118 Cnt[BVH_CNT] = BvhCnt;
119 Cnt[KM_CNT] = KmCnt;
120 Cnt[X_CNT] = XCnt;
121 Cnt[ASYNC_CNT] = AsyncCnt;
122 Cnt[TENSOR_CNT] = TensorCnt;
123 Cnt[VA_VDST_RD] = VaVdstRd;
124 Cnt[VA_VDST_WR] = VaVdstWr;
125 Cnt[VM_VSRC] = VmVsrc;
126 }
127
128 bool hasWait() const {
129 return any_of(Range: Cnt, P: [](unsigned Val) { return Val != ~0u; });
130 }
131
132 bool hasWaitExceptStoreCnt() const {
133 for (InstCounterType T : inst_counter_types()) {
134 if (T == STORE_CNT)
135 continue;
136 if (Cnt[T] != ~0u)
137 return true;
138 }
139 return false;
140 }
141
142 void add(AMDGPU::InstCounterType T, unsigned Count) {
143 set(T, Val: std::min(a: get(T), b: Count));
144 }
145
146 void clear(AMDGPU::InstCounterType T) { set(T, Val: ~0u); }
147
148 bool hasWaitStoreCnt() const { return Cnt[STORE_CNT] != ~0u; }
149
150 bool hasWaitDepctr() const {
151 return Cnt[VA_VDST_RD] != ~0u || Cnt[VA_VDST_WR] != ~0u ||
152 Cnt[VM_VSRC] != ~0u;
153 }
154
155 Waitcnt combined(const Waitcnt &Other) const {
156 // Does the right thing provided self and Other are either both pre-gfx12
157 // or both gfx12+.
158 Waitcnt Wait;
159 for (InstCounterType T : inst_counter_types())
160 Wait.Cnt[T] = std::min(a: Cnt[T], b: Other.Cnt[T]);
161 return Wait;
162 }
163
164 void print(raw_ostream &OS) const {
165 ListSeparator LS;
166 for (InstCounterType T : inst_counter_types())
167 OS << LS << getInstCounterName(T) << ": " << Cnt[T];
168 if (LS.unused())
169 OS << "none";
170 OS << '\n';
171 }
172
173#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
174 LLVM_DUMP_METHOD void dump() const;
175#endif
176
177 friend raw_ostream &operator<<(raw_ostream &OS, const AMDGPU::Waitcnt &Wait) {
178 Wait.print(OS);
179 return OS;
180 }
181};
182
183Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded);
184
185unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded);
186
187// The following are only meaningful on targets that support
188// S_WAIT_LOADCNT_DSCNT and S_WAIT_STORECNT_DSCNT.
189
190/// \returns Decoded Waitcnt structure from given \p LoadcntDscnt for given
191/// isa \p Version.
192Waitcnt decodeLoadcntDscnt(const IsaVersion &Version, unsigned LoadcntDscnt);
193
194/// \returns Decoded Waitcnt structure from given \p StorecntDscnt for given
195/// isa \p Version.
196Waitcnt decodeStorecntDscnt(const IsaVersion &Version, unsigned StorecntDscnt);
197
198/// \returns \p Loadcnt and \p Dscnt components of \p Decoded encoded as an
199/// immediate that can be used with S_WAIT_LOADCNT_DSCNT for given isa
200/// \p Version.
201unsigned encodeLoadcntDscnt(const IsaVersion &Version, const Waitcnt &Decoded);
202
203/// \returns \p Storecnt and \p Dscnt components of \p Decoded encoded as an
204/// immediate that can be used with S_WAIT_STORECNT_DSCNT for given isa
205/// \p Version.
206unsigned encodeStorecntDscnt(const IsaVersion &Version, const Waitcnt &Decoded);
207
208/// Determine if \p MI is a gfx12+ single-counter S_WAIT_*CNT instruction,
209/// and if so, which counter it is waiting on.
210std::optional<AMDGPU::InstCounterType> counterTypeForInstr(unsigned Opcode);
211
212} // namespace AMDGPU
213
214} // namespace llvm
215
216#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUWAITCNTUTILS_H
217