1//===- AMDGPUWaitcntUtils.cpp ---------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AMDGPUWaitcntUtils.h"
10#include "GCNSubtarget.h"
11#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12#include "Utils/AMDGPUBaseInfo.h"
13#include "llvm/IR/Function.h"
14#include "llvm/Support/CommandLine.h"
15
16namespace llvm::AMDGPU {
17
18static cl::opt<bool> ExpertSchedulingModeFlag(
19 "amdgpu-expert-scheduling-mode",
20 cl::desc("Enable expert scheduling mode 2 for all functions (GFX12+ only)"),
21 cl::init(Val: false), cl::Hidden);
22
23bool isExpertSchedulingMode(const GCNSubtarget &ST, const Function &F) {
24 return ST.hasExpertSchedulingMode() &&
25 (ExpertSchedulingModeFlag.getNumOccurrences()
26 ? ExpertSchedulingModeFlag
27 : F.getFnAttribute(Kind: "amdgpu-expert-scheduling-mode")
28 .getValueAsBool());
29}
30
31iota_range<InstCounterType> inst_counter_types(InstCounterType MaxCounter) {
32 return enum_seq(Begin: LOAD_CNT, End: MaxCounter);
33}
34
35StringLiteral getInstCounterName(InstCounterType T) {
36 switch (T) {
37 case LOAD_CNT:
38 return "LOAD_CNT";
39 case DS_CNT:
40 return "DS_CNT";
41 case EXP_CNT:
42 return "EXP_CNT";
43 case STORE_CNT:
44 return "STORE_CNT";
45 case SAMPLE_CNT:
46 return "SAMPLE_CNT";
47 case BVH_CNT:
48 return "BVH_CNT";
49 case KM_CNT:
50 return "KM_CNT";
51 case X_CNT:
52 return "X_CNT";
53 case ASYNC_CNT:
54 return "ASYNC_CNT";
55 case TENSOR_CNT:
56 return "TENSOR_CNT";
57 case VA_VDST_RD:
58 return "VA_VDST_RD";
59 case VA_VDST_WR:
60 return "VA_VDST_WR";
61 case VM_VSRC:
62 return "VM_VSRC";
63 case NUM_INST_CNTS:
64 return "NUM_INST_CNTS";
65 }
66 llvm_unreachable("Unhandled InstCounterType");
67}
68
69HardwareLimits::HardwareLimits(const IsaVersion &IV) {
70 bool HasExtendedWaitCounts = IV.Major >= 12;
71 if (HasExtendedWaitCounts) {
72 LoadcntMax = getLoadcntBitMask(Version: IV);
73 DscntMax = getDscntBitMask(Version: IV);
74 } else {
75 LoadcntMax = getVmcntBitMask(Version: IV);
76 DscntMax = getLgkmcntBitMask(Version: IV);
77 }
78 ExpcntMax = getExpcntBitMask(Version: IV);
79 StorecntMax = getStorecntBitMask(Version: IV);
80 SamplecntMax = getSamplecntBitMask(Version: IV);
81 BvhcntMax = getBvhcntBitMask(Version: IV);
82 KmcntMax = getKmcntBitMask(Version: IV);
83 XcntMax = getXcntBitMask(Version: IV);
84 AsyncMax = getAsynccntBitMask(Version: IV);
85 VaVdstMax = DepCtr::getVaVdstBitMask();
86 VmVsrcMax = DepCtr::getVmVsrcBitMask();
87}
88
89unsigned HardwareLimits::get(InstCounterType T) const {
90 switch (T) {
91 case AMDGPU::LOAD_CNT:
92 return LoadcntMax;
93 case AMDGPU::DS_CNT:
94 return DscntMax;
95 case AMDGPU::EXP_CNT:
96 return ExpcntMax;
97 case AMDGPU::STORE_CNT:
98 return StorecntMax;
99 case AMDGPU::SAMPLE_CNT:
100 return SamplecntMax;
101 case AMDGPU::BVH_CNT:
102 return BvhcntMax;
103 case AMDGPU::KM_CNT:
104 return KmcntMax;
105 case AMDGPU::X_CNT:
106 return XcntMax;
107 case AMDGPU::VA_VDST_RD:
108 case AMDGPU::VA_VDST_WR:
109 return VaVdstMax;
110 case AMDGPU::VM_VSRC:
111 return VmVsrcMax;
112 default:
113 return 0;
114 }
115}
116
117#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
118void Waitcnt::dump() const { dbgs() << *this << '\n'; }
119#endif
120
121Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded) {
122 Waitcnt Decoded;
123 Decoded.set(T: LOAD_CNT, Val: decodeVmcnt(Version, Waitcnt: Encoded));
124 Decoded.set(T: EXP_CNT, Val: decodeExpcnt(Version, Waitcnt: Encoded));
125 Decoded.set(T: DS_CNT, Val: decodeLgkmcnt(Version, Waitcnt: Encoded));
126 return Decoded;
127}
128
129unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded) {
130 return encodeWaitcnt(Version, Vmcnt: Decoded.get(T: LOAD_CNT), Expcnt: Decoded.get(T: EXP_CNT),
131 Lgkmcnt: Decoded.get(T: DS_CNT));
132}
133
134Waitcnt decodeLoadcntDscnt(const IsaVersion &Version, unsigned LoadcntDscnt) {
135 Waitcnt Decoded;
136 Decoded.set(T: LOAD_CNT, Val: decodeLoadcnt(Version, Waitcnt: LoadcntDscnt));
137 Decoded.set(T: DS_CNT, Val: decodeDscnt(Version, Waitcnt: LoadcntDscnt));
138 return Decoded;
139}
140
141Waitcnt decodeStorecntDscnt(const IsaVersion &Version, unsigned StorecntDscnt) {
142 Waitcnt Decoded;
143 Decoded.set(T: STORE_CNT, Val: decodeStorecnt(Version, Waitcnt: StorecntDscnt));
144 Decoded.set(T: DS_CNT, Val: decodeDscnt(Version, Waitcnt: StorecntDscnt));
145 return Decoded;
146}
147
148unsigned encodeLoadcntDscnt(const IsaVersion &Version, const Waitcnt &Decoded) {
149 return encodeLoadcntDscnt(Version, Loadcnt: Decoded.get(T: LOAD_CNT),
150 Dscnt: Decoded.get(T: DS_CNT));
151}
152
153unsigned encodeStorecntDscnt(const IsaVersion &Version,
154 const Waitcnt &Decoded) {
155 return encodeStorecntDscnt(Version, Storecnt: Decoded.get(T: STORE_CNT),
156 Dscnt: Decoded.get(T: DS_CNT));
157}
158
159std::optional<AMDGPU::InstCounterType> counterTypeForInstr(unsigned Opcode) {
160 switch (Opcode) {
161 case AMDGPU::S_WAIT_LOADCNT:
162 return AMDGPU::LOAD_CNT;
163 case AMDGPU::S_WAIT_EXPCNT:
164 return AMDGPU::EXP_CNT;
165 case AMDGPU::S_WAIT_STORECNT:
166 return AMDGPU::STORE_CNT;
167 case AMDGPU::S_WAIT_SAMPLECNT:
168 return AMDGPU::SAMPLE_CNT;
169 case AMDGPU::S_WAIT_BVHCNT:
170 return AMDGPU::BVH_CNT;
171 case AMDGPU::S_WAIT_DSCNT:
172 return AMDGPU::DS_CNT;
173 case AMDGPU::S_WAIT_KMCNT:
174 return AMDGPU::KM_CNT;
175 case AMDGPU::S_WAIT_XCNT:
176 return AMDGPU::X_CNT;
177 case AMDGPU::S_WAIT_ASYNCCNT:
178 return AMDGPU::ASYNC_CNT;
179 case AMDGPU::S_WAIT_TENSORCNT:
180 return AMDGPU::TENSOR_CNT;
181 default:
182 return {};
183 }
184}
185
186} // namespace llvm::AMDGPU
187