1//===- AMDGPUWaitcntUtils.cpp ---------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AMDGPUWaitcntUtils.h"
10#include "GCNSubtarget.h"
11#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12#include "Utils/AMDGPUBaseInfo.h"
13#include "llvm/IR/Function.h"
14
15namespace llvm::AMDGPU {
16
17bool isExpertSchedulingMode(const GCNSubtarget &ST, const Function &F) {
18 return ST.hasExpertSchedulingMode() &&
19 valueOr(X: ST.getCLOpts().expert_scheduling_mode,
20 Default: F.getFnAttribute(Kind: "amdgpu-expert-scheduling-mode")
21 .getValueAsBool());
22}
23
24iota_range<InstCounterType> inst_counter_types(InstCounterType MaxCounter) {
25 return enum_seq(Begin: LOAD_CNT, End: MaxCounter);
26}
27
28StringLiteral getInstCounterName(InstCounterType T) {
29 switch (T) {
30 case LOAD_CNT:
31 return "LOAD_CNT";
32 case DS_CNT:
33 return "DS_CNT";
34 case EXP_CNT:
35 return "EXP_CNT";
36 case STORE_CNT:
37 return "STORE_CNT";
38 case SAMPLE_CNT:
39 return "SAMPLE_CNT";
40 case BVH_CNT:
41 return "BVH_CNT";
42 case KM_CNT:
43 return "KM_CNT";
44 case X_CNT:
45 return "X_CNT";
46 case ASYNC_CNT:
47 return "ASYNC_CNT";
48 case TENSOR_CNT:
49 return "TENSOR_CNT";
50 case VA_VDST_RD:
51 return "VA_VDST_RD";
52 case VA_VDST_WR:
53 return "VA_VDST_WR";
54 case VM_VSRC:
55 return "VM_VSRC";
56 case NUM_INST_CNTS:
57 return "NUM_INST_CNTS";
58 }
59 llvm_unreachable("Unhandled InstCounterType");
60}
61
62HardwareLimits::HardwareLimits(const IsaVersion &IV) {
63 bool HasExtendedWaitCounts = IV.Major >= 12;
64 if (HasExtendedWaitCounts) {
65 LoadcntMax = getLoadcntBitMask(Version: IV);
66 DscntMax = getDscntBitMask(Version: IV);
67 } else {
68 LoadcntMax = getVmcntBitMask(Version: IV);
69 DscntMax = getLgkmcntBitMask(Version: IV);
70 }
71 ExpcntMax = getExpcntBitMask(Version: IV);
72 StorecntMax = getStorecntBitMask(Version: IV);
73 SamplecntMax = getSamplecntBitMask(Version: IV);
74 BvhcntMax = getBvhcntBitMask(Version: IV);
75 KmcntMax = getKmcntBitMask(Version: IV);
76 XcntMax = getXcntBitMask(Version: IV);
77 AsyncMax = getAsynccntBitMask(Version: IV);
78 VaVdstMax = DepCtr::getVaVdstBitMask();
79 VmVsrcMax = DepCtr::getVmVsrcBitMask();
80}
81
82unsigned HardwareLimits::get(InstCounterType T) const {
83 switch (T) {
84 case AMDGPU::LOAD_CNT:
85 return LoadcntMax;
86 case AMDGPU::DS_CNT:
87 return DscntMax;
88 case AMDGPU::EXP_CNT:
89 return ExpcntMax;
90 case AMDGPU::STORE_CNT:
91 return StorecntMax;
92 case AMDGPU::SAMPLE_CNT:
93 return SamplecntMax;
94 case AMDGPU::BVH_CNT:
95 return BvhcntMax;
96 case AMDGPU::KM_CNT:
97 return KmcntMax;
98 case AMDGPU::X_CNT:
99 return XcntMax;
100 case AMDGPU::VA_VDST_RD:
101 case AMDGPU::VA_VDST_WR:
102 return VaVdstMax;
103 case AMDGPU::VM_VSRC:
104 return VmVsrcMax;
105 default:
106 return 0;
107 }
108}
109
110#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
111void Waitcnt::dump() const { dbgs() << *this << '\n'; }
112#endif
113
114Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded) {
115 Waitcnt Decoded;
116 Decoded.set(T: LOAD_CNT, Val: decodeVmcnt(Version, Waitcnt: Encoded));
117 Decoded.set(T: EXP_CNT, Val: decodeExpcnt(Version, Waitcnt: Encoded));
118 Decoded.set(T: DS_CNT, Val: decodeLgkmcnt(Version, Waitcnt: Encoded));
119 return Decoded;
120}
121
122unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded) {
123 return encodeWaitcnt(Version, Vmcnt: Decoded.get(T: LOAD_CNT), Expcnt: Decoded.get(T: EXP_CNT),
124 Lgkmcnt: Decoded.get(T: DS_CNT));
125}
126
127Waitcnt decodeLoadcntDscnt(const IsaVersion &Version, unsigned LoadcntDscnt) {
128 Waitcnt Decoded;
129 Decoded.set(T: LOAD_CNT, Val: decodeLoadcnt(Version, Waitcnt: LoadcntDscnt));
130 Decoded.set(T: DS_CNT, Val: decodeDscnt(Version, Waitcnt: LoadcntDscnt));
131 return Decoded;
132}
133
134Waitcnt decodeStorecntDscnt(const IsaVersion &Version, unsigned StorecntDscnt) {
135 Waitcnt Decoded;
136 Decoded.set(T: STORE_CNT, Val: decodeStorecnt(Version, Waitcnt: StorecntDscnt));
137 Decoded.set(T: DS_CNT, Val: decodeDscnt(Version, Waitcnt: StorecntDscnt));
138 return Decoded;
139}
140
141unsigned encodeLoadcntDscnt(const IsaVersion &Version, const Waitcnt &Decoded) {
142 return encodeLoadcntDscnt(Version, Loadcnt: Decoded.get(T: LOAD_CNT),
143 Dscnt: Decoded.get(T: DS_CNT));
144}
145
146unsigned encodeStorecntDscnt(const IsaVersion &Version,
147 const Waitcnt &Decoded) {
148 return encodeStorecntDscnt(Version, Storecnt: Decoded.get(T: STORE_CNT),
149 Dscnt: Decoded.get(T: DS_CNT));
150}
151
152std::optional<AMDGPU::InstCounterType> counterTypeForInstr(unsigned Opcode) {
153 switch (Opcode) {
154 case AMDGPU::S_WAIT_LOADCNT:
155 return AMDGPU::LOAD_CNT;
156 case AMDGPU::S_WAIT_EXPCNT:
157 return AMDGPU::EXP_CNT;
158 case AMDGPU::S_WAIT_STORECNT:
159 return AMDGPU::STORE_CNT;
160 case AMDGPU::S_WAIT_SAMPLECNT:
161 return AMDGPU::SAMPLE_CNT;
162 case AMDGPU::S_WAIT_BVHCNT:
163 return AMDGPU::BVH_CNT;
164 case AMDGPU::S_WAIT_DSCNT:
165 return AMDGPU::DS_CNT;
166 case AMDGPU::S_WAIT_KMCNT:
167 return AMDGPU::KM_CNT;
168 case AMDGPU::S_WAIT_XCNT:
169 return AMDGPU::X_CNT;
170 case AMDGPU::S_WAIT_ASYNCCNT:
171 return AMDGPU::ASYNC_CNT;
172 case AMDGPU::S_WAIT_TENSORCNT:
173 return AMDGPU::TENSOR_CNT;
174 default:
175 return {};
176 }
177}
178
179} // namespace llvm::AMDGPU
180