1//===- AMDGPUHWEvents.cpp ---------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AMDGPUHWEvents.h"
10#include "GCNSubtarget.h"
11#include "SIInstrInfo.h"
12#include "llvm/ADT/StringExtras.h"
13#include "llvm/Support/Debug.h"
14#include "llvm/Support/raw_ostream.h"
15
16namespace llvm {
17namespace AMDGPU {
18
19#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
20LLVM_DUMP_METHOD void HWEvents::dump() const { dbgs() << *this << "\n"; }
21#endif
22
23static HWEvents getExpertSchedulingEventType(const MachineInstr &Inst,
24 const SIInstrInfo &TII) {
25 if (TII.isVALU(MI: Inst, /*AllowLDSDMA=*/false)) {
26 // Core/Side-, DP-, XDL- and TRANS-MACC VALU instructions complete
27 // out-of-order with respect to each other, so each of these classes
28 // has its own event.
29
30 if (TII.isXDL(MI: Inst))
31 return HWEvents::VGPR_XDL_READ | HWEvents::VGPR_XDL_WRITE;
32
33 if (TII.isTRANS(MI: Inst))
34 return HWEvents::VGPR_TRANS_READ | HWEvents::VGPR_TRANS_WRITE;
35
36 if (AMDGPU::isDPMACCInstruction(Opc: Inst.getOpcode()))
37 return HWEvents::VGPR_DPMACC_READ | HWEvents::VGPR_DPMACC_WRITE;
38
39 return HWEvents::VGPR_CSMACC_READ | HWEvents::VGPR_CSMACC_WRITE;
40 }
41
42 // FLAT and LDS instructions may read their VGPR sources out-of-order
43 // with respect to each other and all other VMEM instructions, so
44 // each of these also has a separate event.
45
46 if (TII.isFLAT(MI: Inst))
47 return HWEvents::VGPR_FLAT_READ;
48
49 if (TII.isDS(MI: Inst))
50 return HWEvents::VGPR_LDS_READ;
51
52 if (TII.isVMEM(MI: Inst) || TII.isVIMAGE(MI: Inst) || TII.isVSAMPLE(MI: Inst))
53 return HWEvents::VGPR_VMEM_READ;
54
55 // Otherwise, no hazard.
56 return HWEvents::NONE;
57}
58
59HWEvents getSimplifiedVMEMEventsFor(const MachineInstr &Inst,
60 const SIInstrInfo &TII) {
61 switch (Inst.getOpcode()) {
62 case AMDGPU::GLOBAL_INV:
63 case AMDGPU::BUFFER_INV:
64 return HWEvents::VMEM_INV_ACCESS; // tracked using loadcnt/vmcnt, but
65 // doesn't write VGPRs
66 case AMDGPU::GLOBAL_WB:
67 case AMDGPU::GLOBAL_WBINV:
68 return HWEvents::VMEM_WRITE_ACCESS; // tracked using storecnt
69 default:
70 break;
71 }
72
73 assert(SIInstrInfo::isVMEM(Inst));
74 // LDS DMA loads are also stores, but on the LDS side. On the VMEM side
75 // these should use VM_CNT.
76 if (SIInstrInfo::mayWriteLDSThroughDMA(MI: Inst))
77 return HWEvents::VMEM_READ_ACCESS;
78
79 if (Inst.mayStore() &&
80 (!Inst.mayLoad() || SIInstrInfo::isAtomicNoRet(MI: Inst))) {
81 if (TII.mayAccessScratch(MI: Inst))
82 return HWEvents::SCRATCH_WRITE_ACCESS;
83 return HWEvents::VMEM_WRITE_ACCESS;
84 }
85
86 if (SIInstrInfo::isFLAT(MI: Inst))
87 return HWEvents::VMEM_READ_ACCESS;
88
89 if (SIInstrInfo::isImage(MI: Inst)) {
90 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc: Inst.getOpcode());
91 const AMDGPU::MIMGBaseOpcodeInfo *BaseInfo =
92 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode);
93
94 if (BaseInfo->BVH)
95 return HWEvents::VMEM_BVH_READ_ACCESS;
96
97 // We have to make an additional check for isVSAMPLE here since some
98 // instructions don't have a sampler, but are still classified as sampler
99 // instructions for the purposes of e.g. waitcnt.
100 if (BaseInfo->Sampler || BaseInfo->MSAA || SIInstrInfo::isVSAMPLE(MI: Inst))
101 return HWEvents::VMEM_SAMPLER_READ_ACCESS;
102 }
103
104 return HWEvents::VMEM_READ_ACCESS;
105}
106
107static HWEvents getEventsForImpl(const MachineInstr &Inst,
108 const GCNSubtarget &ST, const SIInstrInfo &TII,
109 bool TgSplit) {
110 if (TII.isDS(MI: Inst) && TII.usesLGKM_CNT(MI: Inst)) {
111 if (TII.isAlwaysGDS(Opcode: Inst.getOpcode()) ||
112 TII.hasModifiersSet(MI: Inst, OpName: AMDGPU::OpName::gds))
113 return HWEvents::GDS_ACCESS | HWEvents::GDS_GPR_LOCK;
114
115 return HWEvents::LDS_ACCESS;
116 }
117
118 if (TII.isFLAT(MI: Inst)) {
119 assert(Inst.mayLoadOrStore() ||
120 SIInstrInfo::isGFX12CacheInvOrWBInst(Inst.getOpcode()));
121 HWEvents E = HWEvents::NONE;
122 // Every FLAT instruction except the VGLOBAL tensor ops increments xcnt,
123 // including prefetches, async LDS DMA and LDS-only accesses that do not
124 // use the VMEM counters.
125 if (ST.hasWaitXcnt() && ST.isXNACKEnabled() &&
126 Inst.getOpcode() != AMDGPU::TENSOR_SAVE &&
127 Inst.getOpcode() != AMDGPU::TENSOR_STOP)
128 E |= HWEvents::VMEM_GROUP;
129 if (TII.mayAccessVMEMThroughFlat(MI: Inst))
130 E |= getSimplifiedVMEMEventsFor(Inst, TII);
131
132 if (TII.mayAccessLDSThroughFlat(MI: Inst, TgSplit))
133 E |= HWEvents::LDS_ACCESS;
134
135 if (SIInstrInfo::usesASYNC_CNT(MI: Inst))
136 E |= HWEvents::ASYNC_ACCESS;
137
138 return E;
139 }
140
141 if (SIInstrInfo::usesTENSOR_CNT(MI: Inst))
142 return HWEvents::TENSOR_ACCESS;
143
144 if (SIInstrInfo::isVMEM(MI: Inst) &&
145 (!AMDGPU::getMUBUFIsBufferInv(Opc: Inst.getOpcode()) ||
146 Inst.getOpcode() == AMDGPU::BUFFER_INV ||
147 Inst.getOpcode() == AMDGPU::BUFFER_WBL2)) {
148 // BUFFER_INV increments VM_CNT. BUFFER_WBL2 also needs tracking because an
149 // S_WAITCNT vmcnt(0) must follow it to ensure the writeback has completed.
150 HWEvents E = getSimplifiedVMEMEventsFor(Inst, TII);
151 if (ST.hasWaitXcnt() && ST.isXNACKEnabled())
152 E |= HWEvents::VMEM_GROUP;
153 if (ST.vmemWriteNeedsExpWaitcnt() &&
154 (Inst.mayStore() || SIInstrInfo::isAtomicRet(MI: Inst)))
155 E |= HWEvents::VMW_GPR_LOCK;
156
157 return E;
158 }
159
160 if (TII.isSMRD(MI: Inst)) {
161 if (ST.hasWaitXcnt() && ST.isXNACKEnabled())
162 return HWEvents::SMEM_GROUP | HWEvents::SMEM_ACCESS;
163 return HWEvents::SMEM_ACCESS;
164 }
165
166 if (SIInstrInfo::isLDSDIR(MI: Inst)) {
167 return HWEvents::EXP_LDS_ACCESS;
168 }
169
170 if (SIInstrInfo::isEXP(MI: Inst)) {
171 int64_t Imm = TII.getNamedOperand(MI: Inst, OperandName: AMDGPU::OpName::tgt)->getImm();
172 if (Imm >= AMDGPU::Exp::ET_PARAM0 && Imm <= AMDGPU::Exp::ET_PARAM31)
173 return HWEvents::EXP_PARAM_ACCESS;
174 if (Imm >= AMDGPU::Exp::ET_POS0 && Imm <= AMDGPU::Exp::ET_POS_LAST)
175 return HWEvents::EXP_POS_ACCESS;
176 return HWEvents::EXP_GPR_LOCK;
177 }
178
179 if (SIInstrInfo::isSBarrierSCCWrite(Opcode: Inst.getOpcode())) {
180 return HWEvents::SCC_WRITE;
181 }
182
183 switch (Inst.getOpcode()) {
184 case AMDGPU::S_SENDMSG:
185 case AMDGPU::S_SENDMSG_RTN_B32:
186 case AMDGPU::S_SENDMSG_RTN_B64:
187 case AMDGPU::S_SENDMSGHALT:
188 return HWEvents::SQ_MESSAGE;
189 case AMDGPU::S_MEMTIME:
190 case AMDGPU::S_MEMREALTIME:
191 case AMDGPU::S_GET_BARRIER_STATE_M0:
192 case AMDGPU::S_GET_BARRIER_STATE_IMM:
193 return HWEvents::SMEM_ACCESS;
194 }
195
196 return HWEvents::NONE;
197}
198
199HWEvents getEventsFor(const MachineInstr &Inst, const GCNSubtarget &ST,
200 bool IsExpertMode, bool TgSplit) {
201 const SIInstrInfo &TII = *ST.getInstrInfo();
202
203 if (IsExpertMode)
204 return getEventsForImpl(Inst, ST, TII, TgSplit) |
205 getExpertSchedulingEventType(Inst, TII);
206 return getEventsForImpl(Inst, ST, TII, TgSplit);
207}
208} // namespace AMDGPU
209
210raw_ostream &operator<<(raw_ostream &OS, AMDGPU::HWEvents Events) {
211 ListSeparator LS(" | ");
212#define AMDGPU_HW_EVENT(E, V) \
213 if (Events & AMDGPU::HWEvents::E) \
214 OS << LS << #E << " ";
215#include "AMDGPUHWEvents.def"
216 return OS;
217}
218
219} // namespace llvm
220