1//===- AMDGPUHWEvents.cpp ---------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AMDGPUHWEvents.h"
10#include "GCNSubtarget.h"
11#include "SIInstrInfo.h"
12#include "llvm/ADT/StringExtras.h"
13#include "llvm/Support/Debug.h"
14#include "llvm/Support/raw_ostream.h"
15
16namespace llvm {
17namespace AMDGPU {
18
19#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
20LLVM_DUMP_METHOD void HWEvents::dump() const { dbgs() << *this << "\n"; }
21#endif
22
23static HWEvents getExpertSchedulingEventType(const MachineInstr &Inst,
24 const SIInstrInfo &TII) {
25 if (TII.isVALU(MI: Inst, /*AllowLDSDMA=*/false)) {
26 // Core/Side-, DP-, XDL- and TRANS-MACC VALU instructions complete
27 // out-of-order with respect to each other, so each of these classes
28 // has its own event.
29
30 if (TII.isXDL(MI: Inst))
31 return HWEvents::VGPR_XDL_READ | HWEvents::VGPR_XDL_WRITE;
32
33 if (TII.isTRANS(MI: Inst))
34 return HWEvents::VGPR_TRANS_READ | HWEvents::VGPR_TRANS_WRITE;
35
36 if (AMDGPU::isDPMACCInstruction(Opc: Inst.getOpcode()))
37 return HWEvents::VGPR_DPMACC_READ | HWEvents::VGPR_DPMACC_WRITE;
38
39 return HWEvents::VGPR_CSMACC_READ | HWEvents::VGPR_CSMACC_WRITE;
40 }
41
42 // FLAT and LDS instructions may read their VGPR sources out-of-order
43 // with respect to each other and all other VMEM instructions, so
44 // each of these also has a separate event.
45
46 if (TII.isFLAT(MI: Inst))
47 return HWEvents::VGPR_FLAT_READ;
48
49 if (TII.isDS(MI: Inst))
50 return HWEvents::VGPR_LDS_READ;
51
52 if (TII.isVMEM(MI: Inst) || TII.isVIMAGE(MI: Inst) || TII.isVSAMPLE(MI: Inst))
53 return HWEvents::VGPR_VMEM_READ;
54
55 // Otherwise, no hazard.
56 return HWEvents::NONE;
57}
58
59HWEvents getSimplifiedVMEMEventsFor(const MachineInstr &Inst,
60 const SIInstrInfo &TII) {
61 switch (Inst.getOpcode()) {
62 // FIXME: GLOBAL_INV needs to be tracked with xcnt too.
63 case AMDGPU::GLOBAL_INV:
64 case AMDGPU::BUFFER_INV:
65 return HWEvents::VMEM_INV_ACCESS; // tracked using loadcnt/vmcnt, but
66 // doesn't write VGPRs
67 case AMDGPU::GLOBAL_WB:
68 case AMDGPU::GLOBAL_WBINV:
69 return HWEvents::VMEM_WRITE_ACCESS; // tracked using storecnt
70 default:
71 break;
72 }
73
74 assert(SIInstrInfo::isVMEM(Inst));
75 // LDS DMA loads are also stores, but on the LDS side. On the VMEM side
76 // these should use VM_CNT.
77 if (SIInstrInfo::mayWriteLDSThroughDMA(MI: Inst))
78 return HWEvents::VMEM_READ_ACCESS;
79
80 if (Inst.mayStore() &&
81 (!Inst.mayLoad() || SIInstrInfo::isAtomicNoRet(MI: Inst))) {
82 if (TII.mayAccessScratch(MI: Inst))
83 return HWEvents::SCRATCH_WRITE_ACCESS;
84 return HWEvents::VMEM_WRITE_ACCESS;
85 }
86
87 if (SIInstrInfo::isFLAT(MI: Inst))
88 return HWEvents::VMEM_READ_ACCESS;
89
90 if (SIInstrInfo::isImage(MI: Inst)) {
91 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc: Inst.getOpcode());
92 const AMDGPU::MIMGBaseOpcodeInfo *BaseInfo =
93 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode);
94
95 if (BaseInfo->BVH)
96 return HWEvents::VMEM_BVH_READ_ACCESS;
97
98 // We have to make an additional check for isVSAMPLE here since some
99 // instructions don't have a sampler, but are still classified as sampler
100 // instructions for the purposes of e.g. waitcnt.
101 if (BaseInfo->Sampler || BaseInfo->MSAA || SIInstrInfo::isVSAMPLE(MI: Inst))
102 return HWEvents::VMEM_SAMPLER_READ_ACCESS;
103 }
104
105 return HWEvents::VMEM_READ_ACCESS;
106}
107
108static HWEvents getEventsForImpl(const MachineInstr &Inst,
109 const GCNSubtarget &ST, const SIInstrInfo &TII,
110 bool TgSplit) {
111 if (TII.isDS(MI: Inst) && TII.usesLGKM_CNT(MI: Inst)) {
112 if (TII.isAlwaysGDS(Opcode: Inst.getOpcode()) ||
113 TII.hasModifiersSet(MI: Inst, OpName: AMDGPU::OpName::gds))
114 return HWEvents::GDS_ACCESS | HWEvents::GDS_GPR_LOCK;
115
116 return HWEvents::LDS_ACCESS;
117 }
118
119 if (TII.isFLAT(MI: Inst)) {
120 if (SIInstrInfo::isGFX12CacheInvOrWBInst(Opc: Inst.getOpcode()))
121 return getSimplifiedVMEMEventsFor(Inst, TII);
122
123 assert(Inst.mayLoadOrStore());
124 HWEvents E = HWEvents::NONE;
125 if (TII.mayAccessVMEMThroughFlat(MI: Inst)) {
126 if (ST.hasWaitXcnt())
127 E |= HWEvents::VMEM_GROUP;
128 E |= getSimplifiedVMEMEventsFor(Inst, TII);
129 }
130
131 if (TII.mayAccessLDSThroughFlat(MI: Inst, TgSplit))
132 E |= HWEvents::LDS_ACCESS;
133
134 if (SIInstrInfo::usesASYNC_CNT(MI: Inst))
135 E |= HWEvents::ASYNC_ACCESS;
136
137 return E;
138 }
139
140 if (SIInstrInfo::usesTENSOR_CNT(MI: Inst))
141 return HWEvents::TENSOR_ACCESS;
142
143 if (SIInstrInfo::isVMEM(MI: Inst) &&
144 (!AMDGPU::getMUBUFIsBufferInv(Opc: Inst.getOpcode()) ||
145 Inst.getOpcode() == AMDGPU::BUFFER_INV ||
146 Inst.getOpcode() == AMDGPU::BUFFER_WBL2)) {
147 // BUFFER_INV increments VM_CNT. BUFFER_WBL2 also needs tracking because an
148 // S_WAITCNT vmcnt(0) must follow it to ensure the writeback has completed.
149 HWEvents E = getSimplifiedVMEMEventsFor(Inst, TII);
150 if (ST.hasWaitXcnt())
151 E |= HWEvents::VMEM_GROUP;
152 if (ST.vmemWriteNeedsExpWaitcnt() &&
153 (Inst.mayStore() || SIInstrInfo::isAtomicRet(MI: Inst)))
154 E |= HWEvents::VMW_GPR_LOCK;
155
156 return E;
157 }
158
159 if (TII.isSMRD(MI: Inst)) {
160 if (ST.hasWaitXcnt())
161 return HWEvents::SMEM_GROUP | HWEvents::SMEM_ACCESS;
162 return HWEvents::SMEM_ACCESS;
163 }
164
165 if (SIInstrInfo::isLDSDIR(MI: Inst)) {
166 return HWEvents::EXP_LDS_ACCESS;
167 }
168
169 if (SIInstrInfo::isEXP(MI: Inst)) {
170 unsigned Imm = TII.getNamedOperand(MI: Inst, OperandName: AMDGPU::OpName::tgt)->getImm();
171 if (Imm >= AMDGPU::Exp::ET_PARAM0 && Imm <= AMDGPU::Exp::ET_PARAM31)
172 return HWEvents::EXP_PARAM_ACCESS;
173 if (Imm >= AMDGPU::Exp::ET_POS0 && Imm <= AMDGPU::Exp::ET_POS_LAST)
174 return HWEvents::EXP_POS_ACCESS;
175 return HWEvents::EXP_GPR_LOCK;
176 }
177
178 if (SIInstrInfo::isSBarrierSCCWrite(Opcode: Inst.getOpcode())) {
179 return HWEvents::SCC_WRITE;
180 }
181
182 switch (Inst.getOpcode()) {
183 case AMDGPU::S_SENDMSG:
184 case AMDGPU::S_SENDMSG_RTN_B32:
185 case AMDGPU::S_SENDMSG_RTN_B64:
186 case AMDGPU::S_SENDMSGHALT:
187 return HWEvents::SQ_MESSAGE;
188 case AMDGPU::S_MEMTIME:
189 case AMDGPU::S_MEMREALTIME:
190 case AMDGPU::S_GET_BARRIER_STATE_M0:
191 case AMDGPU::S_GET_BARRIER_STATE_IMM:
192 return HWEvents::SMEM_ACCESS;
193 }
194
195 return HWEvents::NONE;
196}
197
198HWEvents getEventsFor(const MachineInstr &Inst, const GCNSubtarget &ST,
199 bool IsExpertMode, bool TgSplit) {
200 const SIInstrInfo &TII = *ST.getInstrInfo();
201
202 if (IsExpertMode)
203 return getEventsForImpl(Inst, ST, TII, TgSplit) |
204 getExpertSchedulingEventType(Inst, TII);
205 return getEventsForImpl(Inst, ST, TII, TgSplit);
206}
207} // namespace AMDGPU
208
209raw_ostream &operator<<(raw_ostream &OS, AMDGPU::HWEvents Events) {
210 ListSeparator LS(" | ");
211#define AMDGPU_HW_EVENT(E, V) \
212 if (Events & AMDGPU::HWEvents::E) \
213 OS << LS << #E << " ";
214#include "AMDGPUHWEvents.def"
215 return OS;
216}
217
218} // namespace llvm
219