1//===------------------ AMDGPUCustomBehaviour.cpp ---------------*-C++ -* -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9///
10/// This file implements methods from the AMDGPUCustomBehaviour class.
11///
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPUCustomBehaviour.h"
15#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
16#include "TargetInfo/AMDGPUTargetInfo.h"
17#include "Utils/AMDGPUBaseInfo.h"
18#include "llvm/MC/TargetRegistry.h"
19#include "llvm/Support/Compiler.h"
20#include "llvm/Support/WithColor.h"
21#include "llvm/TargetParser/AMDGPUTargetParser.h"
22
23namespace llvm::mca {
24
25void AMDGPUInstrPostProcess::postProcessInstruction(Instruction &Inst,
26 const MCInst &MCI) {
27 switch (MCI.getOpcode()) {
28 case AMDGPU::S_WAITCNT:
29 case AMDGPU::S_WAITCNT_soft:
30 case AMDGPU::S_WAITCNT_EXPCNT:
31 case AMDGPU::S_WAITCNT_LGKMCNT:
32 case AMDGPU::S_WAITCNT_VMCNT:
33 case AMDGPU::S_WAITCNT_VSCNT:
34 case AMDGPU::S_WAITCNT_VSCNT_soft:
35 case AMDGPU::S_WAITCNT_EXPCNT_gfx10:
36 case AMDGPU::S_WAITCNT_LGKMCNT_gfx10:
37 case AMDGPU::S_WAITCNT_VMCNT_gfx10:
38 case AMDGPU::S_WAITCNT_VSCNT_gfx10:
39 case AMDGPU::S_WAITCNT_gfx10:
40 case AMDGPU::S_WAITCNT_gfx6_gfx7:
41 case AMDGPU::S_WAITCNT_vi:
42 return processWaitCnt(Inst, MCI);
43 }
44}
45
46// s_waitcnt instructions encode important information as immediate operands
47// which are lost during the MCInst -> mca::Instruction lowering.
48void AMDGPUInstrPostProcess::processWaitCnt(Instruction &Inst,
49 const MCInst &MCI) {
50 for (unsigned Idx = 0, N = MCI.getNumOperands(); Idx < N; Idx++) {
51 MCAOperand Op;
52 const MCOperand &MCOp = MCI.getOperand(i: Idx);
53 if (MCOp.isReg()) {
54 Op = MCAOperand::createReg(Reg: MCOp.getReg());
55 } else if (MCOp.isImm()) {
56 Op = MCAOperand::createImm(Val: MCOp.getImm());
57 }
58 Op.setIndex(Idx);
59 Inst.addOperand(Op);
60 }
61}
62
63AMDGPUCustomBehaviour::AMDGPUCustomBehaviour(const MCSubtargetInfo &STI,
64 const mca::SourceMgr &SrcMgr,
65 const MCInstrInfo &MCII)
66 : CustomBehaviour(STI, SrcMgr, MCII) {
67 generateWaitCntInfo();
68}
69
70unsigned AMDGPUCustomBehaviour::checkCustomHazard(ArrayRef<InstRef> IssuedInst,
71 const InstRef &IR) {
72 const Instruction &Inst = *IR.getInstruction();
73 unsigned Opcode = Inst.getOpcode();
74
75 // llvm-mca is generally run on fully compiled assembly so we wouldn't see any
76 // pseudo instructions here. However, there are plans for the future to make
77 // it possible to use mca within backend passes. As such, I have left the
78 // pseudo version of s_waitcnt within this switch statement.
79 switch (Opcode) {
80 default:
81 return 0;
82 case AMDGPU::S_WAITCNT: // This instruction
83 case AMDGPU::S_WAITCNT_soft:
84 case AMDGPU::S_WAITCNT_EXPCNT:
85 case AMDGPU::S_WAITCNT_LGKMCNT:
86 case AMDGPU::S_WAITCNT_VMCNT:
87 case AMDGPU::S_WAITCNT_VSCNT:
88 case AMDGPU::S_WAITCNT_VSCNT_soft: // to this instruction are all pseudo.
89 case AMDGPU::S_WAITCNT_EXPCNT_gfx10:
90 case AMDGPU::S_WAITCNT_LGKMCNT_gfx10:
91 case AMDGPU::S_WAITCNT_VMCNT_gfx10:
92 case AMDGPU::S_WAITCNT_VSCNT_gfx10:
93 case AMDGPU::S_WAITCNT_gfx10:
94 case AMDGPU::S_WAITCNT_gfx6_gfx7:
95 case AMDGPU::S_WAITCNT_vi:
96 // s_endpgm also behaves as if there is an implicit
97 // s_waitcnt 0, but I'm not sure if it would be appropriate
98 // to model this in llvm-mca based on how the iterations work
99 // while simulating the pipeline over and over.
100 return handleWaitCnt(IssuedInst, IR);
101 }
102
103 return 0;
104}
105
106unsigned AMDGPUCustomBehaviour::handleWaitCnt(ArrayRef<InstRef> IssuedInst,
107 const InstRef &IR) {
108 // Currently, all s_waitcnt instructions are handled except s_waitcnt_depctr.
109 // I do not know how that instruction works so I did not attempt to model it.
110 // set the max values to begin
111 unsigned Vmcnt = 63;
112 unsigned Expcnt = 7;
113 unsigned Lgkmcnt = 31;
114 unsigned Vscnt = 63;
115 unsigned CurrVmcnt = 0;
116 unsigned CurrExpcnt = 0;
117 unsigned CurrLgkmcnt = 0;
118 unsigned CurrVscnt = 0;
119 unsigned CyclesToWaitVm = ~0U;
120 unsigned CyclesToWaitExp = ~0U;
121 unsigned CyclesToWaitLgkm = ~0U;
122 unsigned CyclesToWaitVs = ~0U;
123
124 computeWaitCnt(IR, Vmcnt, Expcnt, Lgkmcnt, Vscnt);
125
126 // We will now look at each of the currently executing instructions
127 // to find out if this wait instruction still needs to wait.
128 for (const InstRef &PrevIR : IssuedInst) {
129 const Instruction &PrevInst = *PrevIR.getInstruction();
130 const unsigned PrevInstIndex = PrevIR.getSourceIndex() % SrcMgr.size();
131 const WaitCntInfo &PrevInstWaitInfo = InstrWaitCntInfo[PrevInstIndex];
132 const int CyclesLeft = PrevInst.getCyclesLeft();
133 assert(CyclesLeft != UNKNOWN_CYCLES &&
134 "We should know how many cycles are left for this instruction");
135 if (PrevInstWaitInfo.VmCnt) {
136 CurrVmcnt++;
137 if ((unsigned)CyclesLeft < CyclesToWaitVm)
138 CyclesToWaitVm = CyclesLeft;
139 }
140 if (PrevInstWaitInfo.ExpCnt) {
141 CurrExpcnt++;
142 if ((unsigned)CyclesLeft < CyclesToWaitExp)
143 CyclesToWaitExp = CyclesLeft;
144 }
145 if (PrevInstWaitInfo.LgkmCnt) {
146 CurrLgkmcnt++;
147 if ((unsigned)CyclesLeft < CyclesToWaitLgkm)
148 CyclesToWaitLgkm = CyclesLeft;
149 }
150 if (PrevInstWaitInfo.VsCnt) {
151 CurrVscnt++;
152 if ((unsigned)CyclesLeft < CyclesToWaitVs)
153 CyclesToWaitVs = CyclesLeft;
154 }
155 }
156
157 unsigned CyclesToWait = ~0U;
158 if (CurrVmcnt > Vmcnt && CyclesToWaitVm < CyclesToWait)
159 CyclesToWait = CyclesToWaitVm;
160 if (CurrExpcnt > Expcnt && CyclesToWaitExp < CyclesToWait)
161 CyclesToWait = CyclesToWaitExp;
162 if (CurrLgkmcnt > Lgkmcnt && CyclesToWaitLgkm < CyclesToWait)
163 CyclesToWait = CyclesToWaitLgkm;
164 if (CurrVscnt > Vscnt && CyclesToWaitVs < CyclesToWait)
165 CyclesToWait = CyclesToWaitVs;
166
167 // We may underestimate how many cycles we need to wait, but this
168 // isn't a big deal. Our return value is just how many cycles until
169 // this function gets run again. So as long as we don't overestimate
170 // the wait time, we'll still end up stalling at this instruction
171 // for the correct number of cycles.
172
173 if (CyclesToWait == ~0U)
174 return 0;
175 return CyclesToWait;
176}
177
178void AMDGPUCustomBehaviour::computeWaitCnt(const InstRef &IR, unsigned &Vmcnt,
179 unsigned &Expcnt, unsigned &Lgkmcnt,
180 unsigned &Vscnt) {
181 AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(GPU: STI.getCPU());
182 const Instruction &Inst = *IR.getInstruction();
183 unsigned Opcode = Inst.getOpcode();
184
185 switch (Opcode) {
186 case AMDGPU::S_WAITCNT_EXPCNT_gfx10:
187 case AMDGPU::S_WAITCNT_LGKMCNT_gfx10:
188 case AMDGPU::S_WAITCNT_VMCNT_gfx10:
189 case AMDGPU::S_WAITCNT_VSCNT_gfx10: {
190 // Should probably be checking for nullptr
191 // here, but I'm not sure how I should handle the case
192 // where we see a nullptr.
193 const MCAOperand *OpReg = Inst.getOperand(Idx: 0);
194 const MCAOperand *OpImm = Inst.getOperand(Idx: 1);
195 assert(OpReg && OpReg->isReg() && "First operand should be a register.");
196 assert(OpImm && OpImm->isImm() && "Second operand should be an immediate.");
197 if (OpReg->getReg() != AMDGPU::SGPR_NULL) {
198 // Instruction is using a real register.
199 // Since we can't know what value this register will have,
200 // we can't compute what the value of this wait should be.
201 WithColor::warning() << "The register component of "
202 << MCII.getName(Opcode) << " will be completely "
203 << "ignored. So the wait may not be accurate.\n";
204 }
205 unsigned ImmEnc = static_cast<unsigned>(OpImm->getImm());
206 switch (Opcode) {
207 // Redundant switch so I don't have to repeat the code above
208 // for each case. There are more clever ways to avoid this
209 // extra switch and anyone can feel free to implement one of them.
210 case AMDGPU::S_WAITCNT_EXPCNT_gfx10:
211 Expcnt = ImmEnc;
212 break;
213 case AMDGPU::S_WAITCNT_LGKMCNT_gfx10:
214 Lgkmcnt = ImmEnc;
215 break;
216 case AMDGPU::S_WAITCNT_VMCNT_gfx10:
217 Vmcnt = ImmEnc;
218 break;
219 case AMDGPU::S_WAITCNT_VSCNT_gfx10:
220 Vscnt = ImmEnc;
221 break;
222 }
223 return;
224 }
225 case AMDGPU::S_WAITCNT_gfx10:
226 case AMDGPU::S_WAITCNT_gfx6_gfx7:
227 case AMDGPU::S_WAITCNT_vi:
228 unsigned WaitCnt = static_cast<unsigned>(Inst.getOperand(Idx: 0)->getImm());
229 AMDGPU::decodeWaitcnt(Version: IV, Waitcnt: WaitCnt, Vmcnt, Expcnt, Lgkmcnt);
230 return;
231 }
232}
233
234void AMDGPUCustomBehaviour::generateWaitCntInfo() {
235 // The core logic from this function is taken from
236 // SIInsertWaitcnts::updateEventWaitcntAfter() In that pass, the instructions
237 // that are being looked at are in the MachineInstr format, whereas we have
238 // access to the MCInst format. The side effects of this are that we can't use
239 // the mayAccessVMEMThroughFlat(Inst) or mayAccessLDSThroughFlat(Inst)
240 // functions. Therefore, we conservatively assume that these functions will
241 // return true. This may cause a few instructions to be incorrectly tagged
242 // with an extra CNT. However, these are instructions that do interact with at
243 // least one CNT so giving them an extra CNT shouldn't cause issues in most
244 // scenarios.
245 AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(GPU: STI.getCPU());
246 InstrWaitCntInfo.resize(new_size: SrcMgr.size());
247
248 for (const auto &EN : llvm::enumerate(First: SrcMgr.getInstructions())) {
249 const std::unique_ptr<Instruction> &Inst = EN.value();
250 size_t Index = EN.index();
251 unsigned Opcode = Inst->getOpcode();
252 const MCInstrDesc &MCID = MCII.get(Opcode);
253 if (SIInstrFlags::isDS(O: MCID) && SIInstrFlags::usesLGKM_CNT(O: MCID)) {
254 InstrWaitCntInfo[Index].LgkmCnt = true;
255 if (isAlwaysGDS(Opcode) || hasModifiersSet(Inst, OpName: AMDGPU::OpName::gds))
256 InstrWaitCntInfo[Index].ExpCnt = true;
257 } else if (SIInstrFlags::isFLAT(O: MCID)) {
258 // We conservatively assume that mayAccessVMEMThroughFlat(Inst)
259 // and mayAccessLDSThroughFlat(Inst) would both return true for this
260 // instruction. We have to do this because those functions use
261 // information about the memory operands that we don't have access to.
262 InstrWaitCntInfo[Index].LgkmCnt = true;
263 if (!STI.hasFeature(Feature: AMDGPU::FeatureVscnt))
264 InstrWaitCntInfo[Index].VmCnt = true;
265 else if (MCID.mayLoad() && !SIInstrFlags::isAtomicNoRet(O: MCID))
266 InstrWaitCntInfo[Index].VmCnt = true;
267 else
268 InstrWaitCntInfo[Index].VsCnt = true;
269 } else if (SIInstrFlags::isVMEM(O: MCID) &&
270 !AMDGPU::getMUBUFIsBufferInv(Opc: Opcode)) {
271 if (!STI.hasFeature(Feature: AMDGPU::FeatureVscnt))
272 InstrWaitCntInfo[Index].VmCnt = true;
273 else if ((MCID.mayLoad() && !SIInstrFlags::isAtomicNoRet(O: MCID)) ||
274 (SIInstrFlags::isMIMG(O: MCID) && !MCID.mayLoad() &&
275 !MCID.mayStore()))
276 InstrWaitCntInfo[Index].VmCnt = true;
277 else if (MCID.mayStore())
278 InstrWaitCntInfo[Index].VsCnt = true;
279
280 // (IV.Major < 7) is meant to represent
281 // GCNTarget.vmemWriteNeedsExpWaitcnt()
282 // which is defined as
283 // { return getGeneration() < SEA_ISLANDS; }
284 if (IV.Major < 7 && (MCID.mayStore() || SIInstrFlags::isAtomicRet(O: MCID)))
285 InstrWaitCntInfo[Index].ExpCnt = true;
286 } else if (SIInstrFlags::isSMRD(O: MCID)) {
287 InstrWaitCntInfo[Index].LgkmCnt = true;
288 } else if (SIInstrFlags::isEXP(O: MCID)) {
289 InstrWaitCntInfo[Index].ExpCnt = true;
290 } else {
291 switch (Opcode) {
292 case AMDGPU::S_SENDMSG:
293 case AMDGPU::S_SENDMSGHALT:
294 case AMDGPU::S_MEMTIME:
295 case AMDGPU::S_MEMREALTIME:
296 InstrWaitCntInfo[Index].LgkmCnt = true;
297 break;
298 }
299 }
300 }
301}
302
303// taken from SIInstrInfo::hasModifiersSet()
304bool AMDGPUCustomBehaviour::hasModifiersSet(
305 const std::unique_ptr<Instruction> &Inst, AMDGPU::OpName OpName) const {
306 int Idx = AMDGPU::getNamedOperandIdx(Opcode: Inst->getOpcode(), Name: OpName);
307 if (Idx == -1)
308 return false;
309
310 const MCAOperand *Op = Inst->getOperand(Idx);
311 if (Op == nullptr || !Op->isImm() || !Op->getImm())
312 return false;
313
314 return true;
315}
316
317// taken from SIInstrInfo::isAlwaysGDS()
318bool AMDGPUCustomBehaviour::isAlwaysGDS(uint32_t Opcode) const {
319 return Opcode == AMDGPU::DS_ORDERED_COUNT ||
320 Opcode == AMDGPU::DS_ADD_GS_REG_RTN ||
321 Opcode == AMDGPU::DS_SUB_GS_REG_RTN ||
322 SIInstrFlags::isGWS(O: MCII, O: Opcode);
323}
324
325} // namespace llvm::mca
326
327using namespace llvm;
328using namespace mca;
329
330static CustomBehaviour *
331createAMDGPUCustomBehaviour(const MCSubtargetInfo &STI,
332 const mca::SourceMgr &SrcMgr,
333 const MCInstrInfo &MCII) {
334 return new AMDGPUCustomBehaviour(STI, SrcMgr, MCII);
335}
336
337static InstrPostProcess *
338createAMDGPUInstrPostProcess(const MCSubtargetInfo &STI,
339 const MCInstrInfo &MCII) {
340 return new AMDGPUInstrPostProcess(STI, MCII);
341}
342
343/// Extern function to initialize the targets for the AMDGPU backend
344
345extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
346LLVMInitializeAMDGPUTargetMCA() {
347 TargetRegistry::RegisterCustomBehaviour(T&: getTheR600Target(),
348 Fn: createAMDGPUCustomBehaviour);
349 TargetRegistry::RegisterInstrPostProcess(T&: getTheR600Target(),
350 Fn: createAMDGPUInstrPostProcess);
351
352 TargetRegistry::RegisterCustomBehaviour(T&: getTheGCNTarget(),
353 Fn: createAMDGPUCustomBehaviour);
354 TargetRegistry::RegisterInstrPostProcess(T&: getTheGCNTarget(),
355 Fn: createAMDGPUInstrPostProcess);
356
357 TargetRegistry::RegisterCustomBehaviour(T&: getTheGCNLegacyTarget(),
358 Fn: createAMDGPUCustomBehaviour);
359 TargetRegistry::RegisterInstrPostProcess(T&: getTheGCNLegacyTarget(),
360 Fn: createAMDGPUInstrPostProcess);
361}
362