1//===- SIInsertWaitcnts.cpp - Insert Wait Instructions --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Insert wait instructions for memory reads and writes.
11///
12/// Memory reads and writes are issued asynchronously, so we need to insert
13/// S_WAITCNT instructions when we want to access any of their results or
14/// overwrite any register that's used asynchronously.
15///
16/// TODO: This pass currently keeps one timeline per hardware counter. A more
17/// finely-grained approach that keeps one timeline per event type could
18/// sometimes get away with generating weaker s_waitcnt instructions. For
19/// example, when both SMEM and LDS are in flight and we need to wait for
20/// the i-th-last LDS instruction, then an lgkmcnt(i) is actually sufficient,
21/// but the pass will currently generate a conservative lgkmcnt(0) because
22/// multiple event types are in flight.
23//
24//===----------------------------------------------------------------------===//
25
26#include "AMDGPU.h"
27#include "AMDGPUHWEvents.h"
28#include "AMDGPUWaitcntUtils.h"
29#include "GCNSubtarget.h"
30#include "SIMachineFunctionInfo.h"
31#include "Utils/AMDGPUBaseInfo.h"
32#include "llvm/Analysis/AliasAnalysis.h"
33#include "llvm/CodeGen/MachineFrameInfo.h"
34#include "llvm/CodeGen/MachineLoopInfo.h"
35#include "llvm/CodeGen/MachinePassManager.h"
36#include "llvm/CodeGen/MachinePostDominators.h"
37#include "llvm/IR/Dominators.h"
38#include "llvm/InitializePasses.h"
39#include "llvm/TargetParser/AMDGPUTargetParser.h"
40
41using namespace llvm;
42
43using HWEvents = AMDGPU::HWEvents;
44
45#define DEBUG_TYPE "si-insert-waitcnts"
46
47static cl::opt<bool>
48 ForceEmitZeroFlag("amdgpu-waitcnt-forcezero",
49 cl::desc("Force all waitcnt instrs to be emitted as "
50 "s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"),
51 cl::init(Val: false), cl::Hidden);
52
53static cl::opt<bool> ForceEmitZeroLoadFlag(
54 "amdgpu-waitcnt-load-forcezero",
55 cl::desc("Force all waitcnt load counters to wait until 0"),
56 cl::init(Val: false), cl::Hidden);
57
58namespace {
59
60template <typename EmitWaitcntFn>
61static void EmitExpandedWaitcnt(unsigned Outstanding, unsigned Target,
62 EmitWaitcntFn &&EmitWaitcnt) {
63 // Emit waitcnts from (Outstanding - 1) down to Target.
64 for (unsigned I = Outstanding - 1; I > Target && I != ~0u; --I)
65 EmitWaitcnt(I);
66 EmitWaitcnt(Target);
67}
68
69/// Integer IDs used to track vector memory locations we may have to wait on.
70/// Encoded as u16 chunks:
71///
72/// [0, REGUNITS_END ): MCRegUnit
73/// [LDSDMA_BEGIN, LDSDMA_END ) : LDS DMA IDs
74///
75/// NOTE: The choice of encoding these as "u16 chunks" is arbitrary.
76/// It gives (2 << 16) - 1 entries per category which is more than enough
77/// for all register units. MCPhysReg is u16 so we don't even support >u16
78/// physical register numbers at this time, let alone >u16 register units.
79/// In any case, an assertion in "WaitcntBrackets" ensures REGUNITS_END
80/// is enough for all register units.
81using VMEMID = uint32_t;
82
83enum : VMEMID {
84 TRACKINGID_RANGE_LEN = (1 << 16),
85
86 // Important: MCRegUnits must always be tracked starting from 0, as we
87 // need to be able to convert between a MCRegUnit and a VMEMID freely.
88 REGUNITS_BEGIN = 0,
89 REGUNITS_END = REGUNITS_BEGIN + TRACKINGID_RANGE_LEN,
90
91 // Note for LDSDMA: LDSDMA_BEGIN corresponds to the "common"
92 // entry, which is updated for all LDS DMA operations encountered.
93 // Specific LDS DMA IDs start at LDSDMA_BEGIN + 1.
94 NUM_LDSDMA = TRACKINGID_RANGE_LEN,
95 LDSDMA_BEGIN = REGUNITS_END,
96 LDSDMA_END = LDSDMA_BEGIN + NUM_LDSDMA,
97};
98
99/// Convert a MCRegUnit to a VMEMID.
100static constexpr VMEMID toVMEMID(MCRegUnit RU) {
101 return static_cast<unsigned>(RU);
102}
103
104} // namespace
105
106namespace {
107
108// Maps values of InstCounterType to the instruction that waits on that
109// counter. Only used if GCNSubtarget::hasExtendedWaitCounts()
110// returns true, and does not cover VA_VDST or VM_VSRC.
111static const unsigned
112 instrsForExtendedCounterTypes[AMDGPU::NUM_EXTENDED_INST_CNTS] = {
113 AMDGPU::S_WAIT_LOADCNT, AMDGPU::S_WAIT_DSCNT,
114 AMDGPU::S_WAIT_EXPCNT, AMDGPU::S_WAIT_STORECNT,
115 AMDGPU::S_WAIT_SAMPLECNT, AMDGPU::S_WAIT_BVHCNT,
116 AMDGPU::S_WAIT_KMCNT, AMDGPU::S_WAIT_XCNT,
117 AMDGPU::S_WAIT_ASYNCCNT, AMDGPU::S_WAIT_TENSORCNT};
118
119// ASYNCMARK and WAIT_ASYNCMARK are meta instructions that emit no hardware
120// code but still need to be processed by this pass for async vmcnt tracking.
121static bool isNonWaitcntMetaInst(const MachineInstr &MI) {
122 switch (MI.getOpcode()) {
123 case AMDGPU::ASYNCMARK:
124 case AMDGPU::WAIT_ASYNCMARK:
125 return false;
126 default:
127 return MI.isMetaInstruction();
128 }
129}
130
131static bool updateVMCntOnly(const MachineInstr &Inst) {
132 return (SIInstrInfo::isVMEM(MI: Inst) && !SIInstrInfo::isFLAT(MI: Inst)) ||
133 SIInstrInfo::isFLATGlobal(MI: Inst) || SIInstrInfo::isFLATScratch(MI: Inst);
134}
135
136#ifndef NDEBUG
137static bool isNormalMode(AMDGPU::InstCounterType MaxCounter) {
138 return MaxCounter == AMDGPU::NUM_NORMAL_INST_CNTS;
139}
140#endif // NDEBUG
141
142class WaitcntBrackets;
143
144// This abstracts the logic for generating and updating S_WAIT* instructions
145// away from the analysis that determines where they are needed. This was
146// done because the set of counters and instructions for waiting on them
147// underwent a major shift with gfx12, sufficiently so that having this
148// abstraction allows the main analysis logic to be simpler than it would
149// otherwise have had to become.
150class WaitcntGenerator {
151protected:
152 const GCNSubtarget &ST;
153 const SIInstrInfo &TII;
154 AMDGPU::IsaVersion IV;
155 AMDGPU::InstCounterType MaxCounter;
156 bool OptNone;
157 bool ExpandWaitcntProfiling = false;
158 const AMDGPU::HardwareLimits &Limits;
159
160 // Soft waitcnts whose erasures are postponed so we can confirm if they're
161 // truly redundant or if they're waiting on a loop-carried dependency.
162 DenseSet<MachineInstr *> DeferredErasures;
163
164public:
165 WaitcntGenerator() = delete;
166 WaitcntGenerator(const WaitcntGenerator &) = delete;
167 WaitcntGenerator(const MachineFunction &MF,
168 AMDGPU::InstCounterType MaxCounter,
169 const AMDGPU::HardwareLimits &Limits)
170 : ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()),
171 IV(AMDGPU::getIsaVersion(GPU: ST.getCPU())), MaxCounter(MaxCounter),
172 OptNone(MF.getFunction().hasOptNone() ||
173 MF.getTarget().getOptLevel() == CodeGenOptLevel::None),
174 ExpandWaitcntProfiling(
175 MF.getFunction().hasFnAttribute(Kind: "amdgpu-expand-waitcnt-profiling")),
176 Limits(Limits) {}
177
178 // Return true if the current function should be compiled with no
179 // optimization.
180 bool isOptNone() const { return OptNone; }
181
182 unsigned getLimit(AMDGPU::InstCounterType E) const { return Limits.get(T: E); }
183
184 // Edits an existing sequence of wait count instructions according
185 // to an incoming Waitcnt value, which is itself updated to reflect
186 // any new wait count instructions which may need to be generated by
187 // WaitcntGenerator::createNewWaitcnt(). It will return true if any edits
188 // were made.
189 //
190 // This editing will usually be merely updated operands, but it may also
191 // delete instructions if the incoming Wait value indicates they are not
192 // needed. It may also remove existing instructions for which a wait
193 // is needed if it can be determined that it is better to generate new
194 // instructions later, as can happen on gfx12.
195 virtual bool
196 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
197 MachineInstr &OldWaitcntInstr, AMDGPU::Waitcnt &Wait,
198 MachineBasicBlock::instr_iterator It) = 0;
199
200 bool eraseDeferredWaitcnts() {
201 bool Modified = false;
202 for (MachineInstr *MI : DeferredErasures) {
203 MI->eraseFromParent();
204 Modified = true;
205 }
206 DeferredErasures.clear();
207 return Modified;
208 }
209
210 // Transform a soft waitcnt into a normal one.
211 bool promoteSoftWaitCnt(MachineInstr *Waitcnt) const;
212
213 // Generates new wait count instructions according to the value of
214 // Wait, returning true if any new instructions were created.
215 // ScoreBrackets is used for profiling expansion.
216 virtual bool createNewWaitcnt(MachineBasicBlock &Block,
217 MachineBasicBlock::instr_iterator It,
218 AMDGPU::Waitcnt Wait,
219 const WaitcntBrackets &ScoreBrackets) = 0;
220
221 // Returns the set of HWEvents that corresponds to counter \p T.
222 virtual HWEvents getWaitEvents(AMDGPU::InstCounterType T) const = 0;
223
224 /// \returns the counter that corresponds to event \p E.
225 AMDGPU::InstCounterType getCounterFromEvent(HWEvents E) const {
226 assert(E.size() == 1 && "Cannot handle a mask of events!");
227 for (auto T : AMDGPU::inst_counter_types()) {
228 if (getWaitEvents(T) & E)
229 return T;
230 }
231 llvm_unreachable("event type has no associated counter");
232 }
233
234 // Returns a new waitcnt with all counters except VScnt set to 0. If
235 // IncludeVSCnt is true, VScnt is set to 0, otherwise it is set to ~0u.
236 // AsyncCnt and TensorCnt always default to ~0u (don't wait for it). They
237 // are only updated when a call to @llvm.amdgcn.wait.asyncmark() is
238 // processed.
239 virtual AMDGPU::Waitcnt getAllZeroWaitcnt(bool IncludeVSCnt) const = 0;
240
241 virtual ~WaitcntGenerator() = default;
242};
243
244class WaitcntGeneratorPreGFX12 final : public WaitcntGenerator {
245 static constexpr const HWEvents
246 WaitEventMaskForInstPreGFX12[AMDGPU::NUM_INST_CNTS] = {
247 HWEvents::VMEM_READ_ACCESS | HWEvents::VMEM_SAMPLER_READ_ACCESS |
248 HWEvents::VMEM_BVH_READ_ACCESS | HWEvents::VMEM_INV_ACCESS,
249 HWEvents::SMEM_ACCESS | HWEvents::LDS_ACCESS | HWEvents::GDS_ACCESS |
250 HWEvents::SQ_MESSAGE,
251 HWEvents::EXP_GPR_LOCK | HWEvents::GDS_GPR_LOCK |
252 HWEvents::VMW_GPR_LOCK | HWEvents::EXP_PARAM_ACCESS |
253 HWEvents::EXP_POS_ACCESS | HWEvents::EXP_LDS_ACCESS,
254 HWEvents::VMEM_WRITE_ACCESS | HWEvents::SCRATCH_WRITE_ACCESS,
255 HWEvents::NONE,
256 HWEvents::NONE,
257 HWEvents::NONE,
258 HWEvents::NONE,
259 HWEvents::NONE,
260 HWEvents::NONE,
261 HWEvents::NONE,
262 HWEvents::NONE,
263 HWEvents::NONE};
264
265public:
266 using WaitcntGenerator::WaitcntGenerator;
267 bool applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
268 MachineInstr &OldWaitcntInstr,
269 AMDGPU::Waitcnt &Wait,
270 MachineBasicBlock::instr_iterator It) override;
271
272 bool createNewWaitcnt(MachineBasicBlock &Block,
273 MachineBasicBlock::instr_iterator It,
274 AMDGPU::Waitcnt Wait,
275 const WaitcntBrackets &ScoreBrackets) override;
276
277 HWEvents getWaitEvents(AMDGPU::InstCounterType T) const override {
278 HWEvents EVs = WaitEventMaskForInstPreGFX12[T];
279 if (T == AMDGPU::LOAD_CNT && !ST.hasVscnt())
280 EVs |= WaitEventMaskForInstPreGFX12[AMDGPU::STORE_CNT];
281 return EVs;
282 }
283
284 AMDGPU::Waitcnt getAllZeroWaitcnt(bool IncludeVSCnt) const override;
285};
286
287class WaitcntGeneratorGFX12Plus final : public WaitcntGenerator {
288protected:
289 bool IsExpertMode;
290 static constexpr const HWEvents
291 WaitEventMaskForInstGFX12Plus[AMDGPU::NUM_INST_CNTS] = {
292 HWEvents::VMEM_READ_ACCESS | HWEvents::VMEM_INV_ACCESS,
293 HWEvents::LDS_ACCESS | HWEvents::GDS_ACCESS,
294 HWEvents::EXP_GPR_LOCK | HWEvents::GDS_GPR_LOCK |
295 HWEvents::VMW_GPR_LOCK | HWEvents::EXP_PARAM_ACCESS |
296 HWEvents::EXP_POS_ACCESS | HWEvents::EXP_LDS_ACCESS,
297
298 HWEvents::VMEM_WRITE_ACCESS | HWEvents::SCRATCH_WRITE_ACCESS,
299 HWEvents::VMEM_SAMPLER_READ_ACCESS,
300 HWEvents::VMEM_BVH_READ_ACCESS,
301
302 HWEvents::SMEM_ACCESS | HWEvents::SQ_MESSAGE | HWEvents::SCC_WRITE,
303 HWEvents::VMEM_GROUP | HWEvents::SMEM_GROUP,
304 HWEvents::ASYNC_ACCESS,
305 HWEvents::TENSOR_ACCESS,
306 HWEvents::VGPR_CSMACC_READ | HWEvents::VGPR_DPMACC_READ |
307 HWEvents::VGPR_TRANS_READ | HWEvents::VGPR_XDL_READ,
308 HWEvents::VGPR_CSMACC_WRITE | HWEvents::VGPR_DPMACC_WRITE |
309 HWEvents::VGPR_TRANS_WRITE | HWEvents::VGPR_XDL_WRITE,
310 HWEvents::VGPR_LDS_READ | HWEvents::VGPR_FLAT_READ |
311 HWEvents::VGPR_VMEM_READ};
312
313public:
314 WaitcntGeneratorGFX12Plus() = delete;
315 WaitcntGeneratorGFX12Plus(const MachineFunction &MF,
316 AMDGPU::InstCounterType MaxCounter,
317 const AMDGPU::HardwareLimits &Limits,
318 bool IsExpertMode)
319 : WaitcntGenerator(MF, MaxCounter, Limits), IsExpertMode(IsExpertMode) {}
320
321 bool applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
322 MachineInstr &OldWaitcntInstr,
323 AMDGPU::Waitcnt &Wait,
324 MachineBasicBlock::instr_iterator It) override;
325
326 bool createNewWaitcnt(MachineBasicBlock &Block,
327 MachineBasicBlock::instr_iterator It,
328 AMDGPU::Waitcnt Wait,
329 const WaitcntBrackets &ScoreBrackets) override;
330
331 HWEvents getWaitEvents(AMDGPU::InstCounterType T) const override {
332 return WaitEventMaskForInstGFX12Plus[T];
333 }
334
335 AMDGPU::Waitcnt getAllZeroWaitcnt(bool IncludeVSCnt) const override;
336};
337
338// Flags indicating which counters should be flushed in a loop preheader.
339struct PreheaderFlushFlags {
340 bool FlushVmCnt = false;
341 bool FlushDsCnt = false;
342};
343
344class SIInsertWaitcnts {
345 DenseMap<const Value *, MachineBasicBlock *> SLoadAddresses;
346 DenseMap<MachineBasicBlock *, PreheaderFlushFlags> PreheadersToFlush;
347 MachineLoopInfo &MLI;
348 MachinePostDominatorTree &PDT;
349 AliasAnalysis *AA = nullptr;
350 MachineFunction &MF;
351
352 struct BlockInfo {
353 std::unique_ptr<WaitcntBrackets> Incoming;
354 bool Dirty = true;
355 BlockInfo() = default;
356 BlockInfo(BlockInfo &&) = default;
357 BlockInfo &operator=(BlockInfo &&) = default;
358 ~BlockInfo();
359 };
360
361 MapVector<MachineBasicBlock *, BlockInfo> BlockInfos;
362
363 bool ForceEmitWaitcnt[AMDGPU::NUM_INST_CNTS] = {};
364
365 std::unique_ptr<WaitcntGenerator> WCG;
366
367 // Remember call and return instructions in the function.
368 DenseSet<MachineInstr *> CallInsts;
369 DenseSet<MachineInstr *> ReturnInsts;
370
371 // Remember all S_ENDPGM instructions. The boolean flag is true if there might
372 // be outstanding stores but definitely no outstanding scratch stores, to help
373 // with insertion of DEALLOC_VGPRS messages.
374 DenseMap<MachineInstr *, bool> EndPgmInsts;
375
376 AMDGPU::HardwareLimits Limits;
377
378public:
379 const GCNSubtarget &ST;
380 const SIInstrInfo &TII;
381 const SIRegisterInfo &TRI;
382 const MachineRegisterInfo &MRI;
383 AMDGPU::InstCounterType SmemAccessCounter;
384 AMDGPU::InstCounterType MaxCounter;
385 bool IsExpertMode = false;
386 const bool TgSplit;
387
388 SIInsertWaitcnts(MachineLoopInfo &MLI, MachinePostDominatorTree &PDT,
389 AliasAnalysis *AA, MachineFunction &MF)
390 : MLI(MLI), PDT(PDT), AA(AA), MF(MF), ST(MF.getSubtarget<GCNSubtarget>()),
391 TII(*ST.getInstrInfo()), TRI(TII.getRegisterInfo()),
392 MRI(MF.getRegInfo()),
393 TgSplit(ST.hasTgSplitSupport() &&
394 AMDGPU::isTgSplitEnabled(F: MF.getFunction())) {}
395
396 const AMDGPU::HardwareLimits &getLimits() const { return Limits; }
397
398 PreheaderFlushFlags getPreheaderFlushFlags(MachineLoop *ML,
399 const WaitcntBrackets &Brackets);
400 PreheaderFlushFlags isPreheaderToFlush(MachineBasicBlock &MBB,
401 const WaitcntBrackets &ScoreBrackets);
402 bool isVMEMOrFlatVMEM(const MachineInstr &MI) const;
403 bool isDSRead(const MachineInstr &MI) const;
404 bool mayStoreIncrementingDSCNT(const MachineInstr &MI) const;
405 bool run();
406
407 bool isAsync(const MachineInstr &MI) const {
408 if (!SIInstrInfo::isLDSDMA(MI))
409 return false;
410 if (SIInstrInfo::usesASYNC_CNT(MI))
411 return true;
412 const MachineOperand *Async =
413 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::IsAsync);
414 return Async && (Async->getImm());
415 }
416
417 bool isNonAsyncLdsDmaWrite(const MachineInstr &MI) const {
418 return SIInstrInfo::mayWriteLDSThroughDMA(MI) && !isAsync(MI);
419 }
420
421 bool isAsyncLdsDmaWrite(const MachineInstr &MI) const {
422 return SIInstrInfo::mayWriteLDSThroughDMA(MI) && isAsync(MI);
423 }
424
425 bool shouldUpdateAsyncMark(const MachineInstr &MI,
426 AMDGPU::InstCounterType T) const {
427 if (SIInstrInfo::usesTENSOR_CNT(MI))
428 return T == AMDGPU::TENSOR_CNT;
429 if (!isAsyncLdsDmaWrite(MI))
430 return false;
431 if (SIInstrInfo::usesASYNC_CNT(MI))
432 return T == AMDGPU::ASYNC_CNT;
433 return T == AMDGPU::LOAD_CNT;
434 }
435
436 bool isVmemAccess(const MachineInstr &MI) const;
437 bool generateWaitcntInstBefore(MachineInstr &MI,
438 WaitcntBrackets &ScoreBrackets,
439 MachineInstr *OldWaitcntInstr,
440 PreheaderFlushFlags FlushFlags);
441 bool generateWaitcnt(AMDGPU::Waitcnt Wait,
442 MachineBasicBlock::instr_iterator It,
443 MachineBasicBlock &Block, WaitcntBrackets &ScoreBrackets,
444 MachineInstr *OldWaitcntInstr);
445 void updateEventWaitcntAfter(MachineInstr &Inst,
446 WaitcntBrackets *ScoreBrackets);
447 bool isNextENDPGM(MachineBasicBlock::instr_iterator It,
448 MachineBasicBlock *Block) const;
449 bool insertForcedWaitAfter(MachineInstr &Inst, MachineBasicBlock &Block,
450 WaitcntBrackets &ScoreBrackets);
451 bool insertWaitcntInBlock(MachineFunction &MF, MachineBasicBlock &Block,
452 WaitcntBrackets &ScoreBrackets);
453 /// Removes redundant Soft Xcnt Waitcnts in \p Block emitted by the Memory
454 /// Legalizer. Returns true if block was modified.
455 bool removeRedundantSoftXcnts(MachineBasicBlock &Block);
456 void setSchedulingMode(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
457 bool ExpertMode) const;
458 HWEvents getWaitEvents(AMDGPU::InstCounterType T) const {
459 return WCG->getWaitEvents(T);
460 }
461 AMDGPU::InstCounterType getCounterFromEvent(HWEvents E) const {
462 return WCG->getCounterFromEvent(E);
463 }
464};
465
466// This objects maintains the current score brackets of each wait counter, and
467// a per-register scoreboard for each wait counter.
468//
469// We also maintain the latest score for every event type that can change the
470// waitcnt in order to know if there are multiple types of events within
471// the brackets. When multiple types of event happen in the bracket,
472// wait count may get decreased out of order, therefore we need to put in
473// "s_waitcnt 0" before use.
474class WaitcntBrackets {
475public:
476 WaitcntBrackets(const SIInsertWaitcnts *Context) : Context(Context) {
477 assert(Context->TRI.getNumRegUnits() < REGUNITS_END);
478 }
479
480#ifndef NDEBUG
481 ~WaitcntBrackets() {
482 unsigned NumUnusedVmem = 0, NumUnusedSGPRs = 0;
483 for (auto &[ID, Val] : VMem) {
484 if (Val.empty())
485 ++NumUnusedVmem;
486 }
487 for (auto &[ID, Val] : SGPRs) {
488 if (Val.empty())
489 ++NumUnusedSGPRs;
490 }
491
492 if (NumUnusedVmem || NumUnusedSGPRs) {
493 errs() << "WaitcntBracket had unused entries at destruction time: "
494 << NumUnusedVmem << " VMem and " << NumUnusedSGPRs
495 << " SGPR unused entries\n";
496 std::abort();
497 }
498 }
499#endif
500
501 bool isSmemCounter(AMDGPU::InstCounterType T) const {
502 return T == Context->SmemAccessCounter || T == AMDGPU::X_CNT;
503 }
504
505 unsigned getOutstanding(AMDGPU::InstCounterType T) const {
506 return ScoreUBs[T] - ScoreLBs[T];
507 }
508
509 bool hasPendingVMEM(VMEMID ID, AMDGPU::InstCounterType T) const {
510 return getVMemScore(TID: ID, T) > getScoreLB(T);
511 }
512
513 /// \Return true if we have no score entries for counter \p T.
514 bool empty(AMDGPU::InstCounterType T) const { return getScoreRange(T) == 0; }
515
516private:
517 unsigned getScoreLB(AMDGPU::InstCounterType T) const {
518 assert(T < AMDGPU::NUM_INST_CNTS);
519 return ScoreLBs[T];
520 }
521
522 unsigned getScoreUB(AMDGPU::InstCounterType T) const {
523 assert(T < AMDGPU::NUM_INST_CNTS);
524 return ScoreUBs[T];
525 }
526
527 unsigned getScoreRange(AMDGPU::InstCounterType T) const {
528 return getScoreUB(T) - getScoreLB(T);
529 }
530
531 unsigned getSGPRScore(MCRegUnit RU, AMDGPU::InstCounterType T) const {
532 auto It = SGPRs.find(Val: RU);
533 return It != SGPRs.end() ? It->second.get(T) : 0;
534 }
535
536 unsigned getVMemScore(VMEMID TID, AMDGPU::InstCounterType T) const {
537 auto It = VMem.find(Val: TID);
538 return It != VMem.end() ? It->second.Scores[T] : 0;
539 }
540
541public:
542 // Do some internal consistency checks.
543 void verify() {
544 // These track the same underlying hardware counter so the score ranges
545 // should be identical.
546 assert(getScoreLB(AMDGPU::VA_VDST_RD) == getScoreLB(AMDGPU::VA_VDST_WR));
547 assert(getScoreUB(AMDGPU::VA_VDST_RD) == getScoreUB(AMDGPU::VA_VDST_WR));
548 }
549
550 bool merge(const WaitcntBrackets &Other);
551
552 bool counterOutOfOrder(AMDGPU::InstCounterType T) const;
553 void simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const {
554 simplifyWaitcnt(CheckWait: Wait, UpdateWait&: Wait);
555 }
556 void simplifyWaitcnt(const AMDGPU::Waitcnt &CheckWait,
557 AMDGPU::Waitcnt &UpdateWait) const;
558 void simplifyWaitcnt(AMDGPU::InstCounterType T, unsigned &Count) const;
559 void simplifyWaitcnt(AMDGPU::Waitcnt &Wait, AMDGPU::InstCounterType T) const;
560 void simplifyXcnt(const AMDGPU::Waitcnt &CheckWait,
561 AMDGPU::Waitcnt &UpdateWait) const;
562 void simplifyVmVsrc(const AMDGPU::Waitcnt &CheckWait,
563 AMDGPU::Waitcnt &UpdateWait) const;
564 // GLOBAL_INV/BUFFER_INV increment loadcnt/vmcnt but do not write VGPRs, so
565 // waits that only drain these events can be omitted.
566 void simplifyVmemInvOnlyWait(AMDGPU::Waitcnt &Wait) const {
567 // LOAD_CNT also includes store events on targets without VScnt.
568 HWEvents LoadEvents =
569 PendingEvents & (Context->getWaitEvents(T: AMDGPU::LOAD_CNT) -
570 Context->getWaitEvents(T: AMDGPU::STORE_CNT));
571 if (LoadEvents == HWEvents::VMEM_INV_ACCESS)
572 Wait.clear(T: AMDGPU::LOAD_CNT);
573 }
574
575 void determineWaitForPhysReg(AMDGPU::InstCounterType T, MCRegister Reg,
576 AMDGPU::Waitcnt &Wait,
577 const MachineInstr &MI) const;
578 MCRegister determineVGPR16Dependency(const MachineInstr &MI,
579 AMDGPU::InstCounterType T,
580 MCRegister Reg) const;
581 void determineWaitForLDSDMA(AMDGPU::InstCounterType T, VMEMID TID,
582 AMDGPU::Waitcnt &Wait) const;
583 AMDGPU::Waitcnt determineAsyncWait(unsigned N);
584 void tryClearSCCWriteEvent(MachineInstr *Inst);
585
586 void applyWaitcnt(const AMDGPU::Waitcnt &Wait);
587 void applyWaitcnt(AMDGPU::InstCounterType T, unsigned Count);
588 void applyWaitcnt(const AMDGPU::Waitcnt &Wait, AMDGPU::InstCounterType T);
589 void updateByEvent(HWEvents E, MachineInstr &MI);
590 void recordAsyncMark(MachineInstr &MI);
591
592 HWEvents getPendingEvents() const { return PendingEvents; }
593 bool hasPendingEvent() const { return PendingEvents.any(); }
594 bool hasPendingEvent(HWEvents E) const { return PendingEvents.contains(Other: E); }
595 bool hasPendingEvent(AMDGPU::InstCounterType T) const {
596 bool HasPending = (PendingEvents & Context->getWaitEvents(T)).any();
597 assert(HasPending == !empty(T) &&
598 "Expected pending events iff scoreboard is not empty");
599 return HasPending;
600 }
601
602 bool hasMixedPendingEvents(AMDGPU::InstCounterType T) const {
603 HWEvents Events = PendingEvents & Context->getWaitEvents(T);
604 // Return true if more than one bit is set in Events.
605 return Events.size() > 1;
606 }
607
608 bool hasPendingFlat() const {
609 return ((LastFlatDsCnt > ScoreLBs[AMDGPU::DS_CNT] &&
610 LastFlatDsCnt <= ScoreUBs[AMDGPU::DS_CNT]) ||
611 (LastFlatLoadCnt > ScoreLBs[AMDGPU::LOAD_CNT] &&
612 LastFlatLoadCnt <= ScoreUBs[AMDGPU::LOAD_CNT]));
613 }
614
615 void setPendingFlat() {
616 LastFlatLoadCnt = ScoreUBs[AMDGPU::LOAD_CNT];
617 LastFlatDsCnt = ScoreUBs[AMDGPU::DS_CNT];
618 }
619
620 bool hasPendingGDS() const {
621 return LastGDS > ScoreLBs[AMDGPU::DS_CNT] &&
622 LastGDS <= ScoreUBs[AMDGPU::DS_CNT];
623 }
624
625 unsigned getPendingGDSWait() const {
626 return std::min(a: getScoreUB(T: AMDGPU::DS_CNT) - LastGDS,
627 b: getLimit(T: AMDGPU::DS_CNT) - 1);
628 }
629
630 void setPendingGDS() { LastGDS = ScoreUBs[AMDGPU::DS_CNT]; }
631
632 // Return true if there might be pending writes to the vgpr-interval by VMEM
633 // instructions where the HWEvents in VGPRContext are not contained in E.
634 bool hasDifferentVGPRPendingEvents(MCRegister Reg, HWEvents E) const {
635 for (MCRegUnit RU : regunits(Reg)) {
636 auto It = VMem.find(Val: toVMEMID(RU));
637 if (It != VMem.end() && (It->second.VGPRPendingEvents & ~E).any())
638 return true;
639 }
640 return false;
641 }
642
643 void clearVGPRPendingEvents(MCRegister Reg) {
644 for (MCRegUnit RU : regunits(Reg)) {
645 if (auto It = VMem.find(Val: toVMEMID(RU)); It != VMem.end()) {
646 It->second.VGPRPendingEvents = HWEvents::NONE;
647 if (It->second.empty())
648 VMem.erase(I: It);
649 }
650 }
651 }
652
653 void setStateOnFunctionEntryOrReturn() {
654 setScoreUB(T: AMDGPU::STORE_CNT,
655 Val: getScoreUB(T: AMDGPU::STORE_CNT) + getLimit(T: AMDGPU::STORE_CNT));
656 PendingEvents |= Context->getWaitEvents(T: AMDGPU::STORE_CNT);
657 }
658
659 ArrayRef<const MachineInstr *> getLDSDMAStores() const {
660 return LDSDMAStores;
661 }
662
663 bool hasPointSampleAccel(const MachineInstr &MI) const;
664 bool hasPointSamplePendingVmemTypes(const MachineInstr &MI,
665 MCRegister RU) const;
666
667 void print(raw_ostream &) const;
668 void dump() const { print(dbgs()); }
669
670 // Free up memory by removing empty entries from the DenseMap that track event
671 // scores.
672 void purgeEmptyTrackingData();
673
674private:
675 unsigned getLimit(AMDGPU::InstCounterType T) const {
676 return Context->getLimits().get(T);
677 }
678
679 struct MergeInfo {
680 unsigned OldLB;
681 unsigned OtherLB;
682 unsigned MyShift;
683 unsigned OtherShift;
684 };
685
686 using CounterValueArray = std::array<unsigned, AMDGPU::NUM_INST_CNTS>;
687
688 void determineWaitForScore(AMDGPU::InstCounterType T, unsigned Score,
689 AMDGPU::Waitcnt &Wait) const;
690
691 static bool mergeScore(const MergeInfo &M, unsigned &Score,
692 unsigned OtherScore);
693 bool mergeAsyncMarks(ArrayRef<MergeInfo> MergeInfos,
694 ArrayRef<CounterValueArray> OtherMarks);
695
696 iterator_range<MCRegUnitIterator> regunits(MCRegister Reg) const {
697 assert(Reg != AMDGPU::SCC && "Shouldn't be used on SCC");
698 if (!Context->TRI.isInAllocatableClass(RegNo: Reg))
699 return {{}, {}};
700 return Context->TRI.regunits(Reg);
701 }
702
703 void setScoreLB(AMDGPU::InstCounterType T, unsigned Val) {
704 assert(T < AMDGPU::NUM_INST_CNTS);
705 ScoreLBs[T] = Val;
706 }
707
708 void setScoreUB(AMDGPU::InstCounterType T, unsigned Val) {
709 assert(T < AMDGPU::NUM_INST_CNTS);
710 ScoreUBs[T] = Val;
711
712 if (T != AMDGPU::EXP_CNT)
713 return;
714
715 if (getScoreRange(T: AMDGPU::EXP_CNT) > getLimit(T: AMDGPU::EXP_CNT))
716 ScoreLBs[AMDGPU::EXP_CNT] =
717 ScoreUBs[AMDGPU::EXP_CNT] - getLimit(T: AMDGPU::EXP_CNT);
718 }
719
720 void setRegScore(MCRegister Reg, AMDGPU::InstCounterType T, unsigned Val) {
721 const SIRegisterInfo &TRI = Context->TRI;
722 if (Reg == AMDGPU::SCC) {
723 SCCScore = Val;
724 } else if (TRI.isVectorRegister(MRI: Context->MRI, Reg)) {
725 for (MCRegUnit RU : regunits(Reg))
726 VMem[toVMEMID(RU)].Scores[T] = Val;
727 } else if (TRI.isSGPRReg(MRI: Context->MRI, Reg)) {
728 for (MCRegUnit RU : regunits(Reg))
729 SGPRs[RU].get(T) = Val;
730 } else {
731 llvm_unreachable("Register cannot be tracked/unknown register!");
732 }
733 }
734
735 void setVMemScore(VMEMID TID, AMDGPU::InstCounterType T, unsigned Val) {
736 VMem[TID].Scores[T] = Val;
737 }
738
739 void setScoreByOperand(const MachineOperand &Op,
740 AMDGPU::InstCounterType CntTy, unsigned Val);
741
742 const SIInsertWaitcnts *Context;
743
744 unsigned ScoreLBs[AMDGPU::NUM_INST_CNTS] = {0};
745 unsigned ScoreUBs[AMDGPU::NUM_INST_CNTS] = {0};
746 HWEvents PendingEvents;
747 // Remember the last flat memory operation.
748 unsigned LastFlatDsCnt = 0;
749 unsigned LastFlatLoadCnt = 0;
750 // Remember the last GDS operation.
751 unsigned LastGDS = 0;
752
753 // The score tracking logic is fragmented as follows:
754 // - VMem: VGPR RegUnits and LDS DMA IDs, see the VMEMID encoding.
755 // - SGPRs: SGPR RegUnits
756 // - SCC: Non-allocatable and not general purpose: not a SGPR.
757 //
758 // For the VMem case, if the key is within the range of LDS DMA IDs,
759 // then the corresponding index into the `LDSDMAStores` vector below is:
760 // Key - LDSDMA_BEGIN - 1
761 // This is because LDSDMA_BEGIN is a generic entry and does not have an
762 // associated MachineInstr.
763 //
764 // TODO: Could we track SCC alongside SGPRs so it's not longer a special case?
765
766 struct VMEMInfo {
767 // Scores for all instruction counters. Zero-initialized.
768 CounterValueArray Scores{};
769 // For VGPRs, we need to track an additional fine-grained set of pending
770 // events.
771 HWEvents VGPRPendingEvents;
772
773 bool empty() const {
774 return all_of(Range: Scores, P: equal_to(Arg: 0)) && !VGPRPendingEvents;
775 }
776 };
777
778 /// Wait cnt scores for every sgpr, the DS_CNT (corresponding to LGKMcnt
779 /// pre-gfx12) or KM_CNT (gfx12+ only), and X_CNT (gfx1250) are relevant.
780 class SGPRInfo {
781 /// Either DS_CNT or KM_CNT score.
782 unsigned ScoreDsKmCnt = 0;
783 unsigned ScoreXCnt = 0;
784
785 public:
786 unsigned get(AMDGPU::InstCounterType T) const {
787 assert(
788 (T == AMDGPU::DS_CNT || T == AMDGPU::KM_CNT || T == AMDGPU::X_CNT) &&
789 "Invalid counter");
790 return T == AMDGPU::X_CNT ? ScoreXCnt : ScoreDsKmCnt;
791 }
792 unsigned &get(AMDGPU::InstCounterType T) {
793 assert(
794 (T == AMDGPU::DS_CNT || T == AMDGPU::KM_CNT || T == AMDGPU::X_CNT) &&
795 "Invalid counter");
796 return T == AMDGPU::X_CNT ? ScoreXCnt : ScoreDsKmCnt;
797 }
798
799 bool empty() const { return !ScoreDsKmCnt && !ScoreXCnt; }
800 };
801
802 DenseMap<VMEMID, VMEMInfo> VMem; // VGPR + LDS DMA
803 DenseMap<MCRegUnit, SGPRInfo> SGPRs;
804
805 // Reg score for SCC.
806 unsigned SCCScore = 0;
807 // The unique instruction that has an SCC write pending, if there is one.
808 const MachineInstr *PendingSCCWrite = nullptr;
809
810 // Store representative LDS DMA operations. The only useful info here is
811 // alias info. One store is kept per unique AAInfo.
812 SmallVector<const MachineInstr *> LDSDMAStores;
813
814 // State of all counters at each async mark encountered so far.
815 SmallVector<CounterValueArray> AsyncMarks;
816
817 // But in the rare pathological case, a nest of loops that pushes marks
818 // without waiting on any mark can cause AsyncMarks to grow very large. We cap
819 // it to a reasonable limit. We can tune this later or potentially introduce a
820 // user option to control the value.
821 static constexpr unsigned MaxAsyncMarks = 16;
822
823 // Track the upper bound score for async operations that are not part of a
824 // mark yet. Initialized to all zeros.
825 CounterValueArray AsyncScore{};
826};
827
828SIInsertWaitcnts::BlockInfo::~BlockInfo() = default;
829
830class SIInsertWaitcntsLegacy : public MachineFunctionPass {
831public:
832 static char ID;
833 SIInsertWaitcntsLegacy() : MachineFunctionPass(ID) {}
834
835 bool runOnMachineFunction(MachineFunction &MF) override;
836
837 StringRef getPassName() const override {
838 return "SI insert wait instructions";
839 }
840
841 void getAnalysisUsage(AnalysisUsage &AU) const override {
842 AU.setPreservesCFG();
843 AU.addRequired<MachineLoopInfoWrapperPass>();
844 AU.addRequired<MachinePostDominatorTreeWrapperPass>();
845 AU.addUsedIfAvailable<AAResultsWrapperPass>();
846 AU.addPreserved<AAResultsWrapperPass>();
847 MachineFunctionPass::getAnalysisUsage(AU);
848 }
849};
850
851} // end anonymous namespace
852
853void WaitcntBrackets::setScoreByOperand(const MachineOperand &Op,
854 AMDGPU::InstCounterType CntTy,
855 unsigned Score) {
856 setRegScore(Reg: Op.getReg().asMCReg(), T: CntTy, Val: Score);
857}
858
859// Return true if the subtarget is one that enables Point Sample Acceleration
860// and the MachineInstr passed in is one to which it might be applied (the
861// hardware makes this decision based on several factors, but we can't determine
862// this at compile time, so we have to assume it might be applied if the
863// instruction supports it).
864bool WaitcntBrackets::hasPointSampleAccel(const MachineInstr &MI) const {
865 if (!Context->ST.hasPointSampleAccel() || !SIInstrInfo::isMIMG(MI))
866 return false;
867
868 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc: MI.getOpcode());
869 const AMDGPU::MIMGBaseOpcodeInfo *BaseInfo =
870 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode);
871 return BaseInfo->PointSampleAccel;
872}
873
874// Return true if the subtarget enables Point Sample Acceleration, the supplied
875// MachineInstr is one to which it might be applied and the supplied interval is
876// one that has outstanding writes to vmem-types different than VMEM_NOSAMPLER
877// (this is the type that a point sample accelerated instruction effectively
878// becomes)
879bool WaitcntBrackets::hasPointSamplePendingVmemTypes(const MachineInstr &MI,
880 MCRegister Reg) const {
881 if (!hasPointSampleAccel(MI))
882 return false;
883
884 return hasDifferentVGPRPendingEvents(Reg, E: HWEvents::VMEM_READ_ACCESS);
885}
886
887void WaitcntBrackets::updateByEvent(HWEvents E, MachineInstr &Inst) {
888 assert(E.size() == 1 && "Expected singular event!");
889 AMDGPU::InstCounterType T = Context->getCounterFromEvent(E);
890 assert(T < Context->MaxCounter);
891
892 unsigned UB = getScoreUB(T);
893 unsigned Increment = 1;
894 if ((T == AMDGPU::VA_VDST_RD || T == AMDGPU::VA_VDST_WR) &&
895 AMDGPU::getHasMatrixScale(Opc: Inst.getOpcode()) &&
896 Context->ST.hasVOP3PX2IncrementsVaVdstTwice()) {
897 // V_WMMA_SCALE instructions use VOP3PX2 encoding. Hardware treats this as
898 // two VOP3P instructions and increments VA_VDST twice.
899 Increment = 2;
900 }
901 unsigned CurrScore = UB + Increment;
902 if (CurrScore == 0)
903 report_fatal_error(reason: "InsertWaitcnt score wraparound");
904 // PendingEvents and ScoreUB need to be update regardless if this event
905 // changes the score of a register or not.
906 // Examples including vm_cnt when buffer-store or lgkm_cnt when send-message.
907 PendingEvents |= E;
908 setScoreUB(T, Val: CurrScore);
909
910 const SIRegisterInfo &TRI = Context->TRI;
911 const MachineRegisterInfo &MRI = Context->MRI;
912 const SIInstrInfo &TII = Context->TII;
913
914 if (T == AMDGPU::EXP_CNT) {
915 // Put score on the source vgprs. If this is a store, just use those
916 // specific register(s).
917 if (TII.isDS(MI: Inst) && Inst.mayLoadOrStore()) {
918 // All GDS operations must protect their address register (same as
919 // export.)
920 if (const auto *AddrOp = TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::addr))
921 setScoreByOperand(Op: *AddrOp, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
922
923 if (Inst.mayStore()) {
924 if (const auto *Data0 =
925 TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data0))
926 setScoreByOperand(Op: *Data0, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
927 if (const auto *Data1 =
928 TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data1))
929 setScoreByOperand(Op: *Data1, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
930 } else if (SIInstrInfo::isAtomicRet(MI: Inst) && !SIInstrInfo::isGWS(MI: Inst) &&
931 Inst.getOpcode() != AMDGPU::DS_APPEND &&
932 Inst.getOpcode() != AMDGPU::DS_CONSUME &&
933 Inst.getOpcode() != AMDGPU::DS_ORDERED_COUNT) {
934 for (const MachineOperand &Op : Inst.all_uses()) {
935 if (TRI.isVectorRegister(MRI, Reg: Op.getReg()))
936 setScoreByOperand(Op, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
937 }
938 }
939 } else if (TII.isFLAT(MI: Inst)) {
940 if (Inst.mayStore()) {
941 setScoreByOperand(Op: *TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data),
942 CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
943 } else if (SIInstrInfo::isAtomicRet(MI: Inst)) {
944 setScoreByOperand(Op: *TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data),
945 CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
946 }
947 } else if (TII.isMIMG(MI: Inst)) {
948 if (Inst.mayStore()) {
949 setScoreByOperand(Op: Inst.getOperand(i: 0), CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
950 } else if (SIInstrInfo::isAtomicRet(MI: Inst)) {
951 setScoreByOperand(Op: *TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data),
952 CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
953 }
954 } else if (TII.isMTBUF(MI: Inst)) {
955 if (Inst.mayStore())
956 setScoreByOperand(Op: Inst.getOperand(i: 0), CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
957 } else if (TII.isMUBUF(MI: Inst)) {
958 if (Inst.mayStore()) {
959 setScoreByOperand(Op: Inst.getOperand(i: 0), CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
960 } else if (SIInstrInfo::isAtomicRet(MI: Inst)) {
961 setScoreByOperand(Op: *TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data),
962 CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
963 }
964 } else if (TII.isLDSDIR(MI: Inst)) {
965 // LDSDIR instructions attach the score to the destination.
966 setScoreByOperand(Op: *TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::vdst),
967 CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
968 } else {
969 if (TII.isEXP(MI: Inst)) {
970 // For export the destination registers are really temps that
971 // can be used as the actual source after export patching, so
972 // we need to treat them like sources and set the EXP_CNT
973 // score.
974 for (MachineOperand &DefMO : Inst.all_defs()) {
975 if (TRI.isVGPR(MRI, Reg: DefMO.getReg())) {
976 setScoreByOperand(Op: DefMO, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
977 }
978 }
979 }
980 for (const MachineOperand &Op : Inst.all_uses()) {
981 if (TRI.isVectorRegister(MRI, Reg: Op.getReg()))
982 setScoreByOperand(Op, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
983 }
984 }
985 } else if (T == AMDGPU::X_CNT) {
986 HWEvents OtherEvent =
987 E == HWEvents::SMEM_GROUP ? HWEvents::VMEM_GROUP : HWEvents::SMEM_GROUP;
988 if (PendingEvents.contains(Other: OtherEvent)) {
989 // Hardware inserts an implicit xcnt between interleaved
990 // SMEM and VMEM operations. So there will never be
991 // outstanding address translations for both SMEM and
992 // VMEM at the same time.
993 setScoreLB(T, Val: getScoreUB(T) - 1);
994 PendingEvents -= OtherEvent;
995 }
996 for (const MachineOperand &Op : Inst.all_uses())
997 setScoreByOperand(Op, CntTy: T, Score: CurrScore);
998 } else if (T == AMDGPU::VA_VDST_RD || T == AMDGPU::VA_VDST_WR ||
999 T == AMDGPU::VM_VSRC) {
1000 // Match the score to the VGPR destination or source registers as
1001 // appropriate
1002 for (const MachineOperand &Op : Inst.operands()) {
1003 if (!Op.isReg())
1004 continue;
1005
1006 // Skip based on counter type and operand type
1007 if (T == AMDGPU::VA_VDST_RD && Op.isDef())
1008 continue; // RD tracks reads only
1009 if (T == AMDGPU::VA_VDST_WR && Op.isUse())
1010 continue; // WR tracks writes only
1011 if (T == AMDGPU::VM_VSRC && Op.isDef())
1012 continue;
1013
1014 if (TRI.isVectorRegister(MRI: Context->MRI, Reg: Op.getReg()))
1015 setScoreByOperand(Op, CntTy: T, Score: CurrScore);
1016 }
1017 } else /* LGKM_CNT || EXP_CNT || VS_CNT || NUM_INST_CNTS */ {
1018 // Match the score to the destination registers.
1019 //
1020 // Check only explicit operands. Stores, especially spill stores, include
1021 // implicit uses and defs of their super registers which would create an
1022 // artificial dependency, while these are there only for register liveness
1023 // accounting purposes.
1024 //
1025 // Special cases where implicit register defs exists, such as M0 or VCC,
1026 // but none with memory instructions.
1027 for (const MachineOperand &Op : Inst.defs()) {
1028 if (T == AMDGPU::LOAD_CNT || T == AMDGPU::SAMPLE_CNT ||
1029 T == AMDGPU::BVH_CNT) {
1030 if (!TRI.isVectorRegister(MRI, Reg: Op.getReg())) // TODO: add wrapper
1031 continue;
1032 if (updateVMCntOnly(Inst)) {
1033 // updateVMCntOnly should only leave us with VGPRs
1034 // MUBUF, MTBUF, MIMG, FlatGlobal, and FlatScratch only have VGPR/AGPR
1035 // defs.
1036 assert(TRI.isVectorRegister(MRI, Op.getReg()));
1037 HWEvents VGPRContext =
1038 AMDGPU::getSimplifiedVMEMEventsFor(Inst, TII: Context->TII);
1039 // If instruction can have Point Sample Accel applied, we have to flag
1040 // this with another potential dependency
1041 if (hasPointSampleAccel(MI: Inst))
1042 VGPRContext |= HWEvents::VMEM_READ_ACCESS;
1043 for (MCRegUnit RU : regunits(Reg: Op.getReg().asMCReg()))
1044 VMem[toVMEMID(RU)].VGPRPendingEvents |= VGPRContext;
1045 }
1046 }
1047 setScoreByOperand(Op, CntTy: T, Score: CurrScore);
1048 }
1049 if (Inst.mayStore() &&
1050 (TII.isDS(MI: Inst) || Context->isNonAsyncLdsDmaWrite(MI: Inst))) {
1051 // MUBUF and FLAT LDS DMA operations need a wait on vmcnt before LDS
1052 // written can be accessed. A load from LDS to VMEM does not need a wait.
1053 //
1054 // The "Slot" is the offset from LDSDMA_BEGIN. If it's non-zero, then
1055 // there is a MachineInstr in LDSDMAStores used to track this LDSDMA
1056 // store. The "Slot" is the index into LDSDMAStores + 1.
1057 size_t Slot = 0;
1058 for (const auto *MemOp : Inst.memoperands()) {
1059 if (!MemOp->isStore() ||
1060 MemOp->getAddrSpace() != AMDGPUAS::LOCAL_ADDRESS)
1061 continue;
1062 // Comparing just AA info does not guarantee memoperands are equal
1063 // in general, but this is so for LDS DMA in practice.
1064 auto AAI = MemOp->getAAInfo();
1065 // Alias scope information gives a way to definitely identify an
1066 // original memory object and practically produced in the module LDS
1067 // lowering pass. If there is no scope available we will not be able
1068 // to disambiguate LDS aliasing as after the module lowering all LDS
1069 // is squashed into a single big object.
1070 if (!AAI || !AAI.Scope)
1071 break;
1072 for (size_t I = 0, E = LDSDMAStores.size(); I != E && !Slot; ++I) {
1073 for (const auto *MemOp : LDSDMAStores[I]->memoperands()) {
1074 if (MemOp->isStore() && AAI == MemOp->getAAInfo()) {
1075 Slot = I + 1;
1076 break;
1077 }
1078 }
1079 }
1080 if (Slot)
1081 break;
1082 // The slot may not be valid because it can be >= NUM_LDSDMA which
1083 // means the scoreboard cannot track it. We still want to preserve the
1084 // MI in order to check alias information, though.
1085 LDSDMAStores.push_back(Elt: &Inst);
1086 Slot = LDSDMAStores.size();
1087 break;
1088 }
1089 setVMemScore(TID: LDSDMA_BEGIN, T, Val: CurrScore);
1090 if (Slot && Slot < NUM_LDSDMA)
1091 setVMemScore(TID: LDSDMA_BEGIN + static_cast<VMEMID>(Slot), T, Val: CurrScore);
1092 }
1093
1094 if (Context->shouldUpdateAsyncMark(MI: Inst, T)) {
1095 AsyncScore[T] = CurrScore;
1096 }
1097
1098 if (SIInstrInfo::isSBarrierSCCWrite(Opcode: Inst.getOpcode())) {
1099 setRegScore(Reg: AMDGPU::SCC, T, Val: CurrScore);
1100 PendingSCCWrite = &Inst;
1101 }
1102 }
1103}
1104
1105void WaitcntBrackets::recordAsyncMark(MachineInstr &Inst) {
1106 // In the absence of loops, AsyncMarks can grow linearly with the program
1107 // until we encounter an ASYNCMARK_WAIT. We could drop the oldest mark above a
1108 // limit every time we push a new mark, but that seems like unnecessary work
1109 // in practical cases. We do separately truncate the array when processing a
1110 // loop, which should be sufficient.
1111 AsyncMarks.push_back(Elt: AsyncScore);
1112 LLVM_DEBUG({
1113 dbgs() << "recordAsyncMark:\n" << Inst;
1114 for (const auto &Mark : AsyncMarks) {
1115 llvm::interleaveComma(Mark, dbgs());
1116 dbgs() << '\n';
1117 }
1118 });
1119}
1120
1121void WaitcntBrackets::print(raw_ostream &OS) const {
1122 const GCNSubtarget &ST = Context->ST;
1123
1124 for (auto T : inst_counter_types(MaxCounter: Context->MaxCounter)) {
1125 unsigned SR = getScoreRange(T);
1126 switch (T) {
1127 case AMDGPU::LOAD_CNT:
1128 OS << " " << (ST.hasExtendedWaitCounts() ? "LOAD" : "VM") << "_CNT("
1129 << SR << "):";
1130 break;
1131 case AMDGPU::DS_CNT:
1132 OS << " " << (ST.hasExtendedWaitCounts() ? "DS" : "LGKM") << "_CNT("
1133 << SR << "):";
1134 break;
1135 case AMDGPU::EXP_CNT:
1136 OS << " EXP_CNT(" << SR << "):";
1137 break;
1138 case AMDGPU::STORE_CNT:
1139 OS << " " << (ST.hasExtendedWaitCounts() ? "STORE" : "VS") << "_CNT("
1140 << SR << "):";
1141 break;
1142 case AMDGPU::SAMPLE_CNT:
1143 OS << " SAMPLE_CNT(" << SR << "):";
1144 break;
1145 case AMDGPU::BVH_CNT:
1146 OS << " BVH_CNT(" << SR << "):";
1147 break;
1148 case AMDGPU::KM_CNT:
1149 OS << " KM_CNT(" << SR << "):";
1150 break;
1151 case AMDGPU::X_CNT:
1152 OS << " X_CNT(" << SR << "):";
1153 break;
1154 case AMDGPU::ASYNC_CNT:
1155 OS << " ASYNC_CNT(" << SR << "):";
1156 break;
1157 case AMDGPU::VA_VDST_RD:
1158 OS << " VA_VDST_RD(" << SR << "): ";
1159 break;
1160 case AMDGPU::VA_VDST_WR:
1161 OS << " VA_VDST_WR(" << SR << "): ";
1162 break;
1163 case AMDGPU::VM_VSRC:
1164 OS << " VM_VSRC(" << SR << "): ";
1165 break;
1166 default:
1167 OS << " UNKNOWN(" << SR << "):";
1168 break;
1169 }
1170
1171 if (SR != 0) {
1172 // Print vgpr scores.
1173 unsigned LB = getScoreLB(T);
1174
1175 SmallVector<VMEMID> SortedVMEMIDs(VMem.keys());
1176 sort(C&: SortedVMEMIDs);
1177
1178 for (auto ID : SortedVMEMIDs) {
1179 unsigned RegScore = VMem.at(Val: ID).Scores[T];
1180 if (RegScore <= LB)
1181 continue;
1182 unsigned RelScore = RegScore - LB - 1;
1183 if (ID < REGUNITS_END) {
1184 OS << ' ' << RelScore << ':'
1185 << printRegUnit(Unit: static_cast<MCRegUnit>(ID), TRI: &Context->TRI);
1186 } else {
1187 assert(ID >= LDSDMA_BEGIN && ID < LDSDMA_END &&
1188 "Unhandled/unexpected ID value!");
1189 OS << ' ' << RelScore << ":LDSDMA" << ID;
1190 }
1191 }
1192
1193 // Also need to print sgpr scores for lgkm_cnt or xcnt.
1194 if (isSmemCounter(T)) {
1195 SmallVector<MCRegUnit> SortedSMEMIDs(SGPRs.keys());
1196 sort(C&: SortedSMEMIDs);
1197 for (auto ID : SortedSMEMIDs) {
1198 unsigned RegScore = SGPRs.at(Val: ID).get(T);
1199 if (RegScore <= LB)
1200 continue;
1201 unsigned RelScore = RegScore - LB - 1;
1202 OS << ' ' << RelScore << ':'
1203 << printRegUnit(Unit: static_cast<MCRegUnit>(ID), TRI: &Context->TRI);
1204 }
1205 }
1206
1207 if (T == AMDGPU::KM_CNT && SCCScore > 0)
1208 OS << ' ' << SCCScore << ":scc";
1209 }
1210 OS << '\n';
1211 }
1212
1213 OS << "Pending Events: ";
1214 if (hasPendingEvent()) {
1215 OS << getPendingEvents();
1216 } else {
1217 OS << "none";
1218 }
1219 OS << '\n';
1220
1221 OS << "Async score: ";
1222 if (AsyncScore.empty())
1223 OS << "none";
1224 else
1225 llvm::interleaveComma(c: AsyncScore, os&: OS);
1226 OS << '\n';
1227
1228 OS << "Async marks: " << AsyncMarks.size() << '\n';
1229
1230 for (const auto &Mark : AsyncMarks) {
1231 for (auto T : AMDGPU::inst_counter_types()) {
1232 unsigned MarkedScore = Mark[T];
1233 switch (T) {
1234 case AMDGPU::LOAD_CNT:
1235 OS << " " << (ST.hasExtendedWaitCounts() ? "LOAD" : "VM")
1236 << "_CNT: " << MarkedScore;
1237 break;
1238 case AMDGPU::DS_CNT:
1239 OS << " " << (ST.hasExtendedWaitCounts() ? "DS" : "LGKM")
1240 << "_CNT: " << MarkedScore;
1241 break;
1242 case AMDGPU::EXP_CNT:
1243 OS << " EXP_CNT: " << MarkedScore;
1244 break;
1245 case AMDGPU::STORE_CNT:
1246 OS << " " << (ST.hasExtendedWaitCounts() ? "STORE" : "VS")
1247 << "_CNT: " << MarkedScore;
1248 break;
1249 case AMDGPU::SAMPLE_CNT:
1250 OS << " SAMPLE_CNT: " << MarkedScore;
1251 break;
1252 case AMDGPU::BVH_CNT:
1253 OS << " BVH_CNT: " << MarkedScore;
1254 break;
1255 case AMDGPU::KM_CNT:
1256 OS << " KM_CNT: " << MarkedScore;
1257 break;
1258 case AMDGPU::X_CNT:
1259 OS << " X_CNT: " << MarkedScore;
1260 break;
1261 case AMDGPU::ASYNC_CNT:
1262 OS << " ASYNC_CNT: " << MarkedScore;
1263 break;
1264 default:
1265 OS << " UNKNOWN: " << MarkedScore;
1266 break;
1267 }
1268 }
1269 OS << '\n';
1270 }
1271 OS << '\n';
1272}
1273
1274/// Simplify \p UpdateWait by removing waits that are redundant based on the
1275/// current WaitcntBrackets and any other waits specified in \p CheckWait.
1276void WaitcntBrackets::simplifyWaitcnt(const AMDGPU::Waitcnt &CheckWait,
1277 AMDGPU::Waitcnt &UpdateWait) const {
1278 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::LOAD_CNT);
1279 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::EXP_CNT);
1280 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::DS_CNT);
1281 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::STORE_CNT);
1282 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::SAMPLE_CNT);
1283 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::BVH_CNT);
1284 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::KM_CNT);
1285 simplifyXcnt(CheckWait, UpdateWait);
1286 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::VA_VDST_RD);
1287 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::VA_VDST_WR);
1288 simplifyVmVsrc(CheckWait, UpdateWait);
1289 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::ASYNC_CNT);
1290}
1291
1292void WaitcntBrackets::simplifyWaitcnt(AMDGPU::InstCounterType T,
1293 unsigned &Count) const {
1294 // The number of outstanding events for this type, T, can be calculated
1295 // as (UB - LB). If the current Count is greater than or equal to the number
1296 // of outstanding events, then the wait for this counter is redundant.
1297 if (Count >= getScoreRange(T))
1298 Count = ~0u;
1299}
1300
1301void WaitcntBrackets::simplifyWaitcnt(AMDGPU::Waitcnt &Wait,
1302 AMDGPU::InstCounterType T) const {
1303 unsigned Cnt = Wait.get(T);
1304 simplifyWaitcnt(T, Count&: Cnt);
1305 Wait.set(T, Val: Cnt);
1306}
1307
1308void WaitcntBrackets::simplifyXcnt(const AMDGPU::Waitcnt &CheckWait,
1309 AMDGPU::Waitcnt &UpdateWait) const {
1310 // Try to simplify xcnt further by checking for joint kmcnt and loadcnt
1311 // optimizations. On entry to a block with multiple predescessors, there may
1312 // be pending SMEM and VMEM events active at the same time.
1313 // In such cases, only clear one active event at a time.
1314 // TODO: Revisit xcnt optimizations for gfx1250.
1315 // Wait on XCNT is redundant if we are already waiting for a load to complete.
1316 // SMEM can return out of order, so only omit XCNT wait if we are waiting till
1317 // zero.
1318 if (CheckWait.get(T: AMDGPU::KM_CNT) == 0 &&
1319 hasPendingEvent(E: HWEvents::SMEM_GROUP))
1320 UpdateWait.set(T: AMDGPU::X_CNT, Val: ~0u);
1321 // If we have pending store we cannot optimize XCnt because we do not wait for
1322 // stores. VMEM loads retun in order, so if we only have loads XCnt is
1323 // decremented to the same number as LOADCnt.
1324 if (CheckWait.get(T: AMDGPU::LOAD_CNT) != ~0u &&
1325 hasPendingEvent(E: HWEvents::VMEM_GROUP) &&
1326 !hasPendingEvent(T: AMDGPU::STORE_CNT) &&
1327 CheckWait.get(T: AMDGPU::X_CNT) >= CheckWait.get(T: AMDGPU::LOAD_CNT))
1328 UpdateWait.set(T: AMDGPU::X_CNT, Val: ~0u);
1329 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::X_CNT);
1330}
1331
1332void WaitcntBrackets::simplifyVmVsrc(const AMDGPU::Waitcnt &CheckWait,
1333 AMDGPU::Waitcnt &UpdateWait) const {
1334 // Waiting for a VMEM counter (like LOAD_CNT) implies an equivalent wait for
1335 // VM_VSRC, because if the VMEM operation has completed then it must surely
1336 // have read its VGPR sources, but only if there are no other outstanding VMEM
1337 // operations that use a different counter (like SAMPLE_CNT).
1338 static constexpr AMDGPU::InstCounterType VmemCounters[] = {
1339 AMDGPU::LOAD_CNT, AMDGPU::STORE_CNT, AMDGPU::SAMPLE_CNT, AMDGPU::BVH_CNT,
1340 AMDGPU::DS_CNT, AMDGPU::ASYNC_CNT};
1341 HWEvents VmemEvents = llvm::accumulate(
1342 Range: VmemCounters, Init: HWEvents(), Op: [&](HWEvents Acc, AMDGPU::InstCounterType T) {
1343 return Acc | Context->getWaitEvents(T);
1344 });
1345 HWEvents PendingVmemEvents = PendingEvents & VmemEvents;
1346 for (AMDGPU::InstCounterType T : VmemCounters) {
1347 unsigned CheckCount = CheckWait.get(T);
1348 if (UpdateWait.get(T: AMDGPU::VM_VSRC) >= CheckCount &&
1349 (CheckCount == 0 || !counterOutOfOrder(T)) &&
1350 (PendingVmemEvents & ~Context->getWaitEvents(T)) == 0)
1351 UpdateWait.set(T: AMDGPU::VM_VSRC, Val: ~0u);
1352 }
1353
1354 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::VM_VSRC);
1355}
1356
1357void WaitcntBrackets::purgeEmptyTrackingData() {
1358 VMem.remove_if(Pred: [](const auto &P) { return P.second.empty(); });
1359 SGPRs.remove_if(Pred: [](const auto &P) { return P.second.empty(); });
1360}
1361
1362void WaitcntBrackets::determineWaitForScore(AMDGPU::InstCounterType T,
1363 unsigned ScoreToWait,
1364 AMDGPU::Waitcnt &Wait) const {
1365 const unsigned LB = getScoreLB(T);
1366 const unsigned UB = getScoreUB(T);
1367
1368 // If the score falls within the bracket, we need a waitcnt.
1369 if ((UB >= ScoreToWait) && (ScoreToWait > LB)) {
1370 if ((T == AMDGPU::LOAD_CNT || T == AMDGPU::DS_CNT) && hasPendingFlat() &&
1371 !Context->ST.hasFlatLgkmVMemCountInOrder()) {
1372 // If there is a pending FLAT operation, and this is a VMem or LGKM
1373 // waitcnt and the target can report early completion, then we need
1374 // to force a waitcnt 0.
1375 Wait.add(T, Count: 0);
1376 } else if (counterOutOfOrder(T)) {
1377 // Counter can get decremented out-of-order when there
1378 // are multiple types event in the bracket. Also emit an s_wait counter
1379 // with a conservative value of 0 for the counter.
1380 Wait.add(T, Count: 0);
1381 } else {
1382 // If a counter has been maxed out avoid overflow by waiting for
1383 // MAX(CounterType) - 1 instead.
1384 unsigned NeededWait = std::min(a: UB - ScoreToWait, b: getLimit(T) - 1);
1385 Wait.add(T, Count: NeededWait);
1386 }
1387 }
1388}
1389
1390AMDGPU::Waitcnt WaitcntBrackets::determineAsyncWait(unsigned N) {
1391 LLVM_DEBUG({
1392 dbgs() << "Need " << N << " async marks. Found " << AsyncMarks.size()
1393 << ":\n";
1394 for (const auto &Mark : AsyncMarks) {
1395 llvm::interleaveComma(Mark, dbgs());
1396 dbgs() << '\n';
1397 }
1398 });
1399
1400 if (AsyncMarks.size() == MaxAsyncMarks) {
1401 // Enforcing MaxAsyncMarks here is unnecessary work because the size of
1402 // MaxAsyncMarks is linear when traversing straightline code. But we do
1403 // need to check if truncation may have occured at a merge, and adjust N
1404 // to ensure that a wait is generated.
1405 LLVM_DEBUG(dbgs() << "Possible truncation. Ensuring a non-trivial wait.\n");
1406 N = std::min(a: N, b: (unsigned)MaxAsyncMarks - 1);
1407 }
1408
1409 AMDGPU::Waitcnt Wait;
1410 if (AsyncMarks.size() <= N) {
1411 LLVM_DEBUG(dbgs() << "No additional wait for async mark.\n");
1412 return Wait;
1413 }
1414
1415 size_t MarkIndex = AsyncMarks.size() - N - 1;
1416 const auto &RequiredMark = AsyncMarks[MarkIndex];
1417 for (AMDGPU::InstCounterType T : AMDGPU::inst_counter_types())
1418 determineWaitForScore(T, ScoreToWait: RequiredMark[T], Wait);
1419
1420 // Immediately remove the waited mark and all older ones
1421 // This happens BEFORE the wait is actually inserted, which is fine
1422 // because we've already extracted the wait requirements
1423 LLVM_DEBUG({
1424 dbgs() << "Removing " << (MarkIndex + 1)
1425 << " async marks after determining wait\n";
1426 });
1427 AsyncMarks.erase(CS: AsyncMarks.begin(), CE: AsyncMarks.begin() + MarkIndex + 1);
1428
1429 LLVM_DEBUG(dbgs() << "Waits to add: " << Wait);
1430 return Wait;
1431}
1432
1433// With D16Write32BitVgpr, D16 inst might be clobbered by events running on the
1434// other half 16bit.
1435//
1436// Replace VGPR16 to VGPR32 for wait check if:
1437// 1. MI is a VALU, and there is a wait event on the other half
1438// 2. MI is a LdSt, and there is a wait event on the other half from different
1439// order group
1440MCRegister WaitcntBrackets::determineVGPR16Dependency(const MachineInstr &MI,
1441 AMDGPU::InstCounterType T,
1442 MCRegister Reg) const {
1443 const TargetRegisterClass *RC = Context->TRI.getPhysRegBaseClass(Reg);
1444 uint64_t Size = Context->TRI.getRegSizeInBits(RC: *RC);
1445
1446 if (Size != 16 || !Context->ST.hasD16Writes32BitVgpr())
1447 return Reg;
1448
1449 // With D16Writes32BitVgpr, D16 Inst might clobber the whole vgpr32
1450 // check dependency on the other half
1451 Register Reg32 = Context->TRI.get32BitRegister(Reg);
1452 Register OtherHalf = Context->TRI.getSubReg(
1453 Reg: Reg32,
1454 Idx: AMDGPU::isHi16Reg(Reg, MRI: Context->TRI) ? AMDGPU::lo16 : AMDGPU::hi16);
1455
1456 AMDGPU::Waitcnt Wait;
1457 for (MCRegUnit RU : regunits(Reg: OtherHalf))
1458 determineWaitForScore(T, ScoreToWait: getVMemScore(TID: toVMEMID(RU), T), Wait);
1459
1460 // No wait on otherhalf
1461 if (!Wait.hasWait())
1462 return Reg;
1463
1464 if (Context->TII.isVALU(MI, /*AllowLDSDMA=*/false))
1465 return Reg32;
1466
1467 // If hi/lo16 mixed events
1468 HWEvents MIEvents = AMDGPU::getEventsFor(
1469 Inst: MI, ST: Context->ST, IsExpertMode: Context->IsExpertMode, TgSplit: Context->TgSplit);
1470 HWEvents OtherHalfEvents = Context->getWaitEvents(T);
1471 HWEvents Events = MIEvents & OtherHalfEvents;
1472 if (Events.size() > 1)
1473 return Reg32;
1474 return Reg;
1475}
1476
1477void WaitcntBrackets::determineWaitForPhysReg(AMDGPU::InstCounterType T,
1478 MCRegister Reg,
1479 AMDGPU::Waitcnt &Wait,
1480 const MachineInstr &MI) const {
1481 if (Reg == AMDGPU::SCC) {
1482 determineWaitForScore(T, ScoreToWait: SCCScore, Wait);
1483 } else {
1484 bool IsVGPR = Context->TRI.isVectorRegister(MRI: Context->MRI, Reg);
1485 if (IsVGPR)
1486 Reg = determineVGPR16Dependency(MI, T, Reg);
1487 for (MCRegUnit RU : regunits(Reg))
1488 determineWaitForScore(
1489 T, ScoreToWait: IsVGPR ? getVMemScore(TID: toVMEMID(RU), T) : getSGPRScore(RU, T),
1490 Wait);
1491 }
1492}
1493
1494void WaitcntBrackets::determineWaitForLDSDMA(AMDGPU::InstCounterType T,
1495 VMEMID TID,
1496 AMDGPU::Waitcnt &Wait) const {
1497 assert(TID >= LDSDMA_BEGIN && TID < LDSDMA_END);
1498 determineWaitForScore(T, ScoreToWait: getVMemScore(TID, T), Wait);
1499}
1500
1501void WaitcntBrackets::tryClearSCCWriteEvent(MachineInstr *Inst) {
1502 // S_BARRIER_WAIT on the same barrier guarantees that the pending write to
1503 // SCC has landed
1504 if (PendingSCCWrite &&
1505 PendingSCCWrite->getOpcode() == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM &&
1506 PendingSCCWrite->getOperand(i: 0).getImm() == Inst->getOperand(i: 0).getImm()) {
1507 HWEvents SCC_WRITE_PendingEvent = HWEvents::SCC_WRITE;
1508 // If this SCC_WRITE is the only pending KM_CNT event, clear counter.
1509 if ((PendingEvents & Context->getWaitEvents(T: AMDGPU::KM_CNT)) ==
1510 SCC_WRITE_PendingEvent) {
1511 setScoreLB(T: AMDGPU::KM_CNT, Val: getScoreUB(T: AMDGPU::KM_CNT));
1512 }
1513
1514 PendingEvents -= SCC_WRITE_PendingEvent;
1515 PendingSCCWrite = nullptr;
1516 }
1517}
1518
1519void WaitcntBrackets::applyWaitcnt(const AMDGPU::Waitcnt &Wait) {
1520 for (AMDGPU::InstCounterType T : AMDGPU::inst_counter_types()) {
1521 unsigned Cnt;
1522 if (T == AMDGPU::VA_VDST_RD || T == AMDGPU::VA_VDST_WR) {
1523 Cnt =
1524 std::min(a: Wait.get(T: AMDGPU::VA_VDST_RD), b: Wait.get(T: AMDGPU::VA_VDST_WR));
1525 } else {
1526 Cnt = Wait.get(T);
1527 }
1528 applyWaitcnt(T, Count: Cnt);
1529 }
1530}
1531
1532void WaitcntBrackets::applyWaitcnt(AMDGPU::InstCounterType T, unsigned Count) {
1533 const unsigned UB = getScoreUB(T);
1534 if (Count >= UB)
1535 return;
1536 if (Count != 0) {
1537 if (counterOutOfOrder(T))
1538 return;
1539 setScoreLB(T, Val: std::max(a: getScoreLB(T), b: UB - Count));
1540 } else {
1541 setScoreLB(T, Val: UB);
1542 PendingEvents -= Context->getWaitEvents(T);
1543 }
1544
1545 if (T == AMDGPU::KM_CNT && Count == 0 &&
1546 hasPendingEvent(E: HWEvents::SMEM_GROUP)) {
1547 if (!hasMixedPendingEvents(T: AMDGPU::X_CNT))
1548 applyWaitcnt(T: AMDGPU::X_CNT, Count: 0);
1549 else
1550 PendingEvents -= HWEvents::SMEM_GROUP;
1551 }
1552 if (T == AMDGPU::LOAD_CNT && hasPendingEvent(E: HWEvents::VMEM_GROUP) &&
1553 !hasPendingEvent(T: AMDGPU::STORE_CNT)) {
1554 if (!hasMixedPendingEvents(T: AMDGPU::X_CNT))
1555 applyWaitcnt(T: AMDGPU::X_CNT, Count);
1556 else if (Count == 0)
1557 PendingEvents -= HWEvents::VMEM_GROUP;
1558 }
1559}
1560
1561void WaitcntBrackets::applyWaitcnt(const AMDGPU::Waitcnt &Wait,
1562 AMDGPU::InstCounterType T) {
1563 unsigned Cnt = Wait.get(T);
1564 applyWaitcnt(T, Count: Cnt);
1565}
1566
1567// Where there are multiple types of event in the bracket of a counter,
1568// the decrement may go out of order.
1569bool WaitcntBrackets::counterOutOfOrder(AMDGPU::InstCounterType T) const {
1570 // Scalar memory read always can go out of order.
1571 if ((T == Context->SmemAccessCounter &&
1572 hasPendingEvent(E: HWEvents::SMEM_ACCESS)) ||
1573 (T == AMDGPU::X_CNT && hasPendingEvent(E: HWEvents::SMEM_GROUP)))
1574 return true;
1575
1576 if (T == AMDGPU::LOAD_CNT) {
1577
1578 // On targets without VScnt, LOAD_CNT includes all of STORE_CNT as well.
1579 // All these events use one counter and do not go out of order with respect
1580 // to each other.
1581 if (!Context->ST.hasVscnt())
1582 return false;
1583
1584 HWEvents Events = PendingEvents & Context->getWaitEvents(T);
1585
1586 // If the target does not have extended counters, VMEM_BVH/SAMPLE_READ
1587 // events are equivalent to VMEM_READ_ACCESS. We do not go out of order in
1588 // such cases.
1589 static constexpr HWEvents ExtendedImageEvents =
1590 HWEvents::VMEM_SAMPLER_READ_ACCESS | HWEvents::VMEM_BVH_READ_ACCESS;
1591 if (!Context->ST.hasExtendedWaitCounts() &&
1592 (Events & ExtendedImageEvents).any()) {
1593 Events -= ExtendedImageEvents;
1594 Events |= HWEvents::VMEM_READ_ACCESS;
1595 }
1596
1597 // GLOBAL_INV/BUFFER_INV complete in-order with VMEM_READ_ACCESS, so mixing
1598 // VMEM_INV_ACCESS with other LOAD_CNT events does not cause out-of-order
1599 // completion.
1600 Events -= HWEvents::VMEM_INV_ACCESS;
1601
1602 return Events.size() > 1;
1603 }
1604
1605 return hasMixedPendingEvents(T);
1606}
1607
1608INITIALIZE_PASS_BEGIN(SIInsertWaitcntsLegacy, DEBUG_TYPE, "SI Insert Waitcnts",
1609 false, false)
1610INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass)
1611INITIALIZE_PASS_DEPENDENCY(MachinePostDominatorTreeWrapperPass)
1612INITIALIZE_PASS_END(SIInsertWaitcntsLegacy, DEBUG_TYPE, "SI Insert Waitcnts",
1613 false, false)
1614
1615char SIInsertWaitcntsLegacy::ID = 0;
1616
1617char &llvm::SIInsertWaitcntsID = SIInsertWaitcntsLegacy::ID;
1618
1619FunctionPass *llvm::createSIInsertWaitcntsPass() {
1620 return new SIInsertWaitcntsLegacy();
1621}
1622
1623static bool updateOperandIfDifferent(MachineInstr &MI, AMDGPU::OpName OpName,
1624 unsigned NewEnc) {
1625 int OpIdx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpName);
1626 assert(OpIdx >= 0);
1627
1628 MachineOperand &MO = MI.getOperand(i: OpIdx);
1629
1630 if (NewEnc == MO.getImm())
1631 return false;
1632
1633 MO.setImm(NewEnc);
1634 return true;
1635}
1636
1637bool WaitcntGenerator::promoteSoftWaitCnt(MachineInstr *Waitcnt) const {
1638 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode: Waitcnt->getOpcode());
1639 if (Opcode == Waitcnt->getOpcode())
1640 return false;
1641
1642 Waitcnt->setDesc(TII.get(Opcode));
1643 return true;
1644}
1645
1646/// Combine consecutive S_WAITCNT and S_WAITCNT_VSCNT instructions that
1647/// precede \p It and follow \p OldWaitcntInstr and apply any extra waits
1648/// from \p Wait that were added by previous passes. Currently this pass
1649/// conservatively assumes that these preexisting waits are required for
1650/// correctness.
1651bool WaitcntGeneratorPreGFX12::applyPreexistingWaitcnt(
1652 WaitcntBrackets &ScoreBrackets, MachineInstr &OldWaitcntInstr,
1653 AMDGPU::Waitcnt &Wait, MachineBasicBlock::instr_iterator It) {
1654 assert(isNormalMode(MaxCounter));
1655
1656 bool Modified = false;
1657 MachineInstr *WaitcntInstr = nullptr;
1658 MachineInstr *WaitcntVsCntInstr = nullptr;
1659
1660 LLVM_DEBUG({
1661 dbgs() << "PreGFX12::applyPreexistingWaitcnt at: ";
1662 if (It.isEnd())
1663 dbgs() << "end of block\n";
1664 else
1665 dbgs() << *It;
1666 });
1667
1668 for (auto &II :
1669 make_early_inc_range(Range: make_range(x: OldWaitcntInstr.getIterator(), y: It))) {
1670 LLVM_DEBUG(dbgs() << "pre-existing iter: " << II);
1671 if (isNonWaitcntMetaInst(MI: II)) {
1672 LLVM_DEBUG(dbgs() << "skipped meta instruction\n");
1673 continue;
1674 }
1675
1676 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode: II.getOpcode());
1677 bool TrySimplify = Opcode != II.getOpcode() && !OptNone;
1678
1679 // Update required wait count. If this is a soft waitcnt (= it was added
1680 // by an earlier pass), it may be entirely removed.
1681 if (Opcode == AMDGPU::S_WAITCNT) {
1682 unsigned IEnc = static_cast<unsigned>(II.getOperand(i: 0).getImm());
1683 AMDGPU::Waitcnt OldWait = AMDGPU::decodeWaitcnt(Version: IV, Encoded: IEnc);
1684 if (TrySimplify) {
1685 ScoreBrackets.simplifyWaitcnt(Wait&: OldWait);
1686 ScoreBrackets.simplifyVmemInvOnlyWait(Wait&: OldWait);
1687 }
1688 Wait = Wait.combined(Other: OldWait);
1689
1690 // Merge consecutive waitcnt of the same type by erasing multiples.
1691 if (WaitcntInstr) {
1692 DeferredErasures.erase(V: &II);
1693 II.eraseFromParent();
1694 Modified = true;
1695 } else if (!Wait.hasWaitExceptStoreCnt() && TrySimplify) {
1696 // Might or might not be redundant, depending on if there's a loop
1697 // carried dependency. Stash it away so we can erase it if nothing saves
1698 // it.
1699 DeferredErasures.insert(V: &II);
1700 } else {
1701 WaitcntInstr = &II;
1702 DeferredErasures.erase(V: &II);
1703 }
1704 } else if (Opcode == AMDGPU::S_WAITCNT_lds_direct) {
1705 assert(ST.hasVMemToLDSLoad());
1706 LLVM_DEBUG(dbgs() << "Processing S_WAITCNT_lds_direct: " << II
1707 << "Before: " << Wait << '\n';);
1708 ScoreBrackets.determineWaitForLDSDMA(T: AMDGPU::LOAD_CNT, TID: LDSDMA_BEGIN,
1709 Wait);
1710 LLVM_DEBUG(dbgs() << "After: " << Wait << '\n';);
1711
1712 // It is possible (but unlikely) that this is the only wait instruction,
1713 // in which case, we exit this loop without a WaitcntInstr to consume
1714 // `Wait`. But that works because `Wait` was passed in by reference, and
1715 // the callee eventually calls createNewWaitcnt on it. We test this
1716 // possibility in an articial MIR test since such a situation cannot be
1717 // recreated by running the memory legalizer.
1718 II.eraseFromParent();
1719 } else if (Opcode == AMDGPU::WAIT_ASYNCMARK) {
1720 unsigned N = static_cast<unsigned>(II.getOperand(i: 0).getImm());
1721 LLVM_DEBUG(dbgs() << "Processing WAIT_ASYNCMARK: " << II << '\n';);
1722 AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(N);
1723 Wait = Wait.combined(Other: OldWait);
1724 } else {
1725 assert(Opcode == AMDGPU::S_WAITCNT_VSCNT);
1726 assert(II.getOperand(0).getReg() == AMDGPU::SGPR_NULL);
1727
1728 unsigned OldVSCnt = static_cast<unsigned>(
1729 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm());
1730 if (TrySimplify)
1731 ScoreBrackets.simplifyWaitcnt(T: AMDGPU::STORE_CNT, Count&: OldVSCnt);
1732 Wait.set(T: AMDGPU::STORE_CNT,
1733 Val: std::min(a: Wait.get(T: AMDGPU::STORE_CNT), b: OldVSCnt));
1734
1735 if (WaitcntVsCntInstr) {
1736 DeferredErasures.erase(V: &II);
1737 II.eraseFromParent();
1738 Modified = true;
1739 } else if (!Wait.hasWaitStoreCnt() && TrySimplify) {
1740 // Loop-carried dependencies guard, see S_WAITCNT case above.
1741 DeferredErasures.insert(V: &II);
1742 } else {
1743 WaitcntVsCntInstr = &II;
1744 DeferredErasures.erase(V: &II);
1745 }
1746 }
1747 }
1748
1749 if (WaitcntInstr) {
1750 Modified |= updateOperandIfDifferent(MI&: *WaitcntInstr, OpName: AMDGPU::OpName::simm16,
1751 NewEnc: AMDGPU::encodeWaitcnt(Version: IV, Decoded: Wait));
1752 Modified |= promoteSoftWaitCnt(Waitcnt: WaitcntInstr);
1753
1754 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::LOAD_CNT);
1755 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::EXP_CNT);
1756 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::DS_CNT);
1757 Wait.set(T: AMDGPU::LOAD_CNT, Val: ~0u);
1758 Wait.set(T: AMDGPU::EXP_CNT, Val: ~0u);
1759 Wait.set(T: AMDGPU::DS_CNT, Val: ~0u);
1760
1761 LLVM_DEBUG(It.isEnd() ? dbgs() << "applied pre-existing waitcnt\n"
1762 << "New Instr at block end: "
1763 << *WaitcntInstr << '\n'
1764 : dbgs() << "applied pre-existing waitcnt\n"
1765 << "Old Instr: " << *It
1766 << "New Instr: " << *WaitcntInstr << '\n');
1767 }
1768
1769 if (WaitcntVsCntInstr) {
1770 Modified |=
1771 updateOperandIfDifferent(MI&: *WaitcntVsCntInstr, OpName: AMDGPU::OpName::simm16,
1772 NewEnc: Wait.get(T: AMDGPU::STORE_CNT));
1773 Modified |= promoteSoftWaitCnt(Waitcnt: WaitcntVsCntInstr);
1774
1775 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::STORE_CNT);
1776 Wait.set(T: AMDGPU::STORE_CNT, Val: ~0u);
1777
1778 LLVM_DEBUG(It.isEnd()
1779 ? dbgs() << "applied pre-existing waitcnt\n"
1780 << "New Instr at block end: " << *WaitcntVsCntInstr
1781 << '\n'
1782 : dbgs() << "applied pre-existing waitcnt\n"
1783 << "Old Instr: " << *It
1784 << "New Instr: " << *WaitcntVsCntInstr << '\n');
1785 }
1786
1787 return Modified;
1788}
1789
1790/// Generate S_WAITCNT and/or S_WAITCNT_VSCNT instructions for any
1791/// required counters in \p Wait
1792bool WaitcntGeneratorPreGFX12::createNewWaitcnt(
1793 MachineBasicBlock &Block, MachineBasicBlock::instr_iterator It,
1794 AMDGPU::Waitcnt Wait, const WaitcntBrackets &ScoreBrackets) {
1795 assert(isNormalMode(MaxCounter));
1796
1797 bool Modified = false;
1798 const DebugLoc &DL = Block.findDebugLoc(MBBI: It);
1799
1800 // Waits for VMcnt, LKGMcnt and/or EXPcnt are encoded together into a
1801 // single instruction while VScnt has its own instruction.
1802 if (Wait.hasWaitExceptStoreCnt()) {
1803 // If profiling expansion is enabled, emit an expanded sequence
1804 if (ExpandWaitcntProfiling) {
1805 // Check if any of the counters to be waited on are out-of-order.
1806 // If so, fall back to normal (non-expanded) behavior since expansion
1807 // would provide misleading profiling information.
1808 bool AnyOutOfOrder = false;
1809 for (auto CT : {AMDGPU::LOAD_CNT, AMDGPU::DS_CNT, AMDGPU::EXP_CNT}) {
1810 unsigned WaitCnt = Wait.get(T: CT);
1811 if (WaitCnt != ~0u && ScoreBrackets.counterOutOfOrder(T: CT)) {
1812 AnyOutOfOrder = true;
1813 break;
1814 }
1815 }
1816
1817 if (AnyOutOfOrder) {
1818 // Fall back to non-expanded wait
1819 unsigned Enc = AMDGPU::encodeWaitcnt(Version: IV, Decoded: Wait);
1820 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT)).addImm(Val: Enc);
1821 Modified = true;
1822 } else {
1823 // All counters are in-order, safe to expand
1824 for (auto CT : {AMDGPU::LOAD_CNT, AMDGPU::DS_CNT, AMDGPU::EXP_CNT}) {
1825 unsigned WaitCnt = Wait.get(T: CT);
1826 if (WaitCnt == ~0u)
1827 continue;
1828
1829 unsigned Outstanding =
1830 std::min(a: ScoreBrackets.getOutstanding(T: CT), b: getLimit(E: CT) - 1);
1831 EmitExpandedWaitcnt(Outstanding, Target: WaitCnt, EmitWaitcnt: [&](unsigned Count) {
1832 AMDGPU::Waitcnt W;
1833 W.set(T: CT, Val: Count);
1834 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT))
1835 .addImm(Val: AMDGPU::encodeWaitcnt(Version: IV, Decoded: W));
1836 });
1837 Modified = true;
1838 }
1839 }
1840 } else {
1841 // Normal behavior: emit single combined waitcnt
1842 unsigned Enc = AMDGPU::encodeWaitcnt(Version: IV, Decoded: Wait);
1843 [[maybe_unused]] auto SWaitInst =
1844 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT)).addImm(Val: Enc);
1845 Modified = true;
1846
1847 LLVM_DEBUG(dbgs() << "PreGFX12::createNewWaitcnt\n";
1848 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
1849 dbgs() << "New Instr: " << *SWaitInst << '\n');
1850 }
1851 }
1852
1853 if (Wait.hasWaitStoreCnt()) {
1854 assert(ST.hasVscnt());
1855
1856 if (ExpandWaitcntProfiling && Wait.get(T: AMDGPU::STORE_CNT) != ~0u &&
1857 !ScoreBrackets.counterOutOfOrder(T: AMDGPU::STORE_CNT)) {
1858 // Only expand if counter is not out-of-order
1859 unsigned Outstanding =
1860 std::min(a: ScoreBrackets.getOutstanding(T: AMDGPU::STORE_CNT),
1861 b: getLimit(E: AMDGPU::STORE_CNT) - 1);
1862 EmitExpandedWaitcnt(
1863 Outstanding, Target: Wait.get(T: AMDGPU::STORE_CNT), EmitWaitcnt: [&](unsigned Count) {
1864 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_VSCNT))
1865 .addReg(RegNo: AMDGPU::SGPR_NULL, Flags: RegState::Undef)
1866 .addImm(Val: Count);
1867 });
1868 Modified = true;
1869 } else {
1870 [[maybe_unused]] auto SWaitInst =
1871 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_VSCNT))
1872 .addReg(RegNo: AMDGPU::SGPR_NULL, Flags: RegState::Undef)
1873 .addImm(Val: Wait.get(T: AMDGPU::STORE_CNT));
1874 Modified = true;
1875
1876 LLVM_DEBUG(dbgs() << "PreGFX12::createNewWaitcnt\n";
1877 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
1878 dbgs() << "New Instr: " << *SWaitInst << '\n');
1879 }
1880 }
1881
1882 return Modified;
1883}
1884
1885AMDGPU::Waitcnt
1886WaitcntGeneratorPreGFX12::getAllZeroWaitcnt(bool IncludeVSCnt) const {
1887 return AMDGPU::Waitcnt(0, 0, 0, IncludeVSCnt && ST.hasVscnt() ? 0 : ~0u);
1888}
1889
1890AMDGPU::Waitcnt
1891WaitcntGeneratorGFX12Plus::getAllZeroWaitcnt(bool IncludeVSCnt) const {
1892 unsigned ExpertVal = IsExpertMode ? 0 : ~0u;
1893 return AMDGPU::Waitcnt(0, 0, 0, IncludeVSCnt ? 0 : ~0u, 0, 0, 0,
1894 ~0u /* XCNT */, ~0u /* ASYNC_CNT */,
1895 ~0u /* TENSOR_CNT */, ExpertVal, ExpertVal, ExpertVal);
1896}
1897
1898/// Combine consecutive S_WAIT_*CNT instructions that precede \p It and
1899/// follow \p OldWaitcntInstr and apply any extra waits from \p Wait that
1900/// were added by previous passes. Currently this pass conservatively
1901/// assumes that these preexisting waits are required for correctness.
1902bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
1903 WaitcntBrackets &ScoreBrackets, MachineInstr &OldWaitcntInstr,
1904 AMDGPU::Waitcnt &Wait, MachineBasicBlock::instr_iterator It) {
1905 assert(!isNormalMode(MaxCounter));
1906
1907 bool Modified = false;
1908 MachineInstr *CombinedLoadDsCntInstr = nullptr;
1909 MachineInstr *CombinedStoreDsCntInstr = nullptr;
1910 MachineInstr *WaitcntDepctrInstr = nullptr;
1911 MachineInstr *WaitInstrs[AMDGPU::NUM_EXTENDED_INST_CNTS] = {};
1912
1913 LLVM_DEBUG({
1914 dbgs() << "GFX12Plus::applyPreexistingWaitcnt at: ";
1915 if (It.isEnd())
1916 dbgs() << "end of block\n";
1917 else
1918 dbgs() << *It;
1919 });
1920
1921 // Accumulate waits that should not be simplified.
1922 AMDGPU::Waitcnt RequiredWait;
1923
1924 for (auto &II :
1925 make_early_inc_range(Range: make_range(x: OldWaitcntInstr.getIterator(), y: It))) {
1926 LLVM_DEBUG(dbgs() << "pre-existing iter: " << II);
1927 if (isNonWaitcntMetaInst(MI: II)) {
1928 LLVM_DEBUG(dbgs() << "skipped meta instruction\n");
1929 continue;
1930 }
1931
1932 // Update required wait count. If this is a soft waitcnt (= it was added
1933 // by an earlier pass), it may be entirely removed.
1934
1935 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode: II.getOpcode());
1936 bool TrySimplify = Opcode != II.getOpcode() && !OptNone;
1937
1938 // Don't crash if the programmer used legacy waitcnt intrinsics, but don't
1939 // attempt to do more than that either.
1940 if (Opcode == AMDGPU::S_WAITCNT)
1941 continue;
1942
1943 if (Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT) {
1944 unsigned OldEnc = static_cast<unsigned>(
1945 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm());
1946 AMDGPU::Waitcnt OldWait = AMDGPU::decodeLoadcntDscnt(Version: IV, LoadcntDscnt: OldEnc);
1947 if (TrySimplify)
1948 Wait = Wait.combined(Other: OldWait);
1949 else
1950 RequiredWait = RequiredWait.combined(Other: OldWait);
1951 // Keep the first wait_loadcnt, erase the rest.
1952 if (CombinedLoadDsCntInstr == nullptr) {
1953 CombinedLoadDsCntInstr = &II;
1954 } else {
1955 II.eraseFromParent();
1956 Modified = true;
1957 }
1958 } else if (Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT) {
1959 unsigned OldEnc = static_cast<unsigned>(
1960 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm());
1961 AMDGPU::Waitcnt OldWait = AMDGPU::decodeStorecntDscnt(Version: IV, StorecntDscnt: OldEnc);
1962 if (TrySimplify)
1963 Wait = Wait.combined(Other: OldWait);
1964 else
1965 RequiredWait = RequiredWait.combined(Other: OldWait);
1966 // Keep the first wait_storecnt, erase the rest.
1967 if (CombinedStoreDsCntInstr == nullptr) {
1968 CombinedStoreDsCntInstr = &II;
1969 } else {
1970 II.eraseFromParent();
1971 Modified = true;
1972 }
1973 } else if (Opcode == AMDGPU::S_WAITCNT_DEPCTR) {
1974 unsigned OldEnc = static_cast<unsigned>(
1975 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm());
1976 AMDGPU::Waitcnt OldWait;
1977 // Set both counters to the decoded value from the single hardware field
1978 unsigned VaVdst = AMDGPU::DepCtr::decodeFieldVaVdst(Encoded: OldEnc);
1979 OldWait.set(T: AMDGPU::VA_VDST_RD, Val: VaVdst);
1980 OldWait.set(T: AMDGPU::VA_VDST_WR, Val: VaVdst);
1981 OldWait.set(T: AMDGPU::VM_VSRC, Val: AMDGPU::DepCtr::decodeFieldVmVsrc(Encoded: OldEnc));
1982 if (TrySimplify)
1983 ScoreBrackets.simplifyWaitcnt(Wait&: OldWait);
1984 Wait = Wait.combined(Other: OldWait);
1985 if (WaitcntDepctrInstr == nullptr) {
1986 WaitcntDepctrInstr = &II;
1987 } else {
1988 // S_WAITCNT_DEPCTR requires special care. Don't remove a
1989 // duplicate if it is waiting on things other than VA_VDST or
1990 // VM_VSRC. If that is the case, just make sure the VA_VDST and
1991 // VM_VSRC subfields of the operand are set to the "no wait"
1992 // values.
1993
1994 unsigned Enc = static_cast<unsigned>(
1995 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm());
1996 Enc = AMDGPU::DepCtr::encodeFieldVmVsrc(Encoded: Enc, VmVsrc: ~0u);
1997 // Encode min(VA_VDST_RD, VA_VDST_WR) into the single hardware field
1998 unsigned VaVdst = std::min(a: Wait.get(T: AMDGPU::VA_VDST_RD),
1999 b: Wait.get(T: AMDGPU::VA_VDST_WR));
2000 Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Encoded: Enc, VaVdst);
2001
2002 if (Enc != (unsigned)AMDGPU::DepCtr::getDefaultDepCtrEncoding(STI: ST)) {
2003 Modified |= updateOperandIfDifferent(MI&: II, OpName: AMDGPU::OpName::simm16, NewEnc: Enc);
2004 Modified |= promoteSoftWaitCnt(Waitcnt: &II);
2005 } else {
2006 II.eraseFromParent();
2007 Modified = true;
2008 }
2009 }
2010 } else if (Opcode == AMDGPU::S_WAITCNT_lds_direct) {
2011 // Architectures higher than GFX10 do not have direct loads to
2012 // LDS, so no work required here yet.
2013 II.eraseFromParent();
2014 Modified = true;
2015 } else if (Opcode == AMDGPU::WAIT_ASYNCMARK) {
2016 // Update the Waitcnt, but don't erase the wait.asyncmark() itself. It
2017 // shows up in the assembly as a comment with the original parameter N.
2018 unsigned N = static_cast<unsigned>(II.getOperand(i: 0).getImm());
2019 AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(N);
2020 Wait = Wait.combined(Other: OldWait);
2021 } else {
2022 std::optional<AMDGPU::InstCounterType> CT =
2023 AMDGPU::counterTypeForInstr(Opcode);
2024 assert(CT.has_value());
2025 unsigned OldCnt = static_cast<unsigned>(
2026 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm());
2027 if (TrySimplify)
2028 Wait.add(T: CT.value(), Count: OldCnt);
2029 else
2030 RequiredWait.add(T: CT.value(), Count: OldCnt);
2031 // Keep the first wait of its kind, erase the rest.
2032 if (WaitInstrs[CT.value()] == nullptr) {
2033 WaitInstrs[CT.value()] = &II;
2034 } else {
2035 II.eraseFromParent();
2036 Modified = true;
2037 }
2038 }
2039 }
2040
2041 ScoreBrackets.simplifyWaitcnt(CheckWait: Wait.combined(Other: RequiredWait), UpdateWait&: Wait);
2042 Wait = Wait.combined(Other: RequiredWait);
2043
2044 if (CombinedLoadDsCntInstr) {
2045 // Only keep an S_WAIT_LOADCNT_DSCNT if both counters actually need
2046 // to be waited for. Otherwise, let the instruction be deleted so
2047 // the appropriate single counter wait instruction can be inserted
2048 // instead, when new S_WAIT_*CNT instructions are inserted by
2049 // createNewWaitcnt(). As a side effect, resetting the wait counts will
2050 // cause any redundant S_WAIT_LOADCNT or S_WAIT_DSCNT to be removed by
2051 // the loop below that deals with single counter instructions.
2052 //
2053 // A wait for LOAD_CNT or DS_CNT implies a wait for VM_VSRC, since
2054 // instructions that have decremented LOAD_CNT or DS_CNT on completion
2055 // will have needed to wait for their register sources to be available
2056 // first.
2057 if (Wait.get(T: AMDGPU::LOAD_CNT) != ~0u && Wait.get(T: AMDGPU::DS_CNT) != ~0u) {
2058 unsigned NewEnc = AMDGPU::encodeLoadcntDscnt(Version: IV, Decoded: Wait);
2059 Modified |= updateOperandIfDifferent(MI&: *CombinedLoadDsCntInstr,
2060 OpName: AMDGPU::OpName::simm16, NewEnc);
2061 Modified |= promoteSoftWaitCnt(Waitcnt: CombinedLoadDsCntInstr);
2062 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::LOAD_CNT);
2063 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::DS_CNT);
2064 Wait.set(T: AMDGPU::LOAD_CNT, Val: ~0u);
2065 Wait.set(T: AMDGPU::DS_CNT, Val: ~0u);
2066
2067 LLVM_DEBUG(It.isEnd() ? dbgs() << "applied pre-existing waitcnt\n"
2068 << "New Instr at block end: "
2069 << *CombinedLoadDsCntInstr << '\n'
2070 : dbgs() << "applied pre-existing waitcnt\n"
2071 << "Old Instr: " << *It << "New Instr: "
2072 << *CombinedLoadDsCntInstr << '\n');
2073 } else {
2074 CombinedLoadDsCntInstr->eraseFromParent();
2075 Modified = true;
2076 }
2077 }
2078
2079 if (CombinedStoreDsCntInstr) {
2080 // Similarly for S_WAIT_STORECNT_DSCNT.
2081 if (Wait.get(T: AMDGPU::STORE_CNT) != ~0u && Wait.get(T: AMDGPU::DS_CNT) != ~0u) {
2082 unsigned NewEnc = AMDGPU::encodeStorecntDscnt(Version: IV, Decoded: Wait);
2083 Modified |= updateOperandIfDifferent(MI&: *CombinedStoreDsCntInstr,
2084 OpName: AMDGPU::OpName::simm16, NewEnc);
2085 Modified |= promoteSoftWaitCnt(Waitcnt: CombinedStoreDsCntInstr);
2086 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::STORE_CNT);
2087 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::DS_CNT);
2088 Wait.set(T: AMDGPU::STORE_CNT, Val: ~0u);
2089 Wait.set(T: AMDGPU::DS_CNT, Val: ~0u);
2090
2091 LLVM_DEBUG(It.isEnd() ? dbgs() << "applied pre-existing waitcnt\n"
2092 << "New Instr at block end: "
2093 << *CombinedStoreDsCntInstr << '\n'
2094 : dbgs() << "applied pre-existing waitcnt\n"
2095 << "Old Instr: " << *It << "New Instr: "
2096 << *CombinedStoreDsCntInstr << '\n');
2097 } else {
2098 CombinedStoreDsCntInstr->eraseFromParent();
2099 Modified = true;
2100 }
2101 }
2102
2103 // Look for an opportunity to convert existing S_WAIT_LOADCNT,
2104 // S_WAIT_STORECNT and S_WAIT_DSCNT into new S_WAIT_LOADCNT_DSCNT
2105 // or S_WAIT_STORECNT_DSCNT. This is achieved by selectively removing
2106 // instructions so that createNewWaitcnt() will create new combined
2107 // instructions to replace them.
2108
2109 if (Wait.get(T: AMDGPU::DS_CNT) != ~0u) {
2110 // This is a vector of addresses in WaitInstrs pointing to instructions
2111 // that should be removed if they are present.
2112 SmallVector<MachineInstr **, 2> WaitsToErase;
2113
2114 // If it's known that both DScnt and either LOADcnt or STOREcnt (but not
2115 // both) need to be waited for, ensure that there are no existing
2116 // individual wait count instructions for these.
2117
2118 if (Wait.get(T: AMDGPU::LOAD_CNT) != ~0u) {
2119 WaitsToErase.push_back(Elt: &WaitInstrs[AMDGPU::LOAD_CNT]);
2120 WaitsToErase.push_back(Elt: &WaitInstrs[AMDGPU::DS_CNT]);
2121 } else if (Wait.get(T: AMDGPU::STORE_CNT) != ~0u) {
2122 WaitsToErase.push_back(Elt: &WaitInstrs[AMDGPU::STORE_CNT]);
2123 WaitsToErase.push_back(Elt: &WaitInstrs[AMDGPU::DS_CNT]);
2124 }
2125
2126 for (MachineInstr **WI : WaitsToErase) {
2127 if (!*WI)
2128 continue;
2129
2130 DeferredErasures.erase(V: *WI);
2131 (*WI)->eraseFromParent();
2132 *WI = nullptr;
2133 Modified = true;
2134 }
2135 }
2136
2137 for (auto CT : inst_counter_types(MaxCounter: AMDGPU::NUM_EXTENDED_INST_CNTS)) {
2138 if (!WaitInstrs[CT])
2139 continue;
2140
2141 unsigned NewCnt = Wait.get(T: CT);
2142 if (NewCnt != ~0u) {
2143 Modified |= updateOperandIfDifferent(MI&: *WaitInstrs[CT],
2144 OpName: AMDGPU::OpName::simm16, NewEnc: NewCnt);
2145 Modified |= promoteSoftWaitCnt(Waitcnt: WaitInstrs[CT]);
2146 DeferredErasures.erase(V: WaitInstrs[CT]);
2147
2148 ScoreBrackets.applyWaitcnt(T: CT, Count: NewCnt);
2149 Wait.clear(T: CT);
2150
2151 LLVM_DEBUG(It.isEnd()
2152 ? dbgs() << "applied pre-existing waitcnt\n"
2153 << "New Instr at block end: " << *WaitInstrs[CT]
2154 << '\n'
2155 : dbgs() << "applied pre-existing waitcnt\n"
2156 << "Old Instr: " << *It
2157 << "New Instr: " << *WaitInstrs[CT] << '\n');
2158 } else {
2159 // Defer erasure in case there's a loop-carried dependency.
2160 DeferredErasures.insert(V: WaitInstrs[CT]);
2161 }
2162 }
2163
2164 if (WaitcntDepctrInstr) {
2165 // Get the encoded Depctr immediate and override the VA_VDST and VM_VSRC
2166 // subfields with the new required values.
2167 unsigned Enc = static_cast<unsigned>(
2168 TII.getNamedOperand(MI&: *WaitcntDepctrInstr, OperandName: AMDGPU::OpName::simm16)
2169 ->getImm());
2170 Enc = AMDGPU::DepCtr::encodeFieldVmVsrc(Encoded: Enc, VmVsrc: Wait.get(T: AMDGPU::VM_VSRC));
2171 // Encode min(VA_VDST_RD, VA_VDST_WR) into the single hardware field
2172 unsigned VaVdst =
2173 std::min(a: Wait.get(T: AMDGPU::VA_VDST_RD), b: Wait.get(T: AMDGPU::VA_VDST_WR));
2174 Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Encoded: Enc, VaVdst);
2175
2176 ScoreBrackets.applyWaitcnt(T: AMDGPU::VA_VDST_RD, Count: VaVdst);
2177 ScoreBrackets.applyWaitcnt(T: AMDGPU::VA_VDST_WR, Count: VaVdst);
2178 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::VM_VSRC);
2179 Wait.set(T: AMDGPU::VA_VDST_RD, Val: ~0u);
2180 Wait.set(T: AMDGPU::VA_VDST_WR, Val: ~0u);
2181 Wait.set(T: AMDGPU::VM_VSRC, Val: ~0u);
2182
2183 // If that new encoded Depctr immediate would actually still wait
2184 // for anything, update the instruction's operand. Otherwise it can
2185 // just be deleted.
2186 if (Enc != (unsigned)AMDGPU::DepCtr::getDefaultDepCtrEncoding(STI: ST)) {
2187 Modified |= updateOperandIfDifferent(MI&: *WaitcntDepctrInstr,
2188 OpName: AMDGPU::OpName::simm16, NewEnc: Enc);
2189 LLVM_DEBUG(It.isEnd() ? dbgs() << "applyPreexistingWaitcnt\n"
2190 << "New Instr at block end: "
2191 << *WaitcntDepctrInstr << '\n'
2192 : dbgs() << "applyPreexistingWaitcnt\n"
2193 << "Old Instr: " << *It << "New Instr: "
2194 << *WaitcntDepctrInstr << '\n');
2195 } else {
2196 WaitcntDepctrInstr->eraseFromParent();
2197 Modified = true;
2198 }
2199 }
2200
2201 return Modified;
2202}
2203
2204/// Generate S_WAIT_*CNT instructions for any required counters in \p Wait
2205bool WaitcntGeneratorGFX12Plus::createNewWaitcnt(
2206 MachineBasicBlock &Block, MachineBasicBlock::instr_iterator It,
2207 AMDGPU::Waitcnt Wait, const WaitcntBrackets &ScoreBrackets) {
2208 assert(!isNormalMode(MaxCounter));
2209
2210 bool Modified = false;
2211 const DebugLoc &DL = Block.findDebugLoc(MBBI: It);
2212
2213 // For GFX12+, we use separate wait instructions, which makes expansion
2214 // simpler
2215 if (ExpandWaitcntProfiling) {
2216 for (auto CT : inst_counter_types(MaxCounter: AMDGPU::NUM_EXTENDED_INST_CNTS)) {
2217 unsigned Count = Wait.get(T: CT);
2218 if (Count == ~0u)
2219 continue;
2220
2221 // Skip expansion for out-of-order counters - emit normal wait instead
2222 if (ScoreBrackets.counterOutOfOrder(T: CT)) {
2223 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: instrsForExtendedCounterTypes[CT]))
2224 .addImm(Val: Count);
2225 Modified = true;
2226 continue;
2227 }
2228
2229 unsigned Outstanding =
2230 std::min(a: ScoreBrackets.getOutstanding(T: CT), b: getLimit(E: CT) - 1);
2231 EmitExpandedWaitcnt(Outstanding, Target: Count, EmitWaitcnt: [&](unsigned Val) {
2232 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: instrsForExtendedCounterTypes[CT]))
2233 .addImm(Val);
2234 });
2235 Modified = true;
2236 }
2237 return Modified;
2238 }
2239
2240 // Normal behavior (no expansion)
2241 // Check for opportunities to use combined wait instructions.
2242 if (Wait.get(T: AMDGPU::DS_CNT) != ~0u) {
2243 MachineInstr *SWaitInst = nullptr;
2244
2245 if (Wait.get(T: AMDGPU::LOAD_CNT) != ~0u) {
2246 unsigned Enc = AMDGPU::encodeLoadcntDscnt(Version: IV, Decoded: Wait);
2247
2248 SWaitInst = BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAIT_LOADCNT_DSCNT))
2249 .addImm(Val: Enc);
2250
2251 Wait.set(T: AMDGPU::LOAD_CNT, Val: ~0u);
2252 Wait.set(T: AMDGPU::DS_CNT, Val: ~0u);
2253 } else if (Wait.get(T: AMDGPU::STORE_CNT) != ~0u) {
2254 unsigned Enc = AMDGPU::encodeStorecntDscnt(Version: IV, Decoded: Wait);
2255
2256 SWaitInst = BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAIT_STORECNT_DSCNT))
2257 .addImm(Val: Enc);
2258
2259 Wait.set(T: AMDGPU::STORE_CNT, Val: ~0u);
2260 Wait.set(T: AMDGPU::DS_CNT, Val: ~0u);
2261 }
2262
2263 if (SWaitInst) {
2264 Modified = true;
2265
2266 LLVM_DEBUG(dbgs() << "GFX12Plus::createNewWaitcnt\n";
2267 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
2268 dbgs() << "New Instr: " << *SWaitInst << '\n');
2269 }
2270 }
2271
2272 // Generate an instruction for any remaining counter that needs
2273 // waiting for.
2274
2275 for (auto CT : inst_counter_types(MaxCounter: AMDGPU::NUM_EXTENDED_INST_CNTS)) {
2276 unsigned Count = Wait.get(T: CT);
2277 if (Count == ~0u)
2278 continue;
2279
2280 [[maybe_unused]] auto SWaitInst =
2281 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: instrsForExtendedCounterTypes[CT]))
2282 .addImm(Val: Count);
2283
2284 Modified = true;
2285
2286 LLVM_DEBUG(dbgs() << "GFX12Plus::createNewWaitcnt\n";
2287 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
2288 dbgs() << "New Instr: " << *SWaitInst << '\n');
2289 }
2290
2291 if (Wait.hasWaitDepctr()) {
2292 assert(IsExpertMode);
2293 unsigned Enc =
2294 AMDGPU::DepCtr::encodeFieldVmVsrc(VmVsrc: Wait.get(T: AMDGPU::VM_VSRC), STI: ST);
2295 // Encode min(VA_VDST_RD, VA_VDST_WR) into the single hardware field
2296 unsigned VaVdst =
2297 std::min(a: Wait.get(T: AMDGPU::VA_VDST_RD), b: Wait.get(T: AMDGPU::VA_VDST_WR));
2298 Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Encoded: Enc, VaVdst);
2299
2300 [[maybe_unused]] auto SWaitInst =
2301 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR)).addImm(Val: Enc);
2302
2303 Modified = true;
2304
2305 LLVM_DEBUG(dbgs() << "generateWaitcnt\n";
2306 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
2307 dbgs() << "New Instr: " << *SWaitInst << '\n');
2308 }
2309
2310 return Modified;
2311}
2312
2313/// Generate s_waitcnt instruction to be placed before cur_Inst.
2314/// Instructions of a given type are returned in order,
2315/// but instructions of different types can complete out of order.
2316/// We rely on this in-order completion
2317/// and simply assign a score to the memory access instructions.
2318/// We keep track of the active "score bracket" to determine
2319/// if an access of a memory read requires an s_waitcnt
2320/// and if so what the value of each counter is.
2321/// The "score bracket" is bound by the lower bound and upper bound
2322/// scores (*_score_LB and *_score_ub respectively).
2323/// If FlushFlags.FlushVmCnt is true, we want to flush the vmcnt counter here.
2324/// If FlushFlags.FlushDsCnt is true, we want to flush the dscnt counter here
2325/// (GFX12+ only, where DS_CNT is a separate counter).
2326bool SIInsertWaitcnts::generateWaitcntInstBefore(
2327 MachineInstr &MI, WaitcntBrackets &ScoreBrackets,
2328 MachineInstr *OldWaitcntInstr, PreheaderFlushFlags FlushFlags) {
2329 LLVM_DEBUG(dbgs() << "\n*** GenerateWaitcntInstBefore: "; MI.print(dbgs()););
2330
2331 assert(!isNonWaitcntMetaInst(MI));
2332
2333 AMDGPU::Waitcnt Wait;
2334 const unsigned Opc = MI.getOpcode();
2335
2336 switch (Opc) {
2337 case AMDGPU::BUFFER_WBINVL1:
2338 case AMDGPU::BUFFER_WBINVL1_SC:
2339 case AMDGPU::BUFFER_WBINVL1_VOL:
2340 case AMDGPU::BUFFER_GL0_INV:
2341 case AMDGPU::BUFFER_GL1_INV: {
2342 // FIXME: This should have already been handled by the memory legalizer.
2343 // Removing this currently doesn't affect any lit tests, but we need to
2344 // verify that nothing was relying on this. The number of buffer invalidates
2345 // being handled here should not be expanded.
2346 Wait.set(T: AMDGPU::LOAD_CNT, Val: 0);
2347 break;
2348 }
2349 case AMDGPU::SI_RETURN_TO_EPILOG:
2350 case AMDGPU::SI_RETURN:
2351 case AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN:
2352 case AMDGPU::S_SETPC_B64_return: {
2353 // All waits must be resolved at call return.
2354 // NOTE: this could be improved with knowledge of all call sites or
2355 // with knowledge of the called routines.
2356 ReturnInsts.insert(V: &MI);
2357 AMDGPU::Waitcnt AllZeroWait =
2358 WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/false);
2359 ScoreBrackets.simplifyVmemInvOnlyWait(Wait&: AllZeroWait);
2360 Wait = AllZeroWait;
2361 break;
2362 }
2363 case AMDGPU::S_ENDPGM:
2364 case AMDGPU::S_ENDPGM_SAVED: {
2365 // In dynamic VGPR mode, we want to release the VGPRs before the wave exits.
2366 // Technically the hardware will do this on its own if we don't, but that
2367 // might cost extra cycles compared to doing it explicitly.
2368 // When not in dynamic VGPR mode, identify S_ENDPGM instructions which may
2369 // have to wait for outstanding VMEM stores. In this case it can be useful
2370 // to send a message to explicitly release all VGPRs before the stores have
2371 // completed, but it is only safe to do this if there are no outstanding
2372 // scratch stores.
2373 EndPgmInsts[&MI] =
2374 !ScoreBrackets.empty(T: AMDGPU::STORE_CNT) &&
2375 !ScoreBrackets.hasPendingEvent(E: HWEvents::SCRATCH_WRITE_ACCESS);
2376 break;
2377 }
2378 case AMDGPU::S_SENDMSG:
2379 case AMDGPU::S_SENDMSGHALT: {
2380 if (ST.hasLegacyGeometry() &&
2381 ((MI.getOperand(i: 0).getImm() & AMDGPU::SendMsg::ID_MASK_PreGFX11_) ==
2382 AMDGPU::SendMsg::ID_GS_DONE_PreGFX11)) {
2383 // Resolve vm waits before gs-done.
2384 Wait.set(T: AMDGPU::LOAD_CNT, Val: 0);
2385 break;
2386 }
2387 [[fallthrough]];
2388 }
2389 default: {
2390
2391 // Export & GDS instructions do not read the EXEC mask until after the
2392 // export is granted (which can occur well after the instruction is issued).
2393 // The shader program must flush all EXP operations on the export-count
2394 // before overwriting the EXEC mask.
2395 if (MI.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &TRI)) {
2396 // Export and GDS are tracked individually, either may trigger a waitcnt
2397 // for EXEC.
2398 if (ScoreBrackets.hasPendingEvent(E: HWEvents::EXP_GPR_LOCK) ||
2399 ScoreBrackets.hasPendingEvent(E: HWEvents::EXP_PARAM_ACCESS) ||
2400 ScoreBrackets.hasPendingEvent(E: HWEvents::EXP_POS_ACCESS) ||
2401 ScoreBrackets.hasPendingEvent(E: HWEvents::GDS_GPR_LOCK)) {
2402 Wait.set(T: AMDGPU::EXP_CNT, Val: 0);
2403 }
2404 }
2405
2406 // Wait for any pending GDS instruction to complete before any
2407 // "Always GDS" instruction.
2408 if (TII.isAlwaysGDS(Opcode: Opc) && ScoreBrackets.hasPendingGDS())
2409 Wait.add(T: AMDGPU::DS_CNT, Count: ScoreBrackets.getPendingGDSWait());
2410
2411 if (MI.isCall()) {
2412 // The function is going to insert a wait on everything in its prolog.
2413 // This still needs to be careful if the call target is a load (e.g. a GOT
2414 // load). We also need to check WAW dependency with saved PC.
2415 CallInsts.insert(V: &MI);
2416 Wait = AMDGPU::Waitcnt();
2417
2418 const MachineOperand &CallAddrOp = TII.getCalleeOperand(MI);
2419 if (CallAddrOp.isReg()) {
2420 ScoreBrackets.determineWaitForPhysReg(
2421 T: SmemAccessCounter, Reg: CallAddrOp.getReg().asMCReg(), Wait, MI);
2422
2423 if (const auto *RtnAddrOp =
2424 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::dst)) {
2425 ScoreBrackets.determineWaitForPhysReg(
2426 T: SmemAccessCounter, Reg: RtnAddrOp->getReg().asMCReg(), Wait, MI);
2427 }
2428 }
2429 } else if (Opc == AMDGPU::S_BARRIER_WAIT) {
2430 ScoreBrackets.tryClearSCCWriteEvent(Inst: &MI);
2431 } else {
2432 // FIXME: Should not be relying on memoperands.
2433 // Look at the source operands of every instruction to see if
2434 // any of them results from a previous memory operation that affects
2435 // its current usage. If so, an s_waitcnt instruction needs to be
2436 // emitted.
2437 // If the source operand was defined by a load, add the s_waitcnt
2438 // instruction.
2439 //
2440 // Two cases are handled for destination operands:
2441 // 1) If the destination operand was defined by a load, add the s_waitcnt
2442 // instruction to guarantee the right WAW order.
2443 // 2) If a destination operand that was used by a recent export/store ins,
2444 // add s_waitcnt on exp_cnt to guarantee the WAR order.
2445
2446 for (const MachineMemOperand *Memop : MI.memoperands()) {
2447 const Value *Ptr = Memop->getValue();
2448 if (Memop->isStore()) {
2449 if (auto It = SLoadAddresses.find(Val: Ptr); It != SLoadAddresses.end()) {
2450 Wait.add(T: SmemAccessCounter, Count: 0);
2451 if (PDT.dominates(A: MI.getParent(), B: It->second))
2452 SLoadAddresses.erase(I: It);
2453 }
2454 }
2455 unsigned AS = Memop->getAddrSpace();
2456 if (AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::FLAT_ADDRESS)
2457 continue;
2458 // No need to wait before load from VMEM to LDS.
2459 if (TII.mayWriteLDSThroughDMA(MI))
2460 continue;
2461
2462 // LOAD_CNT is only relevant to vgpr or LDS.
2463 VMEMID TID = LDSDMA_BEGIN;
2464 if (Ptr && Memop->getAAInfo()) {
2465 const auto &LDSDMAStores = ScoreBrackets.getLDSDMAStores();
2466 for (size_t I = 0, E = LDSDMAStores.size(); I != E; ++I) {
2467 if (MI.mayAlias(AA, Other: *LDSDMAStores[I], UseTBAA: true)) {
2468 size_t Slot = I + 1;
2469 if (Slot >= NUM_LDSDMA) {
2470 // We didn't have enough slot to track this LDS DMA store, it
2471 // has been tracked using the common RegNo (FIRST_LDS_VGPR).
2472 ScoreBrackets.determineWaitForLDSDMA(T: AMDGPU::LOAD_CNT, TID,
2473 Wait);
2474 break;
2475 }
2476
2477 ScoreBrackets.determineWaitForLDSDMA(
2478 T: AMDGPU::LOAD_CNT, TID: TID + static_cast<VMEMID>(Slot), Wait);
2479 }
2480 }
2481 } else {
2482 ScoreBrackets.determineWaitForLDSDMA(T: AMDGPU::LOAD_CNT, TID, Wait);
2483 }
2484 if (Memop->isStore()) {
2485 ScoreBrackets.determineWaitForLDSDMA(T: AMDGPU::EXP_CNT, TID, Wait);
2486 }
2487 }
2488
2489 // Loop over use and def operands.
2490 for (const MachineOperand &Op : MI.operands()) {
2491 if (!Op.isReg())
2492 continue;
2493
2494 // If the instruction does not read tied source, skip the operand.
2495 if (Op.isTied() && Op.isUse() && TII.doesNotReadTiedSource(MI))
2496 continue;
2497
2498 MCRegister Reg = Op.getReg().asMCReg();
2499
2500 const bool IsVGPR = TRI.isVectorRegister(MRI, Reg: Op.getReg());
2501 if (IsVGPR) {
2502 // Implicit VGPR defs and uses are never a part of the memory
2503 // instructions description and usually present to account for
2504 // super-register liveness.
2505 // TODO: Most of the other instructions also have implicit uses
2506 // for the liveness accounting only.
2507 if (Op.isImplicit() && MI.mayLoadOrStore())
2508 continue;
2509
2510 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::VA_VDST_WR, Reg, Wait,
2511 MI);
2512 if (Op.isDef()) {
2513 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::VA_VDST_RD, Reg, Wait,
2514 MI);
2515 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::VM_VSRC, Reg, Wait,
2516 MI);
2517 }
2518
2519 // RAW always needs an s_waitcnt. WAW needs an s_waitcnt unless the
2520 // previous write and this write are the same type of VMEM
2521 // instruction, in which case they are (in some architectures)
2522 // guaranteed to write their results in order anyway.
2523 // Additionally check instructions where Point Sample Acceleration
2524 // might be applied.
2525 if (Op.isUse() || !updateVMCntOnly(Inst: MI) ||
2526 ScoreBrackets.hasDifferentVGPRPendingEvents(
2527 Reg, E: AMDGPU::getSimplifiedVMEMEventsFor(Inst: MI, TII)) ||
2528 ScoreBrackets.hasPointSamplePendingVmemTypes(MI, Reg) ||
2529 !ST.hasVmemWriteVgprInOrder()) {
2530 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::LOAD_CNT, Reg, Wait,
2531 MI);
2532 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::SAMPLE_CNT, Reg, Wait,
2533 MI);
2534 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::BVH_CNT, Reg, Wait,
2535 MI);
2536 ScoreBrackets.clearVGPRPendingEvents(Reg);
2537 }
2538
2539 if (Op.isDef() ||
2540 ScoreBrackets.hasPendingEvent(E: HWEvents::EXP_LDS_ACCESS)) {
2541 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::EXP_CNT, Reg, Wait,
2542 MI);
2543 }
2544 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::DS_CNT, Reg, Wait, MI);
2545 } else if (Op.getReg() == AMDGPU::SCC) {
2546 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::KM_CNT, Reg, Wait, MI);
2547 } else {
2548 ScoreBrackets.determineWaitForPhysReg(T: SmemAccessCounter, Reg, Wait,
2549 MI);
2550 }
2551
2552 if (ST.hasWaitXcnt() && ST.isXNACKEnabled() && Op.isDef())
2553 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::X_CNT, Reg, Wait, MI);
2554 }
2555 }
2556 }
2557 }
2558
2559 // Ensure safety against exceptions from outstanding memory operations while
2560 // waiting for a barrier:
2561 //
2562 // * Some subtargets safely handle backing off the barrier in hardware
2563 // when an exception occurs.
2564 // * Some subtargets have an implicit S_WAITCNT 0 before barriers, so that
2565 // there can be no outstanding memory operations during the wait.
2566 // * Subtargets with split barriers don't need to back off the barrier; it
2567 // is up to the trap handler to preserve the user barrier state correctly.
2568 //
2569 // In all other cases, ensure safety by ensuring that there are no outstanding
2570 // memory operations.
2571 if (Opc == AMDGPU::S_BARRIER && !ST.hasAutoWaitcntBeforeBarrier() &&
2572 !ST.hasBackOffBarrier()) {
2573 Wait = Wait.combined(Other: WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/true));
2574 }
2575
2576 // TODO: Remove this work-around, enable the assert for Bug 457939
2577 // after fixing the scheduler. Also, the Shader Compiler code is
2578 // independent of target.
2579 if (SIInstrInfo::isCBranchVCCZRead(MI) && ST.hasReadVCCZBug() &&
2580 ScoreBrackets.hasPendingEvent(E: HWEvents::SMEM_ACCESS)) {
2581 Wait.set(T: AMDGPU::DS_CNT, Val: 0);
2582 }
2583
2584 // Verify that the wait is actually needed.
2585 ScoreBrackets.simplifyWaitcnt(Wait);
2586
2587 // It is only necessary to insert an S_WAITCNT_DEPCTR instruction that
2588 // waits on VA_VDST if the instruction it would precede is not a VALU
2589 // instruction, since hardware handles VALU->VGPR->VALU hazards in
2590 // expert scheduling mode.
2591 if (TII.isVALU(MI, /*AllowLDSDMA=*/false)) {
2592 Wait.set(T: AMDGPU::VA_VDST_RD, Val: ~0u);
2593 Wait.set(T: AMDGPU::VA_VDST_WR, Val: ~0u);
2594 }
2595
2596 // Since the translation for VMEM addresses occur in-order, we can apply the
2597 // XCnt if the current instruction is of VMEM type and has a memory
2598 // dependency with another VMEM instruction in flight.
2599 if (Wait.get(T: AMDGPU::X_CNT) != ~0u && isVmemAccess(MI)) {
2600 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::X_CNT);
2601 Wait.set(T: AMDGPU::X_CNT, Val: ~0u);
2602 }
2603
2604 // When forcing emit, we need to skip terminators because that would break the
2605 // terminators of the MBB if we emit a waitcnt between terminators.
2606 if (ForceEmitZeroFlag && !MI.isTerminator())
2607 Wait = WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/false);
2608
2609 // If we force waitcnt then update Wait accordingly.
2610 for (AMDGPU::InstCounterType T : AMDGPU::inst_counter_types()) {
2611 if (!ForceEmitWaitcnt[T])
2612 continue;
2613 Wait.set(T, Val: 0);
2614 }
2615
2616 if (FlushFlags.FlushVmCnt) {
2617 for (AMDGPU::InstCounterType T :
2618 {AMDGPU::LOAD_CNT, AMDGPU::SAMPLE_CNT, AMDGPU::BVH_CNT})
2619 Wait.set(T, Val: 0);
2620 }
2621
2622 if (FlushFlags.FlushDsCnt && ScoreBrackets.hasPendingEvent(T: AMDGPU::DS_CNT))
2623 Wait.set(T: AMDGPU::DS_CNT, Val: 0);
2624
2625 if (ForceEmitZeroLoadFlag && Wait.get(T: AMDGPU::LOAD_CNT) != ~0u)
2626 Wait.set(T: AMDGPU::LOAD_CNT, Val: 0);
2627
2628 return generateWaitcnt(Wait, It: MI.getIterator(), Block&: *MI.getParent(), ScoreBrackets,
2629 OldWaitcntInstr);
2630}
2631
2632bool SIInsertWaitcnts::generateWaitcnt(AMDGPU::Waitcnt Wait,
2633 MachineBasicBlock::instr_iterator It,
2634 MachineBasicBlock &Block,
2635 WaitcntBrackets &ScoreBrackets,
2636 MachineInstr *OldWaitcntInstr) {
2637 bool Modified = false;
2638
2639 if (OldWaitcntInstr)
2640 // Try to merge the required wait with preexisting waitcnt instructions.
2641 // Also erase redundant waitcnt.
2642 Modified =
2643 WCG->applyPreexistingWaitcnt(ScoreBrackets, OldWaitcntInstr&: *OldWaitcntInstr, Wait, It);
2644
2645 // ExpCnt can be merged into VINTERP.
2646 if (Wait.get(T: AMDGPU::EXP_CNT) != ~0u && It != Block.instr_end() &&
2647 SIInstrInfo::isVINTERP(MI: *It)) {
2648 MachineOperand *WaitExp = TII.getNamedOperand(MI&: *It, OperandName: AMDGPU::OpName::waitexp);
2649 if (Wait.get(T: AMDGPU::EXP_CNT) < WaitExp->getImm()) {
2650 WaitExp->setImm(Wait.get(T: AMDGPU::EXP_CNT));
2651 Modified = true;
2652 }
2653 // Apply ExpCnt before resetting it, so applyWaitcnt below sees all counts.
2654 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::EXP_CNT);
2655 Wait.set(T: AMDGPU::EXP_CNT, Val: ~0u);
2656
2657 LLVM_DEBUG(dbgs() << "generateWaitcnt\n"
2658 << "Update Instr: " << *It);
2659 }
2660
2661 if (WCG->createNewWaitcnt(Block, It, Wait, ScoreBrackets))
2662 Modified = true;
2663
2664 // Any counts that could have been applied to any existing waitcnt
2665 // instructions will have been done so, now deal with any remaining.
2666 ScoreBrackets.applyWaitcnt(Wait);
2667
2668 return Modified;
2669}
2670
2671bool SIInsertWaitcnts::isVmemAccess(const MachineInstr &MI) const {
2672 return (TII.isFLAT(MI) && TII.mayAccessVMEMThroughFlat(MI)) ||
2673 (TII.isVMEM(MI) && !AMDGPU::getMUBUFIsBufferInv(Opc: MI.getOpcode()));
2674}
2675
2676// Return true if the next instruction is S_ENDPGM, following fallthrough
2677// blocks if necessary.
2678bool SIInsertWaitcnts::isNextENDPGM(MachineBasicBlock::instr_iterator It,
2679 MachineBasicBlock *Block) const {
2680 auto BlockEnd = Block->getParent()->end();
2681 auto BlockIter = Block->getIterator();
2682
2683 while (true) {
2684 if (It.isEnd()) {
2685 if (++BlockIter != BlockEnd) {
2686 It = BlockIter->instr_begin();
2687 continue;
2688 }
2689
2690 return false;
2691 }
2692
2693 if (!It->isMetaInstruction())
2694 break;
2695
2696 It++;
2697 }
2698
2699 assert(!It.isEnd());
2700
2701 return It->getOpcode() == AMDGPU::S_ENDPGM;
2702}
2703
2704// Add a wait after an instruction if architecture requirements mandate one.
2705bool SIInsertWaitcnts::insertForcedWaitAfter(MachineInstr &Inst,
2706 MachineBasicBlock &Block,
2707 WaitcntBrackets &ScoreBrackets) {
2708 AMDGPU::Waitcnt Wait;
2709 bool NeedsEndPGMCheck = false;
2710
2711 if (ST.isPreciseMemoryEnabled() && Inst.mayLoadOrStore())
2712 Wait = WCG->getAllZeroWaitcnt(IncludeVSCnt: Inst.mayStore() &&
2713 !SIInstrInfo::isAtomicRet(MI: Inst));
2714
2715 if (TII.isAlwaysGDS(Opcode: Inst.getOpcode())) {
2716 Wait.set(T: AMDGPU::DS_CNT, Val: 0);
2717 NeedsEndPGMCheck = true;
2718 }
2719
2720 ScoreBrackets.simplifyWaitcnt(Wait);
2721
2722 auto SuccessorIt = next_nodbg(It: Inst.getIterator(), End: Block.instr_end());
2723 bool Result = generateWaitcnt(Wait, It: SuccessorIt, Block, ScoreBrackets,
2724 /*OldWaitcntInstr=*/nullptr);
2725
2726 if (Result && NeedsEndPGMCheck && isNextENDPGM(It: SuccessorIt, Block: &Block)) {
2727 BuildMI(BB&: Block, I: SuccessorIt, MIMD: Inst.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_NOP))
2728 .addImm(Val: 0);
2729 }
2730
2731 return Result;
2732}
2733
2734void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
2735 WaitcntBrackets *ScoreBrackets) {
2736
2737 HWEvents InstEvents = AMDGPU::getEventsFor(Inst, ST, IsExpertMode, TgSplit);
2738 for (HWEvents E : InstEvents)
2739 ScoreBrackets->updateByEvent(E, Inst);
2740
2741 if (TII.isDS(MI: Inst) && TII.usesLGKM_CNT(MI: Inst)) {
2742 if (TII.isAlwaysGDS(Opcode: Inst.getOpcode()) ||
2743 TII.hasModifiersSet(MI: Inst, OpName: AMDGPU::OpName::gds)) {
2744 ScoreBrackets->setPendingGDS();
2745 }
2746 } else if (TII.isFLAT(MI: Inst)) {
2747 if (Inst.mayLoadOrStore() && TII.mayAccessVMEMThroughFlat(MI: Inst) &&
2748 TII.mayAccessLDSThroughFlat(MI: Inst, TgSplit) &&
2749 !SIInstrInfo::isLDSDMA(MI: Inst)) {
2750 // Async/LDSDMA operations have FLAT encoding but do not actually use flat
2751 // pointers. They do have two operands that each access global and LDS,
2752 // thus making it appear at this point that they are using a flat pointer.
2753 // Filter them out, and for the rest, generate a dependency on flat
2754 // pointers so that both VM and LGKM counters are flushed.
2755 ScoreBrackets->setPendingFlat();
2756 }
2757 } else if (Inst.isCall()) {
2758 // Act as a wait on everything, but AsyncCnt and TensorCnt are never
2759 // included in such blanket waits.
2760 ScoreBrackets->applyWaitcnt(Wait: WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/false));
2761 ScoreBrackets->setStateOnFunctionEntryOrReturn();
2762 } else if (TII.isVINTERP(MI: Inst)) {
2763 int64_t Imm = TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::waitexp)->getImm();
2764 ScoreBrackets->applyWaitcnt(T: AMDGPU::EXP_CNT, Count: static_cast<unsigned>(Imm));
2765 }
2766
2767 // Set XCNT to zero in the bracket for instructions that implicitly drain
2768 // XCNT.
2769 if (ST.hasWaitXcnt() && ST.isXNACKEnabled() && SIInstrInfo::isXcntDrain(MI: Inst))
2770 ScoreBrackets->applyWaitcnt(T: AMDGPU::X_CNT, Count: 0);
2771}
2772
2773bool WaitcntBrackets::mergeScore(const MergeInfo &M, unsigned &Score,
2774 unsigned OtherScore) {
2775 unsigned MyShifted = Score <= M.OldLB ? 0 : Score + M.MyShift;
2776 unsigned OtherShifted =
2777 OtherScore <= M.OtherLB ? 0 : OtherScore + M.OtherShift;
2778 Score = std::max(a: MyShifted, b: OtherShifted);
2779 return OtherShifted > MyShifted;
2780}
2781
2782bool WaitcntBrackets::mergeAsyncMarks(ArrayRef<MergeInfo> MergeInfos,
2783 ArrayRef<CounterValueArray> OtherMarks) {
2784 bool StrictDom = false;
2785
2786 LLVM_DEBUG(dbgs() << "Merging async marks ...");
2787 // Early exit: nothing to merge when both sides are empty.
2788 if (AsyncMarks.empty() && OtherMarks.empty()) {
2789 LLVM_DEBUG(dbgs() << " nothing to merge\n");
2790 return false;
2791 }
2792 LLVM_DEBUG(dbgs() << '\n');
2793
2794 // Determine maximum length needed after merging
2795 auto MaxSize = (unsigned)std::max(a: AsyncMarks.size(), b: OtherMarks.size());
2796 MaxSize = std::min(a: MaxSize, b: MaxAsyncMarks);
2797
2798 // Keep only the most recent marks within our limit.
2799 if (AsyncMarks.size() > MaxSize)
2800 AsyncMarks.erase(CS: AsyncMarks.begin(),
2801 CE: AsyncMarks.begin() + (AsyncMarks.size() - MaxSize));
2802
2803 // Pad with zero-filled marks if our list is shorter. Zero represents "no
2804 // pending async operations at this checkpoint" and acts as the identity
2805 // element for max() during merging. We pad at the beginning since the marks
2806 // need to be aligned in most-recent order.
2807 constexpr CounterValueArray ZeroMark{};
2808 AsyncMarks.insert(I: AsyncMarks.begin(), NumToInsert: MaxSize - AsyncMarks.size(), Elt: ZeroMark);
2809
2810 LLVM_DEBUG({
2811 dbgs() << "Before merge:\n";
2812 for (const auto &Mark : AsyncMarks) {
2813 llvm::interleaveComma(Mark, dbgs());
2814 dbgs() << '\n';
2815 }
2816 dbgs() << "Other marks:\n";
2817 for (const auto &Mark : OtherMarks) {
2818 llvm::interleaveComma(Mark, dbgs());
2819 dbgs() << '\n';
2820 }
2821 });
2822
2823 // Merge element-wise using the existing mergeScore function and the
2824 // appropriate MergeInfo for each counter type. Rebase and merge EVERY
2825 // surviving mark into the new (widened) frame, aligned by recency from the
2826 // back. A mark with no counterpart on the other side merges against the
2827 // zero/identity mark: mergeScore still applies MyShift, so the mark is
2828 // re-expressed in the new frame instead of being left with a stale
2829 // (pre-merge) score.
2830 const size_t OtherSize = OtherMarks.size();
2831 const size_t OurSize = AsyncMarks.size();
2832 // After the both-empty early-return above, max(AsyncMarks, OtherMarks) >= 1,
2833 // and the erase/pad steps normalize AsyncMarks to exactly MaxSize. Hence
2834 // OurSize == MaxSize >= 1 (as long as MaxAsyncMarks != 0), so the
2835 // seq_inclusive(1, OurSize) below never trips its "Begin <= End" assertion
2836 // the way seq_inclusive(1, MergeCount) could when OtherSize == 0.
2837 assert(OurSize >= 1 &&
2838 "AsyncMarks padded to MaxSize >= 1 (needs MaxAsyncMarks != 0)");
2839
2840 for (auto Idx : seq_inclusive<size_t>(Begin: 1, End: OurSize)) {
2841 const CounterValueArray &OtherMark =
2842 Idx <= OtherSize ? OtherMarks[OtherSize - Idx] : ZeroMark;
2843 for (auto T : inst_counter_types(MaxCounter: Context->MaxCounter))
2844 StrictDom |=
2845 mergeScore(M: MergeInfos[T], Score&: AsyncMarks[OurSize - Idx][T], OtherScore: OtherMark[T]);
2846 }
2847
2848 LLVM_DEBUG({
2849 dbgs() << "After merge:\n";
2850 for (const auto &Mark : AsyncMarks) {
2851 llvm::interleaveComma(Mark, dbgs());
2852 dbgs() << '\n';
2853 }
2854 });
2855
2856 return StrictDom;
2857}
2858
2859/// Merge the pending events and associater score brackets of \p Other into
2860/// this brackets status.
2861///
2862/// Returns whether the merge resulted in a change that requires tighter waits
2863/// (i.e. the merged brackets strictly dominate the original brackets).
2864bool WaitcntBrackets::merge(const WaitcntBrackets &Other) {
2865 bool StrictDom = false;
2866
2867 // Check if "other" has keys we don't have, and create default entries for
2868 // those. If they remain empty after merging, we will clean it up after.
2869 for (auto K : Other.VMem.keys())
2870 VMem.try_emplace(Key: K);
2871 for (auto K : Other.SGPRs.keys())
2872 SGPRs.try_emplace(Key: K);
2873
2874 // Array to store MergeInfo for each counter type
2875 MergeInfo MergeInfos[AMDGPU::NUM_INST_CNTS];
2876
2877 for (auto T : inst_counter_types(MaxCounter: Context->MaxCounter)) {
2878 // Merge event flags for this counter
2879 const HWEvents &EventsForT = Context->getWaitEvents(T);
2880 const HWEvents OldEvents = PendingEvents & EventsForT;
2881 const HWEvents OtherEvents = Other.PendingEvents & EventsForT;
2882 if (!OldEvents.contains(Other: OtherEvents))
2883 StrictDom = true;
2884 PendingEvents |= OtherEvents;
2885
2886 // Merge scores for this counter
2887 const unsigned MyPending = ScoreUBs[T] - ScoreLBs[T];
2888 const unsigned OtherPending = Other.ScoreUBs[T] - Other.ScoreLBs[T];
2889 const unsigned NewUB = ScoreLBs[T] + std::max(a: MyPending, b: OtherPending);
2890 if (NewUB < ScoreLBs[T])
2891 report_fatal_error(reason: "waitcnt score overflow");
2892
2893 MergeInfo &M = MergeInfos[T];
2894 M.OldLB = ScoreLBs[T];
2895 M.OtherLB = Other.ScoreLBs[T];
2896 M.MyShift = NewUB - ScoreUBs[T];
2897 M.OtherShift = NewUB - Other.ScoreUBs[T];
2898
2899 ScoreUBs[T] = NewUB;
2900
2901 if (T == AMDGPU::LOAD_CNT)
2902 StrictDom |= mergeScore(M, Score&: LastFlatLoadCnt, OtherScore: Other.LastFlatLoadCnt);
2903
2904 if (T == AMDGPU::DS_CNT) {
2905 StrictDom |= mergeScore(M, Score&: LastFlatDsCnt, OtherScore: Other.LastFlatDsCnt);
2906 StrictDom |= mergeScore(M, Score&: LastGDS, OtherScore: Other.LastGDS);
2907 }
2908
2909 if (T == AMDGPU::KM_CNT) {
2910 StrictDom |= mergeScore(M, Score&: SCCScore, OtherScore: Other.SCCScore);
2911 if (Other.hasPendingEvent(E: HWEvents::SCC_WRITE)) {
2912 if (!(OldEvents & HWEvents::SCC_WRITE)) {
2913 PendingSCCWrite = Other.PendingSCCWrite;
2914 } else if (PendingSCCWrite != Other.PendingSCCWrite) {
2915 PendingSCCWrite = nullptr;
2916 }
2917 }
2918 }
2919
2920 for (auto &[RegID, Info] : VMem)
2921 StrictDom |= mergeScore(M, Score&: Info.Scores[T], OtherScore: Other.getVMemScore(TID: RegID, T));
2922
2923 if (isSmemCounter(T)) {
2924 for (auto &[RegID, Info] : SGPRs) {
2925 auto It = Other.SGPRs.find(Val: RegID);
2926 unsigned OtherScore = (It != Other.SGPRs.end()) ? It->second.get(T) : 0;
2927 StrictDom |= mergeScore(M, Score&: Info.get(T), OtherScore);
2928 }
2929 }
2930 }
2931
2932 for (auto &[TID, Info] : VMem) {
2933 if (auto It = Other.VMem.find(Val: TID); It != Other.VMem.end()) {
2934 HWEvents NewVGPRContext =
2935 Info.VGPRPendingEvents | It->second.VGPRPendingEvents;
2936 StrictDom |= NewVGPRContext != Info.VGPRPendingEvents;
2937 Info.VGPRPendingEvents = NewVGPRContext;
2938 }
2939 }
2940
2941 StrictDom |= mergeAsyncMarks(MergeInfos, OtherMarks: Other.AsyncMarks);
2942 for (auto T : inst_counter_types(MaxCounter: Context->MaxCounter))
2943 StrictDom |= mergeScore(M: MergeInfos[T], Score&: AsyncScore[T], OtherScore: Other.AsyncScore[T]);
2944
2945 purgeEmptyTrackingData();
2946 return StrictDom;
2947}
2948
2949static bool isWaitInstr(MachineInstr &Inst) {
2950 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode: Inst.getOpcode());
2951 return Opcode == AMDGPU::S_WAITCNT ||
2952 (Opcode == AMDGPU::S_WAITCNT_VSCNT && Inst.getOperand(i: 0).isReg() &&
2953 Inst.getOperand(i: 0).getReg() == AMDGPU::SGPR_NULL) ||
2954 Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT ||
2955 Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT ||
2956 Opcode == AMDGPU::S_WAITCNT_lds_direct ||
2957 Opcode == AMDGPU::WAIT_ASYNCMARK ||
2958 AMDGPU::counterTypeForInstr(Opcode).has_value();
2959}
2960
2961void SIInsertWaitcnts::setSchedulingMode(MachineBasicBlock &MBB,
2962 MachineBasicBlock::iterator I,
2963 bool ExpertMode) const {
2964 const uint64_t EncodedReg = AMDGPU::Hwreg::HwregEncoding::encode(
2965 Values: AMDGPU::Hwreg::ID_SCHED_MODE, Values: AMDGPU::Hwreg::HwregOffset::Default, Values: 2);
2966 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_SETREG_IMM32_B32))
2967 .addImm(Val: ExpertMode ? 2 : 0)
2968 .addImm(Val: EncodedReg);
2969}
2970
2971namespace {
2972// TODO: Remove this work-around after fixing the scheduler.
2973// There are two reasons why vccz might be incorrect; see ST.hasReadVCCZBug()
2974// and ST.partialVCCWritesUpdateVCCZ().
2975// i. VCCZBug: There is a hardware bug on CI/SI where SMRD instruction may
2976// corrupt vccz bit, so when we detect that an instruction may read from
2977// a corrupt vccz bit, we need to:
2978// 1. Insert s_waitcnt lgkm(0) to wait for all outstanding SMRD
2979// operations to complete.
2980// 2. Recompute the correct value of vccz by writing the current value
2981// of vcc back to vcc.
2982// ii. Partial writes to vcc don't update vccz, so we need to recompute the
2983// correct value of vccz by reading vcc and writing it back to vcc.
2984// No waitcnt is needed in this case.
2985class VCCZWorkaround {
2986 const WaitcntBrackets &ScoreBrackets;
2987 const GCNSubtarget &ST;
2988 const SIInstrInfo &TII;
2989 const SIRegisterInfo &TRI;
2990 bool VCCZCorruptionBug = false;
2991 bool VCCZNotUpdatedByPartialWrites = false;
2992 /// vccz could be incorrect at a basic block boundary if a predecessor wrote
2993 /// to vcc and then issued an smem load, so initialize to true.
2994 bool MustRecomputeVCCZ = true;
2995
2996public:
2997 VCCZWorkaround(const WaitcntBrackets &ScoreBrackets, const GCNSubtarget &ST,
2998 const SIInstrInfo &TII, const SIRegisterInfo &TRI)
2999 : ScoreBrackets(ScoreBrackets), ST(ST), TII(TII), TRI(TRI) {
3000 VCCZCorruptionBug = ST.hasReadVCCZBug();
3001 VCCZNotUpdatedByPartialWrites = !ST.partialVCCWritesUpdateVCCZ();
3002 }
3003 /// If \p MI reads vccz and we must recompute it based on MustRecomputeVCCZ,
3004 /// then emit a vccz recompute instruction before \p MI. This needs to be
3005 /// called on every instruction in the basic block because it also tracks the
3006 /// state and updates MustRecomputeVCCZ accordingly. Returns true if it
3007 /// modified the IR.
3008 bool tryRecomputeVCCZ(MachineInstr &MI) {
3009 // No need to run this if neither bug is present.
3010 if (!VCCZCorruptionBug && !VCCZNotUpdatedByPartialWrites)
3011 return false;
3012
3013 // If MI is an SMEM and it can corrupt vccz on this target, then we need
3014 // both to emit a waitcnt and to recompute vccz.
3015 // But we don't actually emit a waitcnt here. This is done in
3016 // generateWaitcntInstBefore() because it tracks all the necessary waitcnt
3017 // state, and can either skip emitting a waitcnt if there is already one in
3018 // the IR, or emit an "optimized" combined waitcnt.
3019 // If this is an smem read, it could complete and clobber vccz at any time.
3020 MustRecomputeVCCZ |= VCCZCorruptionBug && TII.isSMRD(MI);
3021
3022 // If the target partial vcc writes don't update vccz, and MI is such an
3023 // instruction then we must recompute vccz.
3024 // Note: We are using PartiallyWritesToVCCOpt optional to avoid calling
3025 // `definesRegister()` more than needed, because it's not very cheap.
3026 std::optional<bool> PartiallyWritesToVCCOpt;
3027 auto PartiallyWritesToVCC = [](MachineInstr &MI) {
3028 return MI.definesRegister(Reg: AMDGPU::VCC_LO, /*TRI=*/nullptr) ||
3029 MI.definesRegister(Reg: AMDGPU::VCC_HI, /*TRI=*/nullptr);
3030 };
3031 if (VCCZNotUpdatedByPartialWrites) {
3032 PartiallyWritesToVCCOpt = PartiallyWritesToVCC(MI);
3033 // If this is a partial VCC write but won't update vccz, then we must
3034 // recompute vccz.
3035 MustRecomputeVCCZ |= *PartiallyWritesToVCCOpt;
3036 }
3037
3038 // If MI is a vcc write with no pending smem, or there is a pending smem
3039 // but the target does not suffer from the vccz corruption bug, then we
3040 // don't need to recompute vccz as this write will recompute it anyway.
3041 if (!ScoreBrackets.hasPendingEvent(E: HWEvents::SMEM_ACCESS) ||
3042 !VCCZCorruptionBug) {
3043 // Compute PartiallyWritesToVCCOpt if we haven't done so already.
3044 if (!PartiallyWritesToVCCOpt)
3045 PartiallyWritesToVCCOpt = PartiallyWritesToVCC(MI);
3046 bool FullyWritesToVCC = !*PartiallyWritesToVCCOpt &&
3047 MI.definesRegister(Reg: AMDGPU::VCC, /*TRI=*/nullptr);
3048 // If we write to the full vcc or we write partially and the target
3049 // updates vccz on partial writes, then vccz will be updated correctly.
3050 bool UpdatesVCCZ = FullyWritesToVCC || (!VCCZNotUpdatedByPartialWrites &&
3051 *PartiallyWritesToVCCOpt);
3052 if (UpdatesVCCZ)
3053 MustRecomputeVCCZ = false;
3054 }
3055
3056 // If MI is a branch that reads VCCZ then emit a waitcnt and a vccz
3057 // restore instruction if either is needed.
3058 if (SIInstrInfo::isCBranchVCCZRead(MI) && MustRecomputeVCCZ) {
3059 // Recompute the vccz bit. Any time a value is written to vcc, the vccz
3060 // bit is updated, so we can restore the bit by reading the value of vcc
3061 // and then writing it back to the register.
3062 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
3063 MCID: TII.get(Opcode: ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64),
3064 DestReg: TRI.getVCC())
3065 .addReg(RegNo: TRI.getVCC());
3066 MustRecomputeVCCZ = false;
3067 return true;
3068 }
3069 return false;
3070 }
3071};
3072
3073} // namespace
3074
3075// Generate s_waitcnt instructions where needed.
3076bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
3077 MachineBasicBlock &Block,
3078 WaitcntBrackets &ScoreBrackets) {
3079 bool Modified = false;
3080
3081 LLVM_DEBUG({
3082 dbgs() << "*** Begin Block: ";
3083 Block.printName(dbgs());
3084 ScoreBrackets.dump();
3085 });
3086 VCCZWorkaround VCCZW(ScoreBrackets, ST, TII, TRI);
3087
3088 // Walk over the instructions.
3089 MachineInstr *OldWaitcntInstr = nullptr;
3090
3091 // NOTE: We may append instrs after Inst while iterating.
3092 ScoreBrackets.verify();
3093 for (MachineBasicBlock::instr_iterator Iter = Block.instr_begin(),
3094 E = Block.instr_end();
3095 Iter != E; ++Iter) {
3096 MachineInstr &Inst = *Iter;
3097 if (isNonWaitcntMetaInst(MI: Inst))
3098 continue;
3099 // Track pre-existing waitcnts that were added in earlier iterations or by
3100 // the memory legalizer.
3101 if (isWaitInstr(Inst) ||
3102 (IsExpertMode && Inst.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR)) {
3103 if (!OldWaitcntInstr)
3104 OldWaitcntInstr = &Inst;
3105 continue;
3106 }
3107
3108 PreheaderFlushFlags FlushFlags;
3109 if (Block.getFirstTerminator() == Inst)
3110 FlushFlags = isPreheaderToFlush(MBB&: Block, ScoreBrackets);
3111
3112 // Generate an s_waitcnt instruction to be placed before Inst, if needed.
3113 Modified |= generateWaitcntInstBefore(MI&: Inst, ScoreBrackets, OldWaitcntInstr,
3114 FlushFlags);
3115 OldWaitcntInstr = nullptr;
3116
3117 if (Inst.getOpcode() == AMDGPU::ASYNCMARK) {
3118 // Asyncmarks record the current wait state and so should not allow
3119 // waitcnts that occur after them to be merged into waitcnts that occur
3120 // before.
3121 ScoreBrackets.recordAsyncMark(Inst);
3122 continue;
3123 }
3124
3125 if (TII.isSMRD(MI: Inst)) {
3126 for (const MachineMemOperand *Memop : Inst.memoperands()) {
3127 // No need to handle invariant loads when avoiding WAR conflicts, as
3128 // there cannot be a vector store to the same memory location.
3129 if (!Memop->isInvariant()) {
3130 const Value *Ptr = Memop->getValue();
3131 SLoadAddresses.insert(KV: std::pair(Ptr, Inst.getParent()));
3132 }
3133 }
3134 }
3135
3136 updateEventWaitcntAfter(Inst, ScoreBrackets: &ScoreBrackets);
3137
3138 // Note: insertForcedWaitAfter() may add instrs after Iter that need to be
3139 // visited by the loop.
3140 Modified |= insertForcedWaitAfter(Inst, Block, ScoreBrackets);
3141
3142 LLVM_DEBUG({
3143 Inst.print(dbgs());
3144 ScoreBrackets.dump();
3145 });
3146
3147 // If the target suffers from the vccz bugs, this may emit the necessary
3148 // vccz recompute instruction before \p Inst if needed.
3149 Modified |= VCCZW.tryRecomputeVCCZ(MI&: Inst);
3150
3151 ScoreBrackets.verify();
3152 }
3153
3154 // Flush counters at the end of the block if needed (for preheaders with no
3155 // terminator).
3156 AMDGPU::Waitcnt Wait;
3157 if (Block.getFirstTerminator() == Block.end()) {
3158 PreheaderFlushFlags FlushFlags = isPreheaderToFlush(MBB&: Block, ScoreBrackets);
3159 if (FlushFlags.FlushVmCnt) {
3160 if (ScoreBrackets.hasPendingEvent(T: AMDGPU::LOAD_CNT))
3161 Wait.set(T: AMDGPU::LOAD_CNT, Val: 0);
3162 if (ScoreBrackets.hasPendingEvent(T: AMDGPU::SAMPLE_CNT))
3163 Wait.set(T: AMDGPU::SAMPLE_CNT, Val: 0);
3164 if (ScoreBrackets.hasPendingEvent(T: AMDGPU::BVH_CNT))
3165 Wait.set(T: AMDGPU::BVH_CNT, Val: 0);
3166 }
3167 if (FlushFlags.FlushDsCnt && ScoreBrackets.hasPendingEvent(T: AMDGPU::DS_CNT))
3168 Wait.set(T: AMDGPU::DS_CNT, Val: 0);
3169 }
3170
3171 // Combine or remove any redundant waitcnts at the end of the block.
3172 Modified |= generateWaitcnt(Wait, It: Block.instr_end(), Block, ScoreBrackets,
3173 OldWaitcntInstr);
3174
3175 LLVM_DEBUG({
3176 dbgs() << "*** End Block: ";
3177 Block.printName(dbgs());
3178 ScoreBrackets.dump();
3179 });
3180
3181 return Modified;
3182}
3183
3184bool SIInsertWaitcnts::removeRedundantSoftXcnts(MachineBasicBlock &Block) {
3185 if (Block.size() <= 1)
3186 return false;
3187 // The Memory Legalizer conservatively inserts a soft xcnt before each
3188 // atomic RMW operation. However, for sequences of back-to-back atomic
3189 // RMWs, only the first s_wait_xcnt insertion is necessary. Optimize away
3190 // the redundant soft xcnts.
3191 bool Modified = false;
3192 // Remember the last atomic with a soft xcnt right before it.
3193 MachineInstr *LastAtomicWithSoftXcnt = nullptr;
3194
3195 for (MachineInstr &MI : drop_begin(RangeOrContainer&: Block)) {
3196 // Ignore last atomic if non-LDS VMEM and SMEM.
3197 bool IsLDS = TII.isDS(MI) ||
3198 (TII.isFLAT(MI) && TII.mayAccessLDSThroughFlat(MI, TgSplit));
3199 if (!IsLDS && (MI.mayLoad() ^ MI.mayStore()))
3200 LastAtomicWithSoftXcnt = nullptr;
3201
3202 bool IsAtomicRMW =
3203 SIInstrFlags::isMaybeAtomic(O: MI) && MI.mayLoad() && MI.mayStore();
3204 MachineInstr &PrevMI = *MI.getPrevNode();
3205 // This is an atomic with a soft xcnt.
3206 if (PrevMI.getOpcode() == AMDGPU::S_WAIT_XCNT_soft && IsAtomicRMW) {
3207 // If we have already found an atomic with a soft xcnt, remove this soft
3208 // xcnt as it's redundant.
3209 if (LastAtomicWithSoftXcnt) {
3210 PrevMI.eraseFromParent();
3211 Modified = true;
3212 }
3213 LastAtomicWithSoftXcnt = &MI;
3214 }
3215 }
3216 return Modified;
3217}
3218
3219// Return flags indicating which counters should be flushed in the preheader.
3220PreheaderFlushFlags
3221SIInsertWaitcnts::isPreheaderToFlush(MachineBasicBlock &MBB,
3222 const WaitcntBrackets &ScoreBrackets) {
3223 auto [Iterator, IsInserted] =
3224 PreheadersToFlush.try_emplace(Key: &MBB, Args: PreheaderFlushFlags());
3225 if (!IsInserted)
3226 return Iterator->second;
3227
3228 MachineBasicBlock *Succ = MBB.getSingleSuccessor();
3229 if (!Succ)
3230 return PreheaderFlushFlags();
3231
3232 MachineLoop *Loop = MLI.getLoopFor(BB: Succ);
3233 if (!Loop)
3234 return PreheaderFlushFlags();
3235
3236 if (Loop->getLoopPreheader() == &MBB) {
3237 Iterator->second = getPreheaderFlushFlags(ML: Loop, Brackets: ScoreBrackets);
3238 return Iterator->second;
3239 }
3240
3241 return PreheaderFlushFlags();
3242}
3243
3244bool SIInsertWaitcnts::isVMEMOrFlatVMEM(const MachineInstr &MI) const {
3245 if (SIInstrInfo::isFLAT(MI))
3246 return TII.mayAccessVMEMThroughFlat(MI);
3247 return SIInstrInfo::isVMEM(MI);
3248}
3249
3250bool SIInsertWaitcnts::isDSRead(const MachineInstr &MI) const {
3251 return SIInstrInfo::isDS(MI) && MI.mayLoad() && !MI.mayStore();
3252}
3253
3254// Check if instruction is a store to LDS that is counted via DSCNT
3255// (where that counter exists).
3256bool SIInsertWaitcnts::mayStoreIncrementingDSCNT(const MachineInstr &MI) const {
3257 return MI.mayStore() && SIInstrInfo::isDS(MI);
3258}
3259
3260// Return flags indicating which counters should be flushed in the preheader of
3261// the given loop. We currently decide to flush in the following situations:
3262// For VMEM (FlushVmCnt):
3263// 1. The loop contains vmem store(s), no vmem load and at least one use of a
3264// vgpr containing a value that is loaded outside of the loop. (Only on
3265// targets with no vscnt counter).
3266// 2. The loop contains vmem load(s), but the loaded values are not used in the
3267// loop, and at least one use of a vgpr containing a value that is loaded
3268// outside of the loop.
3269// For DS (FlushDsCnt, GFX12+ only):
3270// 3. The loop contains no DS reads, and at least one use of a vgpr containing
3271// a value that is DS read outside of the loop.
3272// 4. The loop contains DS read(s), loaded values are not used in the same
3273// iteration but in the next iteration (prefetch pattern), and at least one
3274// use of a vgpr containing a value that is DS read outside of the loop.
3275// Flushing in preheader reduces wait overhead if the wait requirement in
3276// iteration 1 would otherwise be more strict (but unfortunately preheader
3277// flush decision is taken before knowing that).
3278// 5. (Single-block loops only) The loop has DS prefetch reads with flush point
3279// tracking. Some DS reads may be used in the same iteration (creating
3280// "flush points"), but others remain unflushed at the backedge. When a DS
3281// read is consumed in the same iteration, it and all prior reads are
3282// "flushed" (FIFO order). No DS writes are allowed in the loop.
3283// TODO: Find a way to extend to multi-block loops.
3284PreheaderFlushFlags
3285SIInsertWaitcnts::getPreheaderFlushFlags(MachineLoop *ML,
3286 const WaitcntBrackets &Brackets) {
3287 PreheaderFlushFlags Flags;
3288 bool HasVMemLoad = false;
3289 bool HasVMemStore = false;
3290 bool UsesVgprVMEMLoadedOutside = false;
3291 bool UsesVgprDSReadOutside = false;
3292 bool VMemInvalidated = false;
3293 // DS optimization only applies to GFX12+ where DS_CNT is separate.
3294 // Tracking status for "no DS read in loop" or "pure DS prefetch
3295 // (use only in next iteration)".
3296 bool TrackSimpleDSOpt = ST.hasExtendedWaitCounts();
3297 DenseSet<MCRegUnit> VgprUse;
3298 DenseSet<MCRegUnit> VgprDefVMEM;
3299 DenseSet<MCRegUnit> VgprDefDS;
3300
3301 // Track DS reads for prefetch pattern with flush points (single-block only).
3302 // Keeps track of the last DS read (position counted from the top of the loop)
3303 // to each VGPR. Read is considered consumed (and thus needs flushing) if
3304 // the dest register has a use or is overwritten (by any later opertions).
3305 DenseMap<MCRegUnit, unsigned> LastDSReadPositionMap;
3306 unsigned DSReadPosition = 0;
3307 bool IsSingleBlock = ML->getNumBlocks() == 1;
3308 bool TrackDSFlushPoint = ST.hasExtendedWaitCounts() && IsSingleBlock;
3309 unsigned LastDSFlushPosition = 0;
3310
3311 for (MachineBasicBlock *MBB : ML->blocks()) {
3312 for (MachineInstr &MI : *MBB) {
3313 if (isVMEMOrFlatVMEM(MI)) {
3314 HasVMemLoad |= MI.mayLoad();
3315 HasVMemStore |= MI.mayStore();
3316 }
3317 // TODO: Can we relax DSStore check? There may be cases where
3318 // these DS stores are drained prior to the end of MBB (or loop).
3319 if (mayStoreIncrementingDSCNT(MI)) {
3320 // Early exit if none of the optimizations are feasible.
3321 // Otherwise, set tracking status appropriately and continue.
3322 if (VMemInvalidated)
3323 return Flags;
3324 TrackSimpleDSOpt = false;
3325 TrackDSFlushPoint = false;
3326 }
3327 bool IsDSRead = isDSRead(MI);
3328 if (IsDSRead)
3329 ++DSReadPosition;
3330
3331 // Helper: if RU has a pending DS read, update LastDSFlushPosition
3332 auto updateDSReadFlushTracking = [&](MCRegUnit RU) {
3333 if (!TrackDSFlushPoint)
3334 return;
3335 if (auto It = LastDSReadPositionMap.find(Val: RU);
3336 It != LastDSReadPositionMap.end()) {
3337 // RU defined by DSRead is used or overwritten. Need to complete
3338 // the read, if not already implied by a later DSRead (to any RU)
3339 // needing to complete in FIFO order.
3340 LastDSFlushPosition = std::max(a: LastDSFlushPosition, b: It->second);
3341 }
3342 };
3343
3344 for (const MachineOperand &Op : MI.all_uses()) {
3345 if (Op.isDebug() || !TRI.isVectorRegister(MRI, Reg: Op.getReg()))
3346 continue;
3347 // Vgpr use
3348 for (MCRegUnit RU : TRI.regunits(Reg: Op.getReg().asMCReg())) {
3349 // If we find a register that is loaded inside the loop, 1. and 2.
3350 // are invalidated.
3351 if (VgprDefVMEM.contains(V: RU))
3352 VMemInvalidated = true;
3353
3354 // Check for DS reads used inside the loop
3355 if (VgprDefDS.contains(V: RU))
3356 TrackSimpleDSOpt = false;
3357
3358 // Early exit if all optimizations are invalidated
3359 if (VMemInvalidated && !TrackSimpleDSOpt && !TrackDSFlushPoint)
3360 return Flags;
3361
3362 // Check for flush points (DS read used in same iteration)
3363 updateDSReadFlushTracking(RU);
3364
3365 VgprUse.insert(V: RU);
3366 // Check if this register has a pending VMEM load from outside the
3367 // loop (value loaded outside and used inside).
3368 VMEMID ID = toVMEMID(RU);
3369 if (Brackets.hasPendingVMEM(ID, T: AMDGPU::LOAD_CNT) ||
3370 Brackets.hasPendingVMEM(ID, T: AMDGPU::SAMPLE_CNT) ||
3371 Brackets.hasPendingVMEM(ID, T: AMDGPU::BVH_CNT))
3372 UsesVgprVMEMLoadedOutside = true;
3373 // Check if loaded outside the loop via DS (not VMEM/FLAT).
3374 // Only consider it a DS read if there's no pending VMEM load for
3375 // this register, since FLAT can set both counters.
3376 else if (Brackets.hasPendingVMEM(ID, T: AMDGPU::DS_CNT))
3377 UsesVgprDSReadOutside = true;
3378 }
3379 }
3380
3381 // VMem load vgpr def
3382 if (isVMEMOrFlatVMEM(MI) && MI.mayLoad()) {
3383 for (const MachineOperand &Op : MI.all_defs()) {
3384 for (MCRegUnit RU : TRI.regunits(Reg: Op.getReg().asMCReg())) {
3385 // If we find a register that is loaded inside the loop, 1. and 2.
3386 // are invalidated.
3387 if (VgprUse.contains(V: RU))
3388 VMemInvalidated = true;
3389 VgprDefVMEM.insert(V: RU);
3390 }
3391 }
3392 // Early exit if all optimizations are invalidated
3393 if (VMemInvalidated && !TrackSimpleDSOpt && !TrackDSFlushPoint)
3394 return Flags;
3395 }
3396
3397 // DS read vgpr def
3398 // Note: Unlike VMEM, we DON'T invalidate when VgprUse.contains(RegNo).
3399 // If USE comes before DEF, it's the prefetch pattern (use value from
3400 // previous iteration, read for next iteration). We should still flush
3401 // in preheader so iteration 1 doesn't need to wait inside the loop.
3402 // Only invalidate when DEF comes before USE (same-iteration consumption,
3403 // checked above when processing uses).
3404 if (IsDSRead || TrackDSFlushPoint) {
3405 for (const MachineOperand &Op : MI.all_defs()) {
3406 if (!TRI.isVectorRegister(MRI, Reg: Op.getReg()))
3407 continue;
3408 for (MCRegUnit RU : TRI.regunits(Reg: Op.getReg().asMCReg())) {
3409 // Check for overwrite of pending DS read (flush point) by any
3410 // instruction
3411 updateDSReadFlushTracking(RU);
3412 if (IsDSRead) {
3413 VgprDefDS.insert(V: RU);
3414 if (TrackDSFlushPoint)
3415 LastDSReadPositionMap[RU] = DSReadPosition;
3416 }
3417 }
3418 }
3419 }
3420 }
3421 }
3422
3423 // VMEM flush decision
3424 if (!VMemInvalidated && UsesVgprVMEMLoadedOutside &&
3425 ((!ST.hasVscnt() && HasVMemStore && !HasVMemLoad) ||
3426 (HasVMemLoad && ST.hasVmemWriteVgprInOrder())))
3427 Flags.FlushVmCnt = true;
3428
3429 // DS flush decision:
3430 // Simple DS Opt: flush if loop uses DS read values from outside
3431 // and either has no DS reads in the loop, or DS reads whose results
3432 // are not used in the loop.
3433 bool SimpleDSOpt = TrackSimpleDSOpt && UsesVgprDSReadOutside;
3434 // Prefetch with flush points: some DS reads used in same iteration,
3435 // but unflushed reads remain at backedge
3436 bool HasUnflushedDSReads = DSReadPosition > LastDSFlushPosition;
3437 bool DSFlushPointPrefetch =
3438 TrackDSFlushPoint && UsesVgprDSReadOutside && HasUnflushedDSReads;
3439
3440 if (SimpleDSOpt || DSFlushPointPrefetch)
3441 Flags.FlushDsCnt = true;
3442
3443 return Flags;
3444}
3445
3446bool SIInsertWaitcntsLegacy::runOnMachineFunction(MachineFunction &MF) {
3447 auto &MLI = getAnalysis<MachineLoopInfoWrapperPass>().getLI();
3448 auto &PDT =
3449 getAnalysis<MachinePostDominatorTreeWrapperPass>().getPostDomTree();
3450 AliasAnalysis *AA = nullptr;
3451 if (auto *AAR = getAnalysisIfAvailable<AAResultsWrapperPass>())
3452 AA = &AAR->getAAResults();
3453
3454 return SIInsertWaitcnts(MLI, PDT, AA, MF).run();
3455}
3456
3457PreservedAnalyses
3458SIInsertWaitcntsPass::run(MachineFunction &MF,
3459 MachineFunctionAnalysisManager &MFAM) {
3460 auto &MLI = MFAM.getResult<MachineLoopAnalysis>(IR&: MF);
3461 auto &PDT = MFAM.getResult<MachinePostDominatorTreeAnalysis>(IR&: MF);
3462 auto *AA = MFAM.getResult<FunctionAnalysisManagerMachineFunctionProxy>(IR&: MF)
3463 .getManager()
3464 .getCachedResult<AAManager>(IR&: MF.getFunction());
3465
3466 if (!SIInsertWaitcnts(MLI, PDT, AA, MF).run())
3467 return PreservedAnalyses::all();
3468
3469 return getMachineFunctionPassPreservedAnalyses()
3470 .preserveSet<CFGAnalyses>()
3471 .preserve<AAManager>();
3472}
3473
3474bool SIInsertWaitcnts::run() {
3475 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
3476
3477 AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(GPU: ST.getCPU());
3478
3479 // Initialize hardware limits first, as they're needed by the generators.
3480 Limits = AMDGPU::HardwareLimits(IV);
3481
3482 if (ST.hasExtendedWaitCounts()) {
3483 IsExpertMode = AMDGPU::isExpertSchedulingMode(ST, F: MF.getFunction());
3484 MaxCounter = IsExpertMode ? AMDGPU::NUM_EXPERT_INST_CNTS
3485 : AMDGPU::NUM_EXTENDED_INST_CNTS;
3486 // Initialize WCG per MF. It contains state that depends on MF attributes.
3487 WCG = std::make_unique<WaitcntGeneratorGFX12Plus>(args&: MF, args&: MaxCounter, args&: Limits,
3488 args&: IsExpertMode);
3489 } else {
3490 MaxCounter = AMDGPU::NUM_NORMAL_INST_CNTS;
3491 // Initialize WCG per MF. It contains state that depends on MF attributes.
3492 WCG = std::make_unique<WaitcntGeneratorPreGFX12>(
3493 args&: MF, args: AMDGPU::NUM_NORMAL_INST_CNTS, args&: Limits);
3494 }
3495
3496 SmemAccessCounter = getCounterFromEvent(E: HWEvents::SMEM_ACCESS);
3497
3498 bool Modified = false;
3499
3500 MachineBasicBlock &EntryBB = MF.front();
3501
3502 if (!MFI->isEntryFunction() &&
3503 !MF.getFunction().hasFnAttribute(Kind: Attribute::Naked)) {
3504 // Wait for any outstanding memory operations that the input registers may
3505 // depend on. We can't track them and it's better to do the wait after the
3506 // costly call sequence.
3507
3508 // TODO: Could insert earlier and schedule more liberally with operations
3509 // that only use caller preserved registers.
3510 MachineBasicBlock::iterator I = EntryBB.begin();
3511 while (I != EntryBB.end() && I->isMetaInstruction())
3512 ++I;
3513
3514 if (ST.hasExtendedWaitCounts()) {
3515 BuildMI(BB&: EntryBB, I, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_WAIT_LOADCNT_DSCNT))
3516 .addImm(Val: 0);
3517 for (auto CT : inst_counter_types(MaxCounter: AMDGPU::NUM_EXTENDED_INST_CNTS)) {
3518 if (CT == AMDGPU::LOAD_CNT || CT == AMDGPU::DS_CNT ||
3519 CT == AMDGPU::STORE_CNT || CT == AMDGPU::X_CNT ||
3520 CT == AMDGPU::ASYNC_CNT || CT == AMDGPU::TENSOR_CNT)
3521 continue;
3522
3523 if (!ST.hasImageInsts() &&
3524 (CT == AMDGPU::EXP_CNT || CT == AMDGPU::SAMPLE_CNT ||
3525 CT == AMDGPU::BVH_CNT))
3526 continue;
3527
3528 BuildMI(BB&: EntryBB, I, MIMD: DebugLoc(),
3529 MCID: TII.get(Opcode: instrsForExtendedCounterTypes[CT]))
3530 .addImm(Val: 0);
3531 }
3532 if (IsExpertMode) {
3533 unsigned Enc = AMDGPU::DepCtr::encodeFieldVaVdst(VaVdst: 0, STI: ST);
3534 Enc = AMDGPU::DepCtr::encodeFieldVmVsrc(Encoded: Enc, VmVsrc: 0);
3535 BuildMI(BB&: EntryBB, I, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
3536 .addImm(Val: Enc);
3537 }
3538 } else {
3539 BuildMI(BB&: EntryBB, I, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_WAITCNT)).addImm(Val: 0);
3540 }
3541
3542 auto NonKernelInitialState = std::make_unique<WaitcntBrackets>(args: this);
3543 NonKernelInitialState->setStateOnFunctionEntryOrReturn();
3544 BlockInfos[&EntryBB].Incoming = std::move(NonKernelInitialState);
3545
3546 Modified = true;
3547 }
3548
3549 // Keep iterating over the blocks in reverse post order, inserting and
3550 // updating s_waitcnt where needed, until a fix point is reached.
3551 for (auto *MBB : ReversePostOrderTraversal<MachineFunction *>(&MF))
3552 BlockInfos.try_emplace(Key: MBB);
3553
3554 std::unique_ptr<WaitcntBrackets> Brackets;
3555 bool Repeat;
3556 do {
3557 Repeat = false;
3558
3559 for (auto BII = BlockInfos.begin(), BIE = BlockInfos.end(); BII != BIE;
3560 ++BII) {
3561 MachineBasicBlock *MBB = BII->first;
3562 BlockInfo &BI = BII->second;
3563 if (!BI.Dirty)
3564 continue;
3565
3566 if (BI.Incoming) {
3567 if (!Brackets)
3568 Brackets = std::make_unique<WaitcntBrackets>(args&: *BI.Incoming);
3569 else
3570 *Brackets = *BI.Incoming;
3571 } else {
3572 if (!Brackets) {
3573 Brackets = std::make_unique<WaitcntBrackets>(args: this);
3574 } else {
3575 // Reinitialize in-place. N.B. do not do this by assigning from a
3576 // temporary because the WaitcntBrackets class is large and it could
3577 // cause this function to use an unreasonable amount of stack space.
3578 Brackets->~WaitcntBrackets();
3579 new (Brackets.get()) WaitcntBrackets(this);
3580 }
3581 }
3582
3583 if (ST.hasWaitXcnt() && ST.isXNACKEnabled())
3584 Modified |= removeRedundantSoftXcnts(Block&: *MBB);
3585 Modified |= insertWaitcntInBlock(MF, Block&: *MBB, ScoreBrackets&: *Brackets);
3586 BI.Dirty = false;
3587
3588 if (Brackets->hasPendingEvent()) {
3589 BlockInfo *MoveBracketsToSucc = nullptr;
3590 for (MachineBasicBlock *Succ : MBB->successors()) {
3591 auto *SuccBII = BlockInfos.find(Key: Succ);
3592 BlockInfo &SuccBI = SuccBII->second;
3593 if (!SuccBI.Incoming) {
3594 SuccBI.Dirty = true;
3595 if (SuccBII <= BII) {
3596 LLVM_DEBUG(dbgs() << "Repeat on backedge without merge\n");
3597 Repeat = true;
3598 }
3599 if (!MoveBracketsToSucc) {
3600 MoveBracketsToSucc = &SuccBI;
3601 } else {
3602 SuccBI.Incoming = std::make_unique<WaitcntBrackets>(args&: *Brackets);
3603 }
3604 } else {
3605 LLVM_DEBUG({
3606 dbgs() << "Try to merge ";
3607 MBB->printName(dbgs());
3608 dbgs() << " into ";
3609 Succ->printName(dbgs());
3610 dbgs() << '\n';
3611 });
3612 if (SuccBI.Incoming->merge(Other: *Brackets)) {
3613 SuccBI.Dirty = true;
3614 if (SuccBII <= BII) {
3615 LLVM_DEBUG(dbgs() << "Repeat on backedge with merge\n");
3616 Repeat = true;
3617 }
3618 }
3619 }
3620 }
3621 if (MoveBracketsToSucc)
3622 MoveBracketsToSucc->Incoming = std::move(Brackets);
3623 }
3624 }
3625 } while (Repeat);
3626
3627 Modified |= WCG->eraseDeferredWaitcnts();
3628
3629 if (ST.hasScalarStores()) {
3630 SmallVector<MachineBasicBlock *, 4> EndPgmBlocks;
3631 bool HaveScalarStores = false;
3632
3633 for (MachineBasicBlock &MBB : MF) {
3634 for (MachineInstr &MI : MBB) {
3635 if (!HaveScalarStores && TII.isScalarStore(MI))
3636 HaveScalarStores = true;
3637
3638 if (MI.getOpcode() == AMDGPU::S_ENDPGM ||
3639 MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG)
3640 EndPgmBlocks.push_back(Elt: &MBB);
3641 }
3642 }
3643
3644 if (HaveScalarStores) {
3645 // If scalar writes are used, the cache must be flushed or else the next
3646 // wave to reuse the same scratch memory can be clobbered.
3647 //
3648 // Insert s_dcache_wb at wave termination points if there were any scalar
3649 // stores, and only if the cache hasn't already been flushed. This could
3650 // be improved by looking across blocks for flushes in postdominating
3651 // blocks from the stores but an explicitly requested flush is probably
3652 // very rare.
3653 for (MachineBasicBlock *MBB : EndPgmBlocks) {
3654 bool SeenDCacheWB = false;
3655
3656 for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
3657 I != E; ++I) {
3658 if (I->getOpcode() == AMDGPU::S_DCACHE_WB)
3659 SeenDCacheWB = true;
3660 else if (TII.isScalarStore(MI: *I))
3661 SeenDCacheWB = false;
3662
3663 // FIXME: It would be better to insert this before a waitcnt if any.
3664 if ((I->getOpcode() == AMDGPU::S_ENDPGM ||
3665 I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) &&
3666 !SeenDCacheWB) {
3667 Modified = true;
3668 BuildMI(BB&: *MBB, I, MIMD: I->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_DCACHE_WB));
3669 }
3670 }
3671 }
3672 }
3673 }
3674
3675 if (IsExpertMode) {
3676 // Enable expert scheduling on function entry. To satisfy ABI requirements
3677 // and to allow calls between function with different expert scheduling
3678 // settings, disable it around calls and before returns.
3679
3680 MachineBasicBlock::iterator I = EntryBB.begin();
3681 while (I != EntryBB.end() && I->isMetaInstruction())
3682 ++I;
3683 setSchedulingMode(MBB&: EntryBB, I, ExpertMode: true);
3684
3685 for (MachineInstr *MI : CallInsts) {
3686 MachineBasicBlock &MBB = *MI->getParent();
3687 setSchedulingMode(MBB, I: MI, ExpertMode: false);
3688 setSchedulingMode(MBB, I: std::next(x: MI->getIterator()), ExpertMode: true);
3689 }
3690
3691 for (MachineInstr *MI : ReturnInsts)
3692 setSchedulingMode(MBB&: *MI->getParent(), I: MI, ExpertMode: false);
3693
3694 Modified = true;
3695 }
3696
3697 // Deallocate the VGPRs before previously identified S_ENDPGM instructions.
3698 // This is done in different ways depending on how the VGPRs were allocated
3699 // (i.e. whether we're in dynamic VGPR mode or not).
3700 // Skip deallocation if kernel is waveslot limited vs VGPR limited. A short
3701 // waveslot limited kernel runs slower with the deallocation.
3702 if (!WCG->isOptNone() && MFI->isDynamicVGPREnabled()) {
3703 for (auto [MI, _] : EndPgmInsts) {
3704 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
3705 MCID: TII.get(Opcode: AMDGPU::S_ALLOC_VGPR))
3706 .addImm(Val: 0);
3707 Modified = true;
3708 }
3709 } else if (!WCG->isOptNone() &&
3710 ST.getGeneration() >= AMDGPUSubtarget::GFX11 &&
3711 (MF.getFrameInfo().hasCalls() ||
3712 ST.getOccupancyWithNumVGPRs(
3713 VGPRs: TRI.getNumUsedPhysRegs(MRI, RC: AMDGPU::VGPR_32RegClass),
3714 /*IsDynamicVGPR=*/DynamicVGPRBlockSize: false) < ST.getMaxWavesPerEU())) {
3715 for (auto [MI, Flag] : EndPgmInsts) {
3716 if (Flag) {
3717 if (ST.requiresNopBeforeDeallocVGPRs()) {
3718 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
3719 MCID: TII.get(Opcode: AMDGPU::S_NOP))
3720 .addImm(Val: 0);
3721 }
3722 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
3723 MCID: TII.get(Opcode: AMDGPU::S_SENDMSG))
3724 .addImm(Val: AMDGPU::SendMsg::ID_DEALLOC_VGPRS_GFX11Plus);
3725 Modified = true;
3726 }
3727 }
3728 }
3729
3730 if (MFI->isEntryFunction()) {
3731 MachineBasicBlock::iterator InsertPt = EntryBB.begin();
3732
3733 if (ST.hasWaitXcnt() && ST.isXNACKEnabled()) {
3734 // Set REPLAY_MODE (bit 25) in MODE register to enable multi-group XNACK
3735 // replay. This aligns hardware behavior with the compiler's s_wait_xcnt
3736 // insertion logic, which assumes multi-group mode by default.
3737 uint64_t RegEncoding =
3738 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 25, Values: 1);
3739 BuildMI(BB&: EntryBB, I: InsertPt, MIMD: DebugLoc(),
3740 MCID: TII.get(Opcode: AMDGPU::S_SETREG_IMM32_B32))
3741 .addImm(Val: 1)
3742 .addImm(Val: RegEncoding);
3743 Modified = true;
3744 }
3745
3746 if (ST.hasRequiresInitialUnclausedVmem()) {
3747 // Hardware entrypoints must begin with a specific sequence:
3748 // S_MOV_B64 S[64:65], 0
3749 // V_NOP
3750 // GLOBAL_PREFETCH_B8 V0, S[64:65] SCOPE:SCOPE_SE TH:TH_LOAD_RT
3751 BuildMI(BB&: EntryBB, I: InsertPt, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B64),
3752 DestReg: AMDGPU::SGPR64_SGPR65)
3753 .addImm(Val: 0);
3754 BuildMI(BB&: EntryBB, I: InsertPt, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_NOP_e32));
3755 BuildMI(BB&: EntryBB, I: InsertPt, MIMD: DebugLoc(),
3756 MCID: TII.get(Opcode: AMDGPU::GLOBAL_PREFETCH_B8_SADDR))
3757 .addReg(RegNo: AMDGPU::SGPR64_SGPR65)
3758 .addReg(RegNo: AMDGPU::VGPR0, Flags: RegState::Undef)
3759 .addImm(Val: 0)
3760 .addImm(Val: AMDGPU::CPol::SCOPE_SE | AMDGPU::CPol::TH_RT);
3761 Modified = true;
3762 }
3763 }
3764
3765 return Modified;
3766}
3767