1//===- SIInsertWaitcnts.cpp - Insert Wait Instructions --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Insert wait instructions for memory reads and writes.
11///
12/// Memory reads and writes are issued asynchronously, so we need to insert
13/// S_WAITCNT instructions when we want to access any of their results or
14/// overwrite any register that's used asynchronously.
15///
16/// TODO: This pass currently keeps one timeline per hardware counter. A more
17/// finely-grained approach that keeps one timeline per event type could
18/// sometimes get away with generating weaker s_waitcnt instructions. For
19/// example, when both SMEM and LDS are in flight and we need to wait for
20/// the i-th-last LDS instruction, then an lgkmcnt(i) is actually sufficient,
21/// but the pass will currently generate a conservative lgkmcnt(0) because
22/// multiple event types are in flight.
23//
24//===----------------------------------------------------------------------===//
25
26#include "AMDGPU.h"
27#include "AMDGPUHWEvents.h"
28#include "AMDGPUWaitcntUtils.h"
29#include "GCNSubtarget.h"
30#include "SIMachineFunctionInfo.h"
31#include "Utils/AMDGPUBaseInfo.h"
32#include "llvm/Analysis/AliasAnalysis.h"
33#include "llvm/CodeGen/MachineFrameInfo.h"
34#include "llvm/CodeGen/MachineLoopInfo.h"
35#include "llvm/CodeGen/MachinePassManager.h"
36#include "llvm/CodeGen/MachinePostDominators.h"
37#include "llvm/IR/Dominators.h"
38#include "llvm/InitializePasses.h"
39#include "llvm/TargetParser/AMDGPUTargetParser.h"
40
41using namespace llvm;
42
43using HWEvents = AMDGPU::HWEvents;
44
45#define DEBUG_TYPE "si-insert-waitcnts"
46
47static cl::opt<bool>
48 ForceEmitZeroFlag("amdgpu-waitcnt-forcezero",
49 cl::desc("Force all waitcnt instrs to be emitted as "
50 "s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"),
51 cl::init(Val: false), cl::Hidden);
52
53static cl::opt<bool> ForceEmitZeroLoadFlag(
54 "amdgpu-waitcnt-load-forcezero",
55 cl::desc("Force all waitcnt load counters to wait until 0"),
56 cl::init(Val: false), cl::Hidden);
57
58static cl::opt<bool> ExpertSchedulingModeFlag(
59 "amdgpu-expert-scheduling-mode",
60 cl::desc("Enable expert scheduling mode 2 for all functions (GFX12+ only)"),
61 cl::init(Val: false), cl::Hidden);
62
63namespace {
64
65template <typename EmitWaitcntFn>
66static void EmitExpandedWaitcnt(unsigned Outstanding, unsigned Target,
67 EmitWaitcntFn &&EmitWaitcnt) {
68 // Emit waitcnts from (Outstanding - 1) down to Target.
69 for (unsigned I = Outstanding - 1; I > Target && I != ~0u; --I)
70 EmitWaitcnt(I);
71 EmitWaitcnt(Target);
72}
73
74/// Integer IDs used to track vector memory locations we may have to wait on.
75/// Encoded as u16 chunks:
76///
77/// [0, REGUNITS_END ): MCRegUnit
78/// [LDSDMA_BEGIN, LDSDMA_END ) : LDS DMA IDs
79///
80/// NOTE: The choice of encoding these as "u16 chunks" is arbitrary.
81/// It gives (2 << 16) - 1 entries per category which is more than enough
82/// for all register units. MCPhysReg is u16 so we don't even support >u16
83/// physical register numbers at this time, let alone >u16 register units.
84/// In any case, an assertion in "WaitcntBrackets" ensures REGUNITS_END
85/// is enough for all register units.
86using VMEMID = uint32_t;
87
88enum : VMEMID {
89 TRACKINGID_RANGE_LEN = (1 << 16),
90
91 // Important: MCRegUnits must always be tracked starting from 0, as we
92 // need to be able to convert between a MCRegUnit and a VMEMID freely.
93 REGUNITS_BEGIN = 0,
94 REGUNITS_END = REGUNITS_BEGIN + TRACKINGID_RANGE_LEN,
95
96 // Note for LDSDMA: LDSDMA_BEGIN corresponds to the "common"
97 // entry, which is updated for all LDS DMA operations encountered.
98 // Specific LDS DMA IDs start at LDSDMA_BEGIN + 1.
99 NUM_LDSDMA = TRACKINGID_RANGE_LEN,
100 LDSDMA_BEGIN = REGUNITS_END,
101 LDSDMA_END = LDSDMA_BEGIN + NUM_LDSDMA,
102};
103
104/// Convert a MCRegUnit to a VMEMID.
105static constexpr VMEMID toVMEMID(MCRegUnit RU) {
106 return static_cast<unsigned>(RU);
107}
108
109} // namespace
110
111namespace {
112
113// Maps values of InstCounterType to the instruction that waits on that
114// counter. Only used if GCNSubtarget::hasExtendedWaitCounts()
115// returns true, and does not cover VA_VDST or VM_VSRC.
116static const unsigned
117 instrsForExtendedCounterTypes[AMDGPU::NUM_EXTENDED_INST_CNTS] = {
118 AMDGPU::S_WAIT_LOADCNT, AMDGPU::S_WAIT_DSCNT,
119 AMDGPU::S_WAIT_EXPCNT, AMDGPU::S_WAIT_STORECNT,
120 AMDGPU::S_WAIT_SAMPLECNT, AMDGPU::S_WAIT_BVHCNT,
121 AMDGPU::S_WAIT_KMCNT, AMDGPU::S_WAIT_XCNT,
122 AMDGPU::S_WAIT_ASYNCCNT, AMDGPU::S_WAIT_TENSORCNT};
123
124// ASYNCMARK and WAIT_ASYNCMARK are meta instructions that emit no hardware
125// code but still need to be processed by this pass for async vmcnt tracking.
126static bool isNonWaitcntMetaInst(const MachineInstr &MI) {
127 switch (MI.getOpcode()) {
128 case AMDGPU::ASYNCMARK:
129 case AMDGPU::WAIT_ASYNCMARK:
130 return false;
131 default:
132 return MI.isMetaInstruction();
133 }
134}
135
136static bool updateVMCntOnly(const MachineInstr &Inst) {
137 return (SIInstrInfo::isVMEM(MI: Inst) && !SIInstrInfo::isFLAT(MI: Inst)) ||
138 SIInstrInfo::isFLATGlobal(MI: Inst) || SIInstrInfo::isFLATScratch(MI: Inst);
139}
140
141#ifndef NDEBUG
142static bool isNormalMode(AMDGPU::InstCounterType MaxCounter) {
143 return MaxCounter == AMDGPU::NUM_NORMAL_INST_CNTS;
144}
145#endif // NDEBUG
146
147class WaitcntBrackets;
148
149// This abstracts the logic for generating and updating S_WAIT* instructions
150// away from the analysis that determines where they are needed. This was
151// done because the set of counters and instructions for waiting on them
152// underwent a major shift with gfx12, sufficiently so that having this
153// abstraction allows the main analysis logic to be simpler than it would
154// otherwise have had to become.
155class WaitcntGenerator {
156protected:
157 const GCNSubtarget &ST;
158 const SIInstrInfo &TII;
159 AMDGPU::IsaVersion IV;
160 AMDGPU::InstCounterType MaxCounter;
161 bool OptNone;
162 bool ExpandWaitcntProfiling = false;
163 const AMDGPU::HardwareLimits &Limits;
164
165 // Soft waitcnts whose erasures are postponed so we can confirm if they're
166 // truly redundant or if they're waiting on a loop-carried dependency.
167 DenseSet<MachineInstr *> DeferredErasures;
168
169public:
170 WaitcntGenerator() = delete;
171 WaitcntGenerator(const WaitcntGenerator &) = delete;
172 WaitcntGenerator(const MachineFunction &MF,
173 AMDGPU::InstCounterType MaxCounter,
174 const AMDGPU::HardwareLimits &Limits)
175 : ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()),
176 IV(AMDGPU::getIsaVersion(GPU: ST.getCPU())), MaxCounter(MaxCounter),
177 OptNone(MF.getFunction().hasOptNone() ||
178 MF.getTarget().getOptLevel() == CodeGenOptLevel::None),
179 ExpandWaitcntProfiling(
180 MF.getFunction().hasFnAttribute(Kind: "amdgpu-expand-waitcnt-profiling")),
181 Limits(Limits) {}
182
183 // Return true if the current function should be compiled with no
184 // optimization.
185 bool isOptNone() const { return OptNone; }
186
187 unsigned getLimit(AMDGPU::InstCounterType E) const { return Limits.get(T: E); }
188
189 // Edits an existing sequence of wait count instructions according
190 // to an incoming Waitcnt value, which is itself updated to reflect
191 // any new wait count instructions which may need to be generated by
192 // WaitcntGenerator::createNewWaitcnt(). It will return true if any edits
193 // were made.
194 //
195 // This editing will usually be merely updated operands, but it may also
196 // delete instructions if the incoming Wait value indicates they are not
197 // needed. It may also remove existing instructions for which a wait
198 // is needed if it can be determined that it is better to generate new
199 // instructions later, as can happen on gfx12.
200 virtual bool
201 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
202 MachineInstr &OldWaitcntInstr, AMDGPU::Waitcnt &Wait,
203 MachineBasicBlock::instr_iterator It) = 0;
204
205 bool eraseDeferredWaitcnts() {
206 bool Modified = false;
207 for (MachineInstr *MI : DeferredErasures) {
208 MI->eraseFromParent();
209 Modified = true;
210 }
211 DeferredErasures.clear();
212 return Modified;
213 }
214
215 // Transform a soft waitcnt into a normal one.
216 bool promoteSoftWaitCnt(MachineInstr *Waitcnt) const;
217
218 // Generates new wait count instructions according to the value of
219 // Wait, returning true if any new instructions were created.
220 // ScoreBrackets is used for profiling expansion.
221 virtual bool createNewWaitcnt(MachineBasicBlock &Block,
222 MachineBasicBlock::instr_iterator It,
223 AMDGPU::Waitcnt Wait,
224 const WaitcntBrackets &ScoreBrackets) = 0;
225
226 // Returns the set of HWEvents that corresponds to counter \p T.
227 virtual HWEvents getWaitEvents(AMDGPU::InstCounterType T) const = 0;
228
229 /// \returns the counter that corresponds to event \p E.
230 AMDGPU::InstCounterType getCounterFromEvent(HWEvents E) const {
231 assert(E.size() == 1 && "Cannot handle a mask of events!");
232 for (auto T : AMDGPU::inst_counter_types()) {
233 if (getWaitEvents(T) & E)
234 return T;
235 }
236 llvm_unreachable("event type has no associated counter");
237 }
238
239 // Returns a new waitcnt with all counters except VScnt set to 0. If
240 // IncludeVSCnt is true, VScnt is set to 0, otherwise it is set to ~0u.
241 // AsyncCnt and TensorCnt always default to ~0u (don't wait for it). They
242 // are only updated when a call to @llvm.amdgcn.wait.asyncmark() is
243 // processed.
244 virtual AMDGPU::Waitcnt getAllZeroWaitcnt(bool IncludeVSCnt) const = 0;
245
246 virtual ~WaitcntGenerator() = default;
247};
248
249class WaitcntGeneratorPreGFX12 final : public WaitcntGenerator {
250 static constexpr const HWEvents
251 WaitEventMaskForInstPreGFX12[AMDGPU::NUM_INST_CNTS] = {
252 HWEvents::VMEM_READ_ACCESS | HWEvents::VMEM_SAMPLER_READ_ACCESS |
253 HWEvents::VMEM_BVH_READ_ACCESS | HWEvents::VMEM_INV_ACCESS,
254 HWEvents::SMEM_ACCESS | HWEvents::LDS_ACCESS | HWEvents::GDS_ACCESS |
255 HWEvents::SQ_MESSAGE,
256 HWEvents::EXP_GPR_LOCK | HWEvents::GDS_GPR_LOCK |
257 HWEvents::VMW_GPR_LOCK | HWEvents::EXP_PARAM_ACCESS |
258 HWEvents::EXP_POS_ACCESS | HWEvents::EXP_LDS_ACCESS,
259 HWEvents::VMEM_WRITE_ACCESS | HWEvents::SCRATCH_WRITE_ACCESS,
260 HWEvents::NONE,
261 HWEvents::NONE,
262 HWEvents::NONE,
263 HWEvents::NONE,
264 HWEvents::NONE,
265 HWEvents::NONE,
266 HWEvents::NONE,
267 HWEvents::NONE,
268 HWEvents::NONE};
269
270public:
271 using WaitcntGenerator::WaitcntGenerator;
272 bool applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
273 MachineInstr &OldWaitcntInstr,
274 AMDGPU::Waitcnt &Wait,
275 MachineBasicBlock::instr_iterator It) override;
276
277 bool createNewWaitcnt(MachineBasicBlock &Block,
278 MachineBasicBlock::instr_iterator It,
279 AMDGPU::Waitcnt Wait,
280 const WaitcntBrackets &ScoreBrackets) override;
281
282 HWEvents getWaitEvents(AMDGPU::InstCounterType T) const override {
283 HWEvents EVs = WaitEventMaskForInstPreGFX12[T];
284 if (T == AMDGPU::LOAD_CNT && !ST.hasVscnt())
285 EVs |= WaitEventMaskForInstPreGFX12[AMDGPU::STORE_CNT];
286 return EVs;
287 }
288
289 AMDGPU::Waitcnt getAllZeroWaitcnt(bool IncludeVSCnt) const override;
290};
291
292class WaitcntGeneratorGFX12Plus final : public WaitcntGenerator {
293protected:
294 bool IsExpertMode;
295 static constexpr const HWEvents
296 WaitEventMaskForInstGFX12Plus[AMDGPU::NUM_INST_CNTS] = {
297 HWEvents::VMEM_READ_ACCESS | HWEvents::VMEM_INV_ACCESS,
298 HWEvents::LDS_ACCESS | HWEvents::GDS_ACCESS,
299 HWEvents::EXP_GPR_LOCK | HWEvents::GDS_GPR_LOCK |
300 HWEvents::VMW_GPR_LOCK | HWEvents::EXP_PARAM_ACCESS |
301 HWEvents::EXP_POS_ACCESS | HWEvents::EXP_LDS_ACCESS,
302
303 HWEvents::VMEM_WRITE_ACCESS | HWEvents::SCRATCH_WRITE_ACCESS,
304 HWEvents::VMEM_SAMPLER_READ_ACCESS,
305 HWEvents::VMEM_BVH_READ_ACCESS,
306
307 HWEvents::SMEM_ACCESS | HWEvents::SQ_MESSAGE | HWEvents::SCC_WRITE,
308 HWEvents::VMEM_GROUP | HWEvents::SMEM_GROUP,
309 HWEvents::ASYNC_ACCESS,
310 HWEvents::TENSOR_ACCESS,
311 HWEvents::VGPR_CSMACC_READ | HWEvents::VGPR_DPMACC_READ |
312 HWEvents::VGPR_TRANS_READ | HWEvents::VGPR_XDL_READ,
313 HWEvents::VGPR_CSMACC_WRITE | HWEvents::VGPR_DPMACC_WRITE |
314 HWEvents::VGPR_TRANS_WRITE | HWEvents::VGPR_XDL_WRITE,
315 HWEvents::VGPR_LDS_READ | HWEvents::VGPR_FLAT_READ |
316 HWEvents::VGPR_VMEM_READ};
317
318public:
319 WaitcntGeneratorGFX12Plus() = delete;
320 WaitcntGeneratorGFX12Plus(const MachineFunction &MF,
321 AMDGPU::InstCounterType MaxCounter,
322 const AMDGPU::HardwareLimits &Limits,
323 bool IsExpertMode)
324 : WaitcntGenerator(MF, MaxCounter, Limits), IsExpertMode(IsExpertMode) {}
325
326 bool applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
327 MachineInstr &OldWaitcntInstr,
328 AMDGPU::Waitcnt &Wait,
329 MachineBasicBlock::instr_iterator It) override;
330
331 bool createNewWaitcnt(MachineBasicBlock &Block,
332 MachineBasicBlock::instr_iterator It,
333 AMDGPU::Waitcnt Wait,
334 const WaitcntBrackets &ScoreBrackets) override;
335
336 HWEvents getWaitEvents(AMDGPU::InstCounterType T) const override {
337 return WaitEventMaskForInstGFX12Plus[T];
338 }
339
340 AMDGPU::Waitcnt getAllZeroWaitcnt(bool IncludeVSCnt) const override;
341};
342
343// Flags indicating which counters should be flushed in a loop preheader.
344struct PreheaderFlushFlags {
345 bool FlushVmCnt = false;
346 bool FlushDsCnt = false;
347};
348
349class SIInsertWaitcnts {
350 DenseMap<const Value *, MachineBasicBlock *> SLoadAddresses;
351 DenseMap<MachineBasicBlock *, PreheaderFlushFlags> PreheadersToFlush;
352 MachineLoopInfo &MLI;
353 MachinePostDominatorTree &PDT;
354 AliasAnalysis *AA = nullptr;
355 MachineFunction &MF;
356
357 struct BlockInfo {
358 std::unique_ptr<WaitcntBrackets> Incoming;
359 bool Dirty = true;
360 BlockInfo() = default;
361 BlockInfo(BlockInfo &&) = default;
362 BlockInfo &operator=(BlockInfo &&) = default;
363 ~BlockInfo();
364 };
365
366 MapVector<MachineBasicBlock *, BlockInfo> BlockInfos;
367
368 bool ForceEmitWaitcnt[AMDGPU::NUM_INST_CNTS] = {};
369
370 std::unique_ptr<WaitcntGenerator> WCG;
371
372 // Remember call and return instructions in the function.
373 DenseSet<MachineInstr *> CallInsts;
374 DenseSet<MachineInstr *> ReturnInsts;
375
376 // Remember all S_ENDPGM instructions. The boolean flag is true if there might
377 // be outstanding stores but definitely no outstanding scratch stores, to help
378 // with insertion of DEALLOC_VGPRS messages.
379 DenseMap<MachineInstr *, bool> EndPgmInsts;
380
381 AMDGPU::HardwareLimits Limits;
382
383public:
384 const GCNSubtarget &ST;
385 const SIInstrInfo &TII;
386 const SIRegisterInfo &TRI;
387 const MachineRegisterInfo &MRI;
388 AMDGPU::InstCounterType SmemAccessCounter;
389 AMDGPU::InstCounterType MaxCounter;
390 bool IsExpertMode = false;
391 const bool TgSplit;
392
393 SIInsertWaitcnts(MachineLoopInfo &MLI, MachinePostDominatorTree &PDT,
394 AliasAnalysis *AA, MachineFunction &MF)
395 : MLI(MLI), PDT(PDT), AA(AA), MF(MF), ST(MF.getSubtarget<GCNSubtarget>()),
396 TII(*ST.getInstrInfo()), TRI(TII.getRegisterInfo()),
397 MRI(MF.getRegInfo()),
398 TgSplit(ST.hasTgSplitSupport() &&
399 AMDGPU::isTgSplitEnabled(F: MF.getFunction())) {}
400
401 const AMDGPU::HardwareLimits &getLimits() const { return Limits; }
402
403 PreheaderFlushFlags getPreheaderFlushFlags(MachineLoop *ML,
404 const WaitcntBrackets &Brackets);
405 PreheaderFlushFlags isPreheaderToFlush(MachineBasicBlock &MBB,
406 const WaitcntBrackets &ScoreBrackets);
407 bool isVMEMOrFlatVMEM(const MachineInstr &MI) const;
408 bool isDSRead(const MachineInstr &MI) const;
409 bool mayStoreIncrementingDSCNT(const MachineInstr &MI) const;
410 bool run();
411
412 bool isAsync(const MachineInstr &MI) const {
413 if (!SIInstrInfo::isLDSDMA(MI))
414 return false;
415 if (SIInstrInfo::usesASYNC_CNT(MI))
416 return true;
417 const MachineOperand *Async =
418 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::IsAsync);
419 return Async && (Async->getImm());
420 }
421
422 bool isNonAsyncLdsDmaWrite(const MachineInstr &MI) const {
423 return SIInstrInfo::mayWriteLDSThroughDMA(MI) && !isAsync(MI);
424 }
425
426 bool isAsyncLdsDmaWrite(const MachineInstr &MI) const {
427 return SIInstrInfo::mayWriteLDSThroughDMA(MI) && isAsync(MI);
428 }
429
430 bool shouldUpdateAsyncMark(const MachineInstr &MI,
431 AMDGPU::InstCounterType T) const {
432 if (SIInstrInfo::usesTENSOR_CNT(MI))
433 return T == AMDGPU::TENSOR_CNT;
434 if (!isAsyncLdsDmaWrite(MI))
435 return false;
436 if (SIInstrInfo::usesASYNC_CNT(MI))
437 return T == AMDGPU::ASYNC_CNT;
438 return T == AMDGPU::LOAD_CNT;
439 }
440
441 bool isVmemAccess(const MachineInstr &MI) const;
442 bool generateWaitcntInstBefore(MachineInstr &MI,
443 WaitcntBrackets &ScoreBrackets,
444 MachineInstr *OldWaitcntInstr,
445 PreheaderFlushFlags FlushFlags);
446 bool generateWaitcnt(AMDGPU::Waitcnt Wait,
447 MachineBasicBlock::instr_iterator It,
448 MachineBasicBlock &Block, WaitcntBrackets &ScoreBrackets,
449 MachineInstr *OldWaitcntInstr);
450 void updateEventWaitcntAfter(MachineInstr &Inst,
451 WaitcntBrackets *ScoreBrackets);
452 bool isNextENDPGM(MachineBasicBlock::instr_iterator It,
453 MachineBasicBlock *Block) const;
454 bool insertForcedWaitAfter(MachineInstr &Inst, MachineBasicBlock &Block,
455 WaitcntBrackets &ScoreBrackets);
456 bool insertWaitcntInBlock(MachineFunction &MF, MachineBasicBlock &Block,
457 WaitcntBrackets &ScoreBrackets);
458 /// Removes redundant Soft Xcnt Waitcnts in \p Block emitted by the Memory
459 /// Legalizer. Returns true if block was modified.
460 bool removeRedundantSoftXcnts(MachineBasicBlock &Block);
461 void setSchedulingMode(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
462 bool ExpertMode) const;
463 HWEvents getWaitEvents(AMDGPU::InstCounterType T) const {
464 return WCG->getWaitEvents(T);
465 }
466 AMDGPU::InstCounterType getCounterFromEvent(HWEvents E) const {
467 return WCG->getCounterFromEvent(E);
468 }
469};
470
471// This objects maintains the current score brackets of each wait counter, and
472// a per-register scoreboard for each wait counter.
473//
474// We also maintain the latest score for every event type that can change the
475// waitcnt in order to know if there are multiple types of events within
476// the brackets. When multiple types of event happen in the bracket,
477// wait count may get decreased out of order, therefore we need to put in
478// "s_waitcnt 0" before use.
479class WaitcntBrackets {
480public:
481 WaitcntBrackets(const SIInsertWaitcnts *Context) : Context(Context) {
482 assert(Context->TRI.getNumRegUnits() < REGUNITS_END);
483 }
484
485#ifndef NDEBUG
486 ~WaitcntBrackets() {
487 unsigned NumUnusedVmem = 0, NumUnusedSGPRs = 0;
488 for (auto &[ID, Val] : VMem) {
489 if (Val.empty())
490 ++NumUnusedVmem;
491 }
492 for (auto &[ID, Val] : SGPRs) {
493 if (Val.empty())
494 ++NumUnusedSGPRs;
495 }
496
497 if (NumUnusedVmem || NumUnusedSGPRs) {
498 errs() << "WaitcntBracket had unused entries at destruction time: "
499 << NumUnusedVmem << " VMem and " << NumUnusedSGPRs
500 << " SGPR unused entries\n";
501 std::abort();
502 }
503 }
504#endif
505
506 bool isSmemCounter(AMDGPU::InstCounterType T) const {
507 return T == Context->SmemAccessCounter || T == AMDGPU::X_CNT;
508 }
509
510 unsigned getOutstanding(AMDGPU::InstCounterType T) const {
511 return ScoreUBs[T] - ScoreLBs[T];
512 }
513
514 bool hasPendingVMEM(VMEMID ID, AMDGPU::InstCounterType T) const {
515 return getVMemScore(TID: ID, T) > getScoreLB(T);
516 }
517
518 /// \Return true if we have no score entries for counter \p T.
519 bool empty(AMDGPU::InstCounterType T) const { return getScoreRange(T) == 0; }
520
521private:
522 unsigned getScoreLB(AMDGPU::InstCounterType T) const {
523 assert(T < AMDGPU::NUM_INST_CNTS);
524 return ScoreLBs[T];
525 }
526
527 unsigned getScoreUB(AMDGPU::InstCounterType T) const {
528 assert(T < AMDGPU::NUM_INST_CNTS);
529 return ScoreUBs[T];
530 }
531
532 unsigned getScoreRange(AMDGPU::InstCounterType T) const {
533 return getScoreUB(T) - getScoreLB(T);
534 }
535
536 unsigned getSGPRScore(MCRegUnit RU, AMDGPU::InstCounterType T) const {
537 auto It = SGPRs.find(Val: RU);
538 return It != SGPRs.end() ? It->second.get(T) : 0;
539 }
540
541 unsigned getVMemScore(VMEMID TID, AMDGPU::InstCounterType T) const {
542 auto It = VMem.find(Val: TID);
543 return It != VMem.end() ? It->second.Scores[T] : 0;
544 }
545
546public:
547 // Do some internal consistency checks.
548 void verify() {
549 // These track the same underlying hardware counter so the score ranges
550 // should be identical.
551 assert(getScoreLB(AMDGPU::VA_VDST_RD) == getScoreLB(AMDGPU::VA_VDST_WR));
552 assert(getScoreUB(AMDGPU::VA_VDST_RD) == getScoreUB(AMDGPU::VA_VDST_WR));
553 }
554
555 bool merge(const WaitcntBrackets &Other);
556
557 bool counterOutOfOrder(AMDGPU::InstCounterType T) const;
558 void simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const {
559 simplifyWaitcnt(CheckWait: Wait, UpdateWait&: Wait);
560 }
561 void simplifyWaitcnt(const AMDGPU::Waitcnt &CheckWait,
562 AMDGPU::Waitcnt &UpdateWait) const;
563 void simplifyWaitcnt(AMDGPU::InstCounterType T, unsigned &Count) const;
564 void simplifyWaitcnt(AMDGPU::Waitcnt &Wait, AMDGPU::InstCounterType T) const;
565 void simplifyXcnt(const AMDGPU::Waitcnt &CheckWait,
566 AMDGPU::Waitcnt &UpdateWait) const;
567 void simplifyVmVsrc(const AMDGPU::Waitcnt &CheckWait,
568 AMDGPU::Waitcnt &UpdateWait) const;
569 // GLOBAL_INV/BUFFER_INV increment loadcnt/vmcnt but do not write VGPRs, so
570 // waits that only drain these events can be omitted.
571 void simplifyVmemInvOnlyWait(AMDGPU::Waitcnt &Wait) const {
572 // LOAD_CNT also includes store events on targets without VScnt.
573 HWEvents LoadEvents =
574 PendingEvents & (Context->getWaitEvents(T: AMDGPU::LOAD_CNT) -
575 Context->getWaitEvents(T: AMDGPU::STORE_CNT));
576 if (LoadEvents == HWEvents::VMEM_INV_ACCESS)
577 Wait.clear(T: AMDGPU::LOAD_CNT);
578 }
579
580 void determineWaitForPhysReg(AMDGPU::InstCounterType T, MCPhysReg Reg,
581 AMDGPU::Waitcnt &Wait,
582 const MachineInstr &MI) const;
583 MCPhysReg determineVGPR16Dependency(const MachineInstr &MI,
584 AMDGPU::InstCounterType T,
585 MCPhysReg Reg) const;
586 void determineWaitForLDSDMA(AMDGPU::InstCounterType T, VMEMID TID,
587 AMDGPU::Waitcnt &Wait) const;
588 AMDGPU::Waitcnt determineAsyncWait(unsigned N);
589 void tryClearSCCWriteEvent(MachineInstr *Inst);
590
591 void applyWaitcnt(const AMDGPU::Waitcnt &Wait);
592 void applyWaitcnt(AMDGPU::InstCounterType T, unsigned Count);
593 void applyWaitcnt(const AMDGPU::Waitcnt &Wait, AMDGPU::InstCounterType T);
594 void updateByEvent(HWEvents E, MachineInstr &MI);
595 void recordAsyncMark(MachineInstr &MI);
596
597 HWEvents getPendingEvents() const { return PendingEvents; }
598 bool hasPendingEvent() const { return PendingEvents.any(); }
599 bool hasPendingEvent(HWEvents E) const { return PendingEvents.contains(Other: E); }
600 bool hasPendingEvent(AMDGPU::InstCounterType T) const {
601 bool HasPending = (PendingEvents & Context->getWaitEvents(T)).any();
602 assert(HasPending == !empty(T) &&
603 "Expected pending events iff scoreboard is not empty");
604 return HasPending;
605 }
606
607 bool hasMixedPendingEvents(AMDGPU::InstCounterType T) const {
608 HWEvents Events = PendingEvents & Context->getWaitEvents(T);
609 // Return true if more than one bit is set in Events.
610 return Events.size() > 1;
611 }
612
613 bool hasPendingFlat() const {
614 return ((LastFlatDsCnt > ScoreLBs[AMDGPU::DS_CNT] &&
615 LastFlatDsCnt <= ScoreUBs[AMDGPU::DS_CNT]) ||
616 (LastFlatLoadCnt > ScoreLBs[AMDGPU::LOAD_CNT] &&
617 LastFlatLoadCnt <= ScoreUBs[AMDGPU::LOAD_CNT]));
618 }
619
620 void setPendingFlat() {
621 LastFlatLoadCnt = ScoreUBs[AMDGPU::LOAD_CNT];
622 LastFlatDsCnt = ScoreUBs[AMDGPU::DS_CNT];
623 }
624
625 bool hasPendingGDS() const {
626 return LastGDS > ScoreLBs[AMDGPU::DS_CNT] &&
627 LastGDS <= ScoreUBs[AMDGPU::DS_CNT];
628 }
629
630 unsigned getPendingGDSWait() const {
631 return std::min(a: getScoreUB(T: AMDGPU::DS_CNT) - LastGDS,
632 b: getLimit(T: AMDGPU::DS_CNT) - 1);
633 }
634
635 void setPendingGDS() { LastGDS = ScoreUBs[AMDGPU::DS_CNT]; }
636
637 // Return true if there might be pending writes to the vgpr-interval by VMEM
638 // instructions where the HWEvents in VGPRContext are not contained in E.
639 bool hasDifferentVGPRPendingEvents(MCPhysReg Reg, HWEvents E) const {
640 for (MCRegUnit RU : regunits(Reg)) {
641 auto It = VMem.find(Val: toVMEMID(RU));
642 if (It != VMem.end() && (It->second.VGPRPendingEvents & ~E).any())
643 return true;
644 }
645 return false;
646 }
647
648 void clearVGPRPendingEvents(MCPhysReg Reg) {
649 for (MCRegUnit RU : regunits(Reg)) {
650 if (auto It = VMem.find(Val: toVMEMID(RU)); It != VMem.end()) {
651 It->second.VGPRPendingEvents = HWEvents::NONE;
652 if (It->second.empty())
653 VMem.erase(I: It);
654 }
655 }
656 }
657
658 void setStateOnFunctionEntryOrReturn() {
659 setScoreUB(T: AMDGPU::STORE_CNT,
660 Val: getScoreUB(T: AMDGPU::STORE_CNT) + getLimit(T: AMDGPU::STORE_CNT));
661 PendingEvents |= Context->getWaitEvents(T: AMDGPU::STORE_CNT);
662 }
663
664 ArrayRef<const MachineInstr *> getLDSDMAStores() const {
665 return LDSDMAStores;
666 }
667
668 bool hasPointSampleAccel(const MachineInstr &MI) const;
669 bool hasPointSamplePendingVmemTypes(const MachineInstr &MI,
670 MCPhysReg RU) const;
671
672 void print(raw_ostream &) const;
673 void dump() const { print(dbgs()); }
674
675 // Free up memory by removing empty entries from the DenseMap that track event
676 // scores.
677 void purgeEmptyTrackingData();
678
679private:
680 unsigned getLimit(AMDGPU::InstCounterType T) const {
681 return Context->getLimits().get(T);
682 }
683
684 struct MergeInfo {
685 unsigned OldLB;
686 unsigned OtherLB;
687 unsigned MyShift;
688 unsigned OtherShift;
689 };
690
691 using CounterValueArray = std::array<unsigned, AMDGPU::NUM_INST_CNTS>;
692
693 void determineWaitForScore(AMDGPU::InstCounterType T, unsigned Score,
694 AMDGPU::Waitcnt &Wait) const;
695
696 static bool mergeScore(const MergeInfo &M, unsigned &Score,
697 unsigned OtherScore);
698 bool mergeAsyncMarks(ArrayRef<MergeInfo> MergeInfos,
699 ArrayRef<CounterValueArray> OtherMarks);
700
701 iterator_range<MCRegUnitIterator> regunits(MCPhysReg Reg) const {
702 assert(Reg != AMDGPU::SCC && "Shouldn't be used on SCC");
703 if (!Context->TRI.isInAllocatableClass(RegNo: Reg))
704 return {{}, {}};
705 return Context->TRI.regunits(Reg);
706 }
707
708 void setScoreLB(AMDGPU::InstCounterType T, unsigned Val) {
709 assert(T < AMDGPU::NUM_INST_CNTS);
710 ScoreLBs[T] = Val;
711 }
712
713 void setScoreUB(AMDGPU::InstCounterType T, unsigned Val) {
714 assert(T < AMDGPU::NUM_INST_CNTS);
715 ScoreUBs[T] = Val;
716
717 if (T != AMDGPU::EXP_CNT)
718 return;
719
720 if (getScoreRange(T: AMDGPU::EXP_CNT) > getLimit(T: AMDGPU::EXP_CNT))
721 ScoreLBs[AMDGPU::EXP_CNT] =
722 ScoreUBs[AMDGPU::EXP_CNT] - getLimit(T: AMDGPU::EXP_CNT);
723 }
724
725 void setRegScore(MCPhysReg Reg, AMDGPU::InstCounterType T, unsigned Val) {
726 const SIRegisterInfo &TRI = Context->TRI;
727 if (Reg == AMDGPU::SCC) {
728 SCCScore = Val;
729 } else if (TRI.isVectorRegister(MRI: Context->MRI, Reg)) {
730 for (MCRegUnit RU : regunits(Reg))
731 VMem[toVMEMID(RU)].Scores[T] = Val;
732 } else if (TRI.isSGPRReg(MRI: Context->MRI, Reg)) {
733 for (MCRegUnit RU : regunits(Reg))
734 SGPRs[RU].get(T) = Val;
735 } else {
736 llvm_unreachable("Register cannot be tracked/unknown register!");
737 }
738 }
739
740 void setVMemScore(VMEMID TID, AMDGPU::InstCounterType T, unsigned Val) {
741 VMem[TID].Scores[T] = Val;
742 }
743
744 void setScoreByOperand(const MachineOperand &Op,
745 AMDGPU::InstCounterType CntTy, unsigned Val);
746
747 const SIInsertWaitcnts *Context;
748
749 unsigned ScoreLBs[AMDGPU::NUM_INST_CNTS] = {0};
750 unsigned ScoreUBs[AMDGPU::NUM_INST_CNTS] = {0};
751 HWEvents PendingEvents;
752 // Remember the last flat memory operation.
753 unsigned LastFlatDsCnt = 0;
754 unsigned LastFlatLoadCnt = 0;
755 // Remember the last GDS operation.
756 unsigned LastGDS = 0;
757
758 // The score tracking logic is fragmented as follows:
759 // - VMem: VGPR RegUnits and LDS DMA IDs, see the VMEMID encoding.
760 // - SGPRs: SGPR RegUnits
761 // - SCC: Non-allocatable and not general purpose: not a SGPR.
762 //
763 // For the VMem case, if the key is within the range of LDS DMA IDs,
764 // then the corresponding index into the `LDSDMAStores` vector below is:
765 // Key - LDSDMA_BEGIN - 1
766 // This is because LDSDMA_BEGIN is a generic entry and does not have an
767 // associated MachineInstr.
768 //
769 // TODO: Could we track SCC alongside SGPRs so it's not longer a special case?
770
771 struct VMEMInfo {
772 // Scores for all instruction counters. Zero-initialized.
773 CounterValueArray Scores{};
774 // For VGPRs, we need to track an additional fine-grained set of pending
775 // events.
776 HWEvents VGPRPendingEvents;
777
778 bool empty() const {
779 return all_of(Range: Scores, P: equal_to(Arg: 0)) && !VGPRPendingEvents;
780 }
781 };
782
783 /// Wait cnt scores for every sgpr, the DS_CNT (corresponding to LGKMcnt
784 /// pre-gfx12) or KM_CNT (gfx12+ only), and X_CNT (gfx1250) are relevant.
785 class SGPRInfo {
786 /// Either DS_CNT or KM_CNT score.
787 unsigned ScoreDsKmCnt = 0;
788 unsigned ScoreXCnt = 0;
789
790 public:
791 unsigned get(AMDGPU::InstCounterType T) const {
792 assert(
793 (T == AMDGPU::DS_CNT || T == AMDGPU::KM_CNT || T == AMDGPU::X_CNT) &&
794 "Invalid counter");
795 return T == AMDGPU::X_CNT ? ScoreXCnt : ScoreDsKmCnt;
796 }
797 unsigned &get(AMDGPU::InstCounterType T) {
798 assert(
799 (T == AMDGPU::DS_CNT || T == AMDGPU::KM_CNT || T == AMDGPU::X_CNT) &&
800 "Invalid counter");
801 return T == AMDGPU::X_CNT ? ScoreXCnt : ScoreDsKmCnt;
802 }
803
804 bool empty() const { return !ScoreDsKmCnt && !ScoreXCnt; }
805 };
806
807 DenseMap<VMEMID, VMEMInfo> VMem; // VGPR + LDS DMA
808 DenseMap<MCRegUnit, SGPRInfo> SGPRs;
809
810 // Reg score for SCC.
811 unsigned SCCScore = 0;
812 // The unique instruction that has an SCC write pending, if there is one.
813 const MachineInstr *PendingSCCWrite = nullptr;
814
815 // Store representative LDS DMA operations. The only useful info here is
816 // alias info. One store is kept per unique AAInfo.
817 SmallVector<const MachineInstr *> LDSDMAStores;
818
819 // State of all counters at each async mark encountered so far.
820 SmallVector<CounterValueArray> AsyncMarks;
821
822 // But in the rare pathological case, a nest of loops that pushes marks
823 // without waiting on any mark can cause AsyncMarks to grow very large. We cap
824 // it to a reasonable limit. We can tune this later or potentially introduce a
825 // user option to control the value.
826 static constexpr unsigned MaxAsyncMarks = 16;
827
828 // Track the upper bound score for async operations that are not part of a
829 // mark yet. Initialized to all zeros.
830 CounterValueArray AsyncScore{};
831};
832
833SIInsertWaitcnts::BlockInfo::~BlockInfo() = default;
834
835class SIInsertWaitcntsLegacy : public MachineFunctionPass {
836public:
837 static char ID;
838 SIInsertWaitcntsLegacy() : MachineFunctionPass(ID) {}
839
840 bool runOnMachineFunction(MachineFunction &MF) override;
841
842 StringRef getPassName() const override {
843 return "SI insert wait instructions";
844 }
845
846 void getAnalysisUsage(AnalysisUsage &AU) const override {
847 AU.setPreservesCFG();
848 AU.addRequired<MachineLoopInfoWrapperPass>();
849 AU.addRequired<MachinePostDominatorTreeWrapperPass>();
850 AU.addUsedIfAvailable<AAResultsWrapperPass>();
851 AU.addPreserved<AAResultsWrapperPass>();
852 MachineFunctionPass::getAnalysisUsage(AU);
853 }
854};
855
856} // end anonymous namespace
857
858void WaitcntBrackets::setScoreByOperand(const MachineOperand &Op,
859 AMDGPU::InstCounterType CntTy,
860 unsigned Score) {
861 setRegScore(Reg: Op.getReg().asMCReg(), T: CntTy, Val: Score);
862}
863
864// Return true if the subtarget is one that enables Point Sample Acceleration
865// and the MachineInstr passed in is one to which it might be applied (the
866// hardware makes this decision based on several factors, but we can't determine
867// this at compile time, so we have to assume it might be applied if the
868// instruction supports it).
869bool WaitcntBrackets::hasPointSampleAccel(const MachineInstr &MI) const {
870 if (!Context->ST.hasPointSampleAccel() || !SIInstrInfo::isMIMG(MI))
871 return false;
872
873 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc: MI.getOpcode());
874 const AMDGPU::MIMGBaseOpcodeInfo *BaseInfo =
875 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode);
876 return BaseInfo->PointSampleAccel;
877}
878
879// Return true if the subtarget enables Point Sample Acceleration, the supplied
880// MachineInstr is one to which it might be applied and the supplied interval is
881// one that has outstanding writes to vmem-types different than VMEM_NOSAMPLER
882// (this is the type that a point sample accelerated instruction effectively
883// becomes)
884bool WaitcntBrackets::hasPointSamplePendingVmemTypes(const MachineInstr &MI,
885 MCPhysReg Reg) const {
886 if (!hasPointSampleAccel(MI))
887 return false;
888
889 return hasDifferentVGPRPendingEvents(Reg, E: HWEvents::VMEM_READ_ACCESS);
890}
891
892void WaitcntBrackets::updateByEvent(HWEvents E, MachineInstr &Inst) {
893 assert(E.size() == 1 && "Expected singular event!");
894 AMDGPU::InstCounterType T = Context->getCounterFromEvent(E);
895 assert(T < Context->MaxCounter);
896
897 unsigned UB = getScoreUB(T);
898 unsigned Increment = 1;
899 if ((T == AMDGPU::VA_VDST_RD || T == AMDGPU::VA_VDST_WR) &&
900 AMDGPU::getHasMatrixScale(Opc: Inst.getOpcode()) &&
901 Context->ST.hasVOP3PX2IncrementsVaVdstTwice()) {
902 // V_WMMA_SCALE instructions use VOP3PX2 encoding. Hardware treats this as
903 // two VOP3P instructions and increments VA_VDST twice.
904 Increment = 2;
905 }
906 unsigned CurrScore = UB + Increment;
907 if (CurrScore == 0)
908 report_fatal_error(reason: "InsertWaitcnt score wraparound");
909 // PendingEvents and ScoreUB need to be update regardless if this event
910 // changes the score of a register or not.
911 // Examples including vm_cnt when buffer-store or lgkm_cnt when send-message.
912 PendingEvents |= E;
913 setScoreUB(T, Val: CurrScore);
914
915 const SIRegisterInfo &TRI = Context->TRI;
916 const MachineRegisterInfo &MRI = Context->MRI;
917 const SIInstrInfo &TII = Context->TII;
918
919 if (T == AMDGPU::EXP_CNT) {
920 // Put score on the source vgprs. If this is a store, just use those
921 // specific register(s).
922 if (TII.isDS(MI: Inst) && Inst.mayLoadOrStore()) {
923 // All GDS operations must protect their address register (same as
924 // export.)
925 if (const auto *AddrOp = TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::addr))
926 setScoreByOperand(Op: *AddrOp, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
927
928 if (Inst.mayStore()) {
929 if (const auto *Data0 =
930 TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data0))
931 setScoreByOperand(Op: *Data0, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
932 if (const auto *Data1 =
933 TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data1))
934 setScoreByOperand(Op: *Data1, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
935 } else if (SIInstrInfo::isAtomicRet(MI: Inst) && !SIInstrInfo::isGWS(MI: Inst) &&
936 Inst.getOpcode() != AMDGPU::DS_APPEND &&
937 Inst.getOpcode() != AMDGPU::DS_CONSUME &&
938 Inst.getOpcode() != AMDGPU::DS_ORDERED_COUNT) {
939 for (const MachineOperand &Op : Inst.all_uses()) {
940 if (TRI.isVectorRegister(MRI, Reg: Op.getReg()))
941 setScoreByOperand(Op, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
942 }
943 }
944 } else if (TII.isFLAT(MI: Inst)) {
945 if (Inst.mayStore()) {
946 setScoreByOperand(Op: *TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data),
947 CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
948 } else if (SIInstrInfo::isAtomicRet(MI: Inst)) {
949 setScoreByOperand(Op: *TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data),
950 CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
951 }
952 } else if (TII.isMIMG(MI: Inst)) {
953 if (Inst.mayStore()) {
954 setScoreByOperand(Op: Inst.getOperand(i: 0), CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
955 } else if (SIInstrInfo::isAtomicRet(MI: Inst)) {
956 setScoreByOperand(Op: *TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data),
957 CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
958 }
959 } else if (TII.isMTBUF(MI: Inst)) {
960 if (Inst.mayStore())
961 setScoreByOperand(Op: Inst.getOperand(i: 0), CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
962 } else if (TII.isMUBUF(MI: Inst)) {
963 if (Inst.mayStore()) {
964 setScoreByOperand(Op: Inst.getOperand(i: 0), CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
965 } else if (SIInstrInfo::isAtomicRet(MI: Inst)) {
966 setScoreByOperand(Op: *TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data),
967 CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
968 }
969 } else if (TII.isLDSDIR(MI: Inst)) {
970 // LDSDIR instructions attach the score to the destination.
971 setScoreByOperand(Op: *TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::vdst),
972 CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
973 } else {
974 if (TII.isEXP(MI: Inst)) {
975 // For export the destination registers are really temps that
976 // can be used as the actual source after export patching, so
977 // we need to treat them like sources and set the EXP_CNT
978 // score.
979 for (MachineOperand &DefMO : Inst.all_defs()) {
980 if (TRI.isVGPR(MRI, Reg: DefMO.getReg())) {
981 setScoreByOperand(Op: DefMO, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
982 }
983 }
984 }
985 for (const MachineOperand &Op : Inst.all_uses()) {
986 if (TRI.isVectorRegister(MRI, Reg: Op.getReg()))
987 setScoreByOperand(Op, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
988 }
989 }
990 } else if (T == AMDGPU::X_CNT) {
991 HWEvents OtherEvent =
992 E == HWEvents::SMEM_GROUP ? HWEvents::VMEM_GROUP : HWEvents::SMEM_GROUP;
993 if (PendingEvents.contains(Other: OtherEvent)) {
994 // Hardware inserts an implicit xcnt between interleaved
995 // SMEM and VMEM operations. So there will never be
996 // outstanding address translations for both SMEM and
997 // VMEM at the same time.
998 setScoreLB(T, Val: getScoreUB(T) - 1);
999 PendingEvents -= OtherEvent;
1000 }
1001 for (const MachineOperand &Op : Inst.all_uses())
1002 setScoreByOperand(Op, CntTy: T, Score: CurrScore);
1003 } else if (T == AMDGPU::VA_VDST_RD || T == AMDGPU::VA_VDST_WR ||
1004 T == AMDGPU::VM_VSRC) {
1005 // Match the score to the VGPR destination or source registers as
1006 // appropriate
1007 for (const MachineOperand &Op : Inst.operands()) {
1008 if (!Op.isReg())
1009 continue;
1010
1011 // Skip based on counter type and operand type
1012 if (T == AMDGPU::VA_VDST_RD && Op.isDef())
1013 continue; // RD tracks reads only
1014 if (T == AMDGPU::VA_VDST_WR && Op.isUse())
1015 continue; // WR tracks writes only
1016 if (T == AMDGPU::VM_VSRC && Op.isDef())
1017 continue;
1018
1019 if (TRI.isVectorRegister(MRI: Context->MRI, Reg: Op.getReg()))
1020 setScoreByOperand(Op, CntTy: T, Score: CurrScore);
1021 }
1022 } else /* LGKM_CNT || EXP_CNT || VS_CNT || NUM_INST_CNTS */ {
1023 // Match the score to the destination registers.
1024 //
1025 // Check only explicit operands. Stores, especially spill stores, include
1026 // implicit uses and defs of their super registers which would create an
1027 // artificial dependency, while these are there only for register liveness
1028 // accounting purposes.
1029 //
1030 // Special cases where implicit register defs exists, such as M0 or VCC,
1031 // but none with memory instructions.
1032 for (const MachineOperand &Op : Inst.defs()) {
1033 if (T == AMDGPU::LOAD_CNT || T == AMDGPU::SAMPLE_CNT ||
1034 T == AMDGPU::BVH_CNT) {
1035 if (!TRI.isVectorRegister(MRI, Reg: Op.getReg())) // TODO: add wrapper
1036 continue;
1037 if (updateVMCntOnly(Inst)) {
1038 // updateVMCntOnly should only leave us with VGPRs
1039 // MUBUF, MTBUF, MIMG, FlatGlobal, and FlatScratch only have VGPR/AGPR
1040 // defs.
1041 assert(TRI.isVectorRegister(MRI, Op.getReg()));
1042 HWEvents VGPRContext =
1043 AMDGPU::getSimplifiedVMEMEventsFor(Inst, TII: Context->TII);
1044 // If instruction can have Point Sample Accel applied, we have to flag
1045 // this with another potential dependency
1046 if (hasPointSampleAccel(MI: Inst))
1047 VGPRContext |= HWEvents::VMEM_READ_ACCESS;
1048 for (MCRegUnit RU : regunits(Reg: Op.getReg().asMCReg()))
1049 VMem[toVMEMID(RU)].VGPRPendingEvents |= VGPRContext;
1050 }
1051 }
1052 setScoreByOperand(Op, CntTy: T, Score: CurrScore);
1053 }
1054 if (Inst.mayStore() &&
1055 (TII.isDS(MI: Inst) || Context->isNonAsyncLdsDmaWrite(MI: Inst))) {
1056 // MUBUF and FLAT LDS DMA operations need a wait on vmcnt before LDS
1057 // written can be accessed. A load from LDS to VMEM does not need a wait.
1058 //
1059 // The "Slot" is the offset from LDSDMA_BEGIN. If it's non-zero, then
1060 // there is a MachineInstr in LDSDMAStores used to track this LDSDMA
1061 // store. The "Slot" is the index into LDSDMAStores + 1.
1062 unsigned Slot = 0;
1063 for (const auto *MemOp : Inst.memoperands()) {
1064 if (!MemOp->isStore() ||
1065 MemOp->getAddrSpace() != AMDGPUAS::LOCAL_ADDRESS)
1066 continue;
1067 // Comparing just AA info does not guarantee memoperands are equal
1068 // in general, but this is so for LDS DMA in practice.
1069 auto AAI = MemOp->getAAInfo();
1070 // Alias scope information gives a way to definitely identify an
1071 // original memory object and practically produced in the module LDS
1072 // lowering pass. If there is no scope available we will not be able
1073 // to disambiguate LDS aliasing as after the module lowering all LDS
1074 // is squashed into a single big object.
1075 if (!AAI || !AAI.Scope)
1076 break;
1077 for (unsigned I = 0, E = LDSDMAStores.size(); I != E && !Slot; ++I) {
1078 for (const auto *MemOp : LDSDMAStores[I]->memoperands()) {
1079 if (MemOp->isStore() && AAI == MemOp->getAAInfo()) {
1080 Slot = I + 1;
1081 break;
1082 }
1083 }
1084 }
1085 if (Slot)
1086 break;
1087 // The slot may not be valid because it can be >= NUM_LDSDMA which
1088 // means the scoreboard cannot track it. We still want to preserve the
1089 // MI in order to check alias information, though.
1090 LDSDMAStores.push_back(Elt: &Inst);
1091 Slot = LDSDMAStores.size();
1092 break;
1093 }
1094 setVMemScore(TID: LDSDMA_BEGIN, T, Val: CurrScore);
1095 if (Slot && Slot < NUM_LDSDMA)
1096 setVMemScore(TID: LDSDMA_BEGIN + Slot, T, Val: CurrScore);
1097 }
1098
1099 if (Context->shouldUpdateAsyncMark(MI: Inst, T)) {
1100 AsyncScore[T] = CurrScore;
1101 }
1102
1103 if (SIInstrInfo::isSBarrierSCCWrite(Opcode: Inst.getOpcode())) {
1104 setRegScore(Reg: AMDGPU::SCC, T, Val: CurrScore);
1105 PendingSCCWrite = &Inst;
1106 }
1107 }
1108}
1109
1110void WaitcntBrackets::recordAsyncMark(MachineInstr &Inst) {
1111 // In the absence of loops, AsyncMarks can grow linearly with the program
1112 // until we encounter an ASYNCMARK_WAIT. We could drop the oldest mark above a
1113 // limit every time we push a new mark, but that seems like unnecessary work
1114 // in practical cases. We do separately truncate the array when processing a
1115 // loop, which should be sufficient.
1116 AsyncMarks.push_back(Elt: AsyncScore);
1117 LLVM_DEBUG({
1118 dbgs() << "recordAsyncMark:\n" << Inst;
1119 for (const auto &Mark : AsyncMarks) {
1120 llvm::interleaveComma(Mark, dbgs());
1121 dbgs() << '\n';
1122 }
1123 });
1124}
1125
1126void WaitcntBrackets::print(raw_ostream &OS) const {
1127 const GCNSubtarget &ST = Context->ST;
1128
1129 for (auto T : inst_counter_types(MaxCounter: Context->MaxCounter)) {
1130 unsigned SR = getScoreRange(T);
1131 switch (T) {
1132 case AMDGPU::LOAD_CNT:
1133 OS << " " << (ST.hasExtendedWaitCounts() ? "LOAD" : "VM") << "_CNT("
1134 << SR << "):";
1135 break;
1136 case AMDGPU::DS_CNT:
1137 OS << " " << (ST.hasExtendedWaitCounts() ? "DS" : "LGKM") << "_CNT("
1138 << SR << "):";
1139 break;
1140 case AMDGPU::EXP_CNT:
1141 OS << " EXP_CNT(" << SR << "):";
1142 break;
1143 case AMDGPU::STORE_CNT:
1144 OS << " " << (ST.hasExtendedWaitCounts() ? "STORE" : "VS") << "_CNT("
1145 << SR << "):";
1146 break;
1147 case AMDGPU::SAMPLE_CNT:
1148 OS << " SAMPLE_CNT(" << SR << "):";
1149 break;
1150 case AMDGPU::BVH_CNT:
1151 OS << " BVH_CNT(" << SR << "):";
1152 break;
1153 case AMDGPU::KM_CNT:
1154 OS << " KM_CNT(" << SR << "):";
1155 break;
1156 case AMDGPU::X_CNT:
1157 OS << " X_CNT(" << SR << "):";
1158 break;
1159 case AMDGPU::ASYNC_CNT:
1160 OS << " ASYNC_CNT(" << SR << "):";
1161 break;
1162 case AMDGPU::VA_VDST_RD:
1163 OS << " VA_VDST_RD(" << SR << "): ";
1164 break;
1165 case AMDGPU::VA_VDST_WR:
1166 OS << " VA_VDST_WR(" << SR << "): ";
1167 break;
1168 case AMDGPU::VM_VSRC:
1169 OS << " VM_VSRC(" << SR << "): ";
1170 break;
1171 default:
1172 OS << " UNKNOWN(" << SR << "):";
1173 break;
1174 }
1175
1176 if (SR != 0) {
1177 // Print vgpr scores.
1178 unsigned LB = getScoreLB(T);
1179
1180 SmallVector<VMEMID> SortedVMEMIDs(VMem.keys());
1181 sort(C&: SortedVMEMIDs);
1182
1183 for (auto ID : SortedVMEMIDs) {
1184 unsigned RegScore = VMem.at(Val: ID).Scores[T];
1185 if (RegScore <= LB)
1186 continue;
1187 unsigned RelScore = RegScore - LB - 1;
1188 if (ID < REGUNITS_END) {
1189 OS << ' ' << RelScore << ':'
1190 << printRegUnit(Unit: static_cast<MCRegUnit>(ID), TRI: &Context->TRI);
1191 } else {
1192 assert(ID >= LDSDMA_BEGIN && ID < LDSDMA_END &&
1193 "Unhandled/unexpected ID value!");
1194 OS << ' ' << RelScore << ":LDSDMA" << ID;
1195 }
1196 }
1197
1198 // Also need to print sgpr scores for lgkm_cnt or xcnt.
1199 if (isSmemCounter(T)) {
1200 SmallVector<MCRegUnit> SortedSMEMIDs(SGPRs.keys());
1201 sort(C&: SortedSMEMIDs);
1202 for (auto ID : SortedSMEMIDs) {
1203 unsigned RegScore = SGPRs.at(Val: ID).get(T);
1204 if (RegScore <= LB)
1205 continue;
1206 unsigned RelScore = RegScore - LB - 1;
1207 OS << ' ' << RelScore << ':'
1208 << printRegUnit(Unit: static_cast<MCRegUnit>(ID), TRI: &Context->TRI);
1209 }
1210 }
1211
1212 if (T == AMDGPU::KM_CNT && SCCScore > 0)
1213 OS << ' ' << SCCScore << ":scc";
1214 }
1215 OS << '\n';
1216 }
1217
1218 OS << "Pending Events: ";
1219 if (hasPendingEvent()) {
1220 OS << getPendingEvents();
1221 } else {
1222 OS << "none";
1223 }
1224 OS << '\n';
1225
1226 OS << "Async score: ";
1227 if (AsyncScore.empty())
1228 OS << "none";
1229 else
1230 llvm::interleaveComma(c: AsyncScore, os&: OS);
1231 OS << '\n';
1232
1233 OS << "Async marks: " << AsyncMarks.size() << '\n';
1234
1235 for (const auto &Mark : AsyncMarks) {
1236 for (auto T : AMDGPU::inst_counter_types()) {
1237 unsigned MarkedScore = Mark[T];
1238 switch (T) {
1239 case AMDGPU::LOAD_CNT:
1240 OS << " " << (ST.hasExtendedWaitCounts() ? "LOAD" : "VM")
1241 << "_CNT: " << MarkedScore;
1242 break;
1243 case AMDGPU::DS_CNT:
1244 OS << " " << (ST.hasExtendedWaitCounts() ? "DS" : "LGKM")
1245 << "_CNT: " << MarkedScore;
1246 break;
1247 case AMDGPU::EXP_CNT:
1248 OS << " EXP_CNT: " << MarkedScore;
1249 break;
1250 case AMDGPU::STORE_CNT:
1251 OS << " " << (ST.hasExtendedWaitCounts() ? "STORE" : "VS")
1252 << "_CNT: " << MarkedScore;
1253 break;
1254 case AMDGPU::SAMPLE_CNT:
1255 OS << " SAMPLE_CNT: " << MarkedScore;
1256 break;
1257 case AMDGPU::BVH_CNT:
1258 OS << " BVH_CNT: " << MarkedScore;
1259 break;
1260 case AMDGPU::KM_CNT:
1261 OS << " KM_CNT: " << MarkedScore;
1262 break;
1263 case AMDGPU::X_CNT:
1264 OS << " X_CNT: " << MarkedScore;
1265 break;
1266 case AMDGPU::ASYNC_CNT:
1267 OS << " ASYNC_CNT: " << MarkedScore;
1268 break;
1269 default:
1270 OS << " UNKNOWN: " << MarkedScore;
1271 break;
1272 }
1273 }
1274 OS << '\n';
1275 }
1276 OS << '\n';
1277}
1278
1279/// Simplify \p UpdateWait by removing waits that are redundant based on the
1280/// current WaitcntBrackets and any other waits specified in \p CheckWait.
1281void WaitcntBrackets::simplifyWaitcnt(const AMDGPU::Waitcnt &CheckWait,
1282 AMDGPU::Waitcnt &UpdateWait) const {
1283 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::LOAD_CNT);
1284 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::EXP_CNT);
1285 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::DS_CNT);
1286 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::STORE_CNT);
1287 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::SAMPLE_CNT);
1288 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::BVH_CNT);
1289 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::KM_CNT);
1290 simplifyXcnt(CheckWait, UpdateWait);
1291 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::VA_VDST_RD);
1292 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::VA_VDST_WR);
1293 simplifyVmVsrc(CheckWait, UpdateWait);
1294 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::ASYNC_CNT);
1295}
1296
1297void WaitcntBrackets::simplifyWaitcnt(AMDGPU::InstCounterType T,
1298 unsigned &Count) const {
1299 // The number of outstanding events for this type, T, can be calculated
1300 // as (UB - LB). If the current Count is greater than or equal to the number
1301 // of outstanding events, then the wait for this counter is redundant.
1302 if (Count >= getScoreRange(T))
1303 Count = ~0u;
1304}
1305
1306void WaitcntBrackets::simplifyWaitcnt(AMDGPU::Waitcnt &Wait,
1307 AMDGPU::InstCounterType T) const {
1308 unsigned Cnt = Wait.get(T);
1309 simplifyWaitcnt(T, Count&: Cnt);
1310 Wait.set(T, Val: Cnt);
1311}
1312
1313void WaitcntBrackets::simplifyXcnt(const AMDGPU::Waitcnt &CheckWait,
1314 AMDGPU::Waitcnt &UpdateWait) const {
1315 // Try to simplify xcnt further by checking for joint kmcnt and loadcnt
1316 // optimizations. On entry to a block with multiple predescessors, there may
1317 // be pending SMEM and VMEM events active at the same time.
1318 // In such cases, only clear one active event at a time.
1319 // TODO: Revisit xcnt optimizations for gfx1250.
1320 // Wait on XCNT is redundant if we are already waiting for a load to complete.
1321 // SMEM can return out of order, so only omit XCNT wait if we are waiting till
1322 // zero.
1323 if (CheckWait.get(T: AMDGPU::KM_CNT) == 0 &&
1324 hasPendingEvent(E: HWEvents::SMEM_GROUP))
1325 UpdateWait.set(T: AMDGPU::X_CNT, Val: ~0u);
1326 // If we have pending store we cannot optimize XCnt because we do not wait for
1327 // stores. VMEM loads retun in order, so if we only have loads XCnt is
1328 // decremented to the same number as LOADCnt.
1329 if (CheckWait.get(T: AMDGPU::LOAD_CNT) != ~0u &&
1330 hasPendingEvent(E: HWEvents::VMEM_GROUP) &&
1331 !hasPendingEvent(T: AMDGPU::STORE_CNT) &&
1332 CheckWait.get(T: AMDGPU::X_CNT) >= CheckWait.get(T: AMDGPU::LOAD_CNT))
1333 UpdateWait.set(T: AMDGPU::X_CNT, Val: ~0u);
1334 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::X_CNT);
1335}
1336
1337void WaitcntBrackets::simplifyVmVsrc(const AMDGPU::Waitcnt &CheckWait,
1338 AMDGPU::Waitcnt &UpdateWait) const {
1339 // Waiting for a VMEM counter (like LOAD_CNT) implies an equivalent wait for
1340 // VM_VSRC, because if the VMEM operation has completed then it must surely
1341 // have read its VGPR sources, but only if there are no other outstanding VMEM
1342 // operations that use a different counter (like SAMPLE_CNT).
1343 static constexpr AMDGPU::InstCounterType VmemCounters[] = {
1344 AMDGPU::LOAD_CNT, AMDGPU::STORE_CNT, AMDGPU::SAMPLE_CNT, AMDGPU::BVH_CNT,
1345 AMDGPU::DS_CNT, AMDGPU::ASYNC_CNT};
1346 HWEvents VmemEvents = llvm::accumulate(
1347 Range: VmemCounters, Init: HWEvents(), Op: [&](HWEvents Acc, AMDGPU::InstCounterType T) {
1348 return Acc | Context->getWaitEvents(T);
1349 });
1350 HWEvents PendingVmemEvents = PendingEvents & VmemEvents;
1351 for (AMDGPU::InstCounterType T : VmemCounters) {
1352 unsigned CheckCount = CheckWait.get(T);
1353 if (UpdateWait.get(T: AMDGPU::VM_VSRC) >= CheckCount &&
1354 (CheckCount == 0 || !counterOutOfOrder(T)) &&
1355 (PendingVmemEvents & ~Context->getWaitEvents(T)) == 0)
1356 UpdateWait.set(T: AMDGPU::VM_VSRC, Val: ~0u);
1357 }
1358
1359 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::VM_VSRC);
1360}
1361
1362void WaitcntBrackets::purgeEmptyTrackingData() {
1363 VMem.remove_if(Pred: [](const auto &P) { return P.second.empty(); });
1364 SGPRs.remove_if(Pred: [](const auto &P) { return P.second.empty(); });
1365}
1366
1367void WaitcntBrackets::determineWaitForScore(AMDGPU::InstCounterType T,
1368 unsigned ScoreToWait,
1369 AMDGPU::Waitcnt &Wait) const {
1370 const unsigned LB = getScoreLB(T);
1371 const unsigned UB = getScoreUB(T);
1372
1373 // If the score falls within the bracket, we need a waitcnt.
1374 if ((UB >= ScoreToWait) && (ScoreToWait > LB)) {
1375 if ((T == AMDGPU::LOAD_CNT || T == AMDGPU::DS_CNT) && hasPendingFlat() &&
1376 !Context->ST.hasFlatLgkmVMemCountInOrder()) {
1377 // If there is a pending FLAT operation, and this is a VMem or LGKM
1378 // waitcnt and the target can report early completion, then we need
1379 // to force a waitcnt 0.
1380 Wait.add(T, Count: 0);
1381 } else if (counterOutOfOrder(T)) {
1382 // Counter can get decremented out-of-order when there
1383 // are multiple types event in the bracket. Also emit an s_wait counter
1384 // with a conservative value of 0 for the counter.
1385 Wait.add(T, Count: 0);
1386 } else {
1387 // If a counter has been maxed out avoid overflow by waiting for
1388 // MAX(CounterType) - 1 instead.
1389 unsigned NeededWait = std::min(a: UB - ScoreToWait, b: getLimit(T) - 1);
1390 Wait.add(T, Count: NeededWait);
1391 }
1392 }
1393}
1394
1395AMDGPU::Waitcnt WaitcntBrackets::determineAsyncWait(unsigned N) {
1396 LLVM_DEBUG({
1397 dbgs() << "Need " << N << " async marks. Found " << AsyncMarks.size()
1398 << ":\n";
1399 for (const auto &Mark : AsyncMarks) {
1400 llvm::interleaveComma(Mark, dbgs());
1401 dbgs() << '\n';
1402 }
1403 });
1404
1405 if (AsyncMarks.size() == MaxAsyncMarks) {
1406 // Enforcing MaxAsyncMarks here is unnecessary work because the size of
1407 // MaxAsyncMarks is linear when traversing straightline code. But we do
1408 // need to check if truncation may have occured at a merge, and adjust N
1409 // to ensure that a wait is generated.
1410 LLVM_DEBUG(dbgs() << "Possible truncation. Ensuring a non-trivial wait.\n");
1411 N = std::min(a: N, b: (unsigned)MaxAsyncMarks - 1);
1412 }
1413
1414 AMDGPU::Waitcnt Wait;
1415 if (AsyncMarks.size() <= N) {
1416 LLVM_DEBUG(dbgs() << "No additional wait for async mark.\n");
1417 return Wait;
1418 }
1419
1420 size_t MarkIndex = AsyncMarks.size() - N - 1;
1421 const auto &RequiredMark = AsyncMarks[MarkIndex];
1422 for (AMDGPU::InstCounterType T : AMDGPU::inst_counter_types())
1423 determineWaitForScore(T, ScoreToWait: RequiredMark[T], Wait);
1424
1425 // Immediately remove the waited mark and all older ones
1426 // This happens BEFORE the wait is actually inserted, which is fine
1427 // because we've already extracted the wait requirements
1428 LLVM_DEBUG({
1429 dbgs() << "Removing " << (MarkIndex + 1)
1430 << " async marks after determining wait\n";
1431 });
1432 AsyncMarks.erase(CS: AsyncMarks.begin(), CE: AsyncMarks.begin() + MarkIndex + 1);
1433
1434 LLVM_DEBUG(dbgs() << "Waits to add: " << Wait);
1435 return Wait;
1436}
1437
1438// With D16Write32BitVgpr, D16 inst might be clobbered by events running on the
1439// other half 16bit.
1440//
1441// Replace VGPR16 to VGPR32 for wait check if:
1442// 1. MI is a VALU, and there is a wait event on the other half
1443// 2. MI is a LdSt, and there is a wait event on the other half from different
1444// order group
1445MCPhysReg WaitcntBrackets::determineVGPR16Dependency(const MachineInstr &MI,
1446 AMDGPU::InstCounterType T,
1447 MCPhysReg Reg) const {
1448 const TargetRegisterClass *RC = Context->TRI.getPhysRegBaseClass(Reg);
1449 unsigned Size = Context->TRI.getRegSizeInBits(RC: *RC);
1450
1451 if (Size != 16 || !Context->ST.hasD16Writes32BitVgpr())
1452 return Reg;
1453
1454 // With D16Writes32BitVgpr, D16 Inst might clobber the whole vgpr32
1455 // check dependency on the other half
1456 Register Reg32 = Context->TRI.get32BitRegister(Reg);
1457 Register OtherHalf = Context->TRI.getSubReg(
1458 Reg: Reg32,
1459 Idx: AMDGPU::isHi16Reg(Reg, MRI: Context->TRI) ? AMDGPU::lo16 : AMDGPU::hi16);
1460
1461 AMDGPU::Waitcnt Wait;
1462 for (MCRegUnit RU : regunits(Reg: OtherHalf))
1463 determineWaitForScore(T, ScoreToWait: getVMemScore(TID: toVMEMID(RU), T), Wait);
1464
1465 // No wait on otherhalf
1466 if (!Wait.hasWait())
1467 return Reg;
1468
1469 if (Context->TII.isVALU(MI, /*AllowLDSDMA=*/false))
1470 return Reg32;
1471
1472 // If hi/lo16 mixed events
1473 HWEvents MIEvents = AMDGPU::getEventsFor(
1474 Inst: MI, ST: Context->ST, IsExpertMode: Context->IsExpertMode, TgSplit: Context->TgSplit);
1475 HWEvents OtherHalfEvents = Context->getWaitEvents(T);
1476 HWEvents Events = MIEvents & OtherHalfEvents;
1477 if (Events.size() > 1)
1478 return Reg32;
1479 return Reg;
1480}
1481
1482void WaitcntBrackets::determineWaitForPhysReg(AMDGPU::InstCounterType T,
1483 MCPhysReg Reg,
1484 AMDGPU::Waitcnt &Wait,
1485 const MachineInstr &MI) const {
1486 if (Reg == AMDGPU::SCC) {
1487 determineWaitForScore(T, ScoreToWait: SCCScore, Wait);
1488 } else {
1489 bool IsVGPR = Context->TRI.isVectorRegister(MRI: Context->MRI, Reg);
1490 if (IsVGPR)
1491 Reg = determineVGPR16Dependency(MI, T, Reg);
1492 for (MCRegUnit RU : regunits(Reg))
1493 determineWaitForScore(
1494 T, ScoreToWait: IsVGPR ? getVMemScore(TID: toVMEMID(RU), T) : getSGPRScore(RU, T),
1495 Wait);
1496 }
1497}
1498
1499void WaitcntBrackets::determineWaitForLDSDMA(AMDGPU::InstCounterType T,
1500 VMEMID TID,
1501 AMDGPU::Waitcnt &Wait) const {
1502 assert(TID >= LDSDMA_BEGIN && TID < LDSDMA_END);
1503 determineWaitForScore(T, ScoreToWait: getVMemScore(TID, T), Wait);
1504}
1505
1506void WaitcntBrackets::tryClearSCCWriteEvent(MachineInstr *Inst) {
1507 // S_BARRIER_WAIT on the same barrier guarantees that the pending write to
1508 // SCC has landed
1509 if (PendingSCCWrite &&
1510 PendingSCCWrite->getOpcode() == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM &&
1511 PendingSCCWrite->getOperand(i: 0).getImm() == Inst->getOperand(i: 0).getImm()) {
1512 HWEvents SCC_WRITE_PendingEvent = HWEvents::SCC_WRITE;
1513 // If this SCC_WRITE is the only pending KM_CNT event, clear counter.
1514 if ((PendingEvents & Context->getWaitEvents(T: AMDGPU::KM_CNT)) ==
1515 SCC_WRITE_PendingEvent) {
1516 setScoreLB(T: AMDGPU::KM_CNT, Val: getScoreUB(T: AMDGPU::KM_CNT));
1517 }
1518
1519 PendingEvents -= SCC_WRITE_PendingEvent;
1520 PendingSCCWrite = nullptr;
1521 }
1522}
1523
1524void WaitcntBrackets::applyWaitcnt(const AMDGPU::Waitcnt &Wait) {
1525 for (AMDGPU::InstCounterType T : AMDGPU::inst_counter_types()) {
1526 unsigned Cnt;
1527 if (T == AMDGPU::VA_VDST_RD || T == AMDGPU::VA_VDST_WR) {
1528 Cnt =
1529 std::min(a: Wait.get(T: AMDGPU::VA_VDST_RD), b: Wait.get(T: AMDGPU::VA_VDST_WR));
1530 } else {
1531 Cnt = Wait.get(T);
1532 }
1533 applyWaitcnt(T, Count: Cnt);
1534 }
1535}
1536
1537void WaitcntBrackets::applyWaitcnt(AMDGPU::InstCounterType T, unsigned Count) {
1538 const unsigned UB = getScoreUB(T);
1539 if (Count >= UB)
1540 return;
1541 if (Count != 0) {
1542 if (counterOutOfOrder(T))
1543 return;
1544 setScoreLB(T, Val: std::max(a: getScoreLB(T), b: UB - Count));
1545 } else {
1546 setScoreLB(T, Val: UB);
1547 PendingEvents -= Context->getWaitEvents(T);
1548 }
1549
1550 if (T == AMDGPU::KM_CNT && Count == 0 &&
1551 hasPendingEvent(E: HWEvents::SMEM_GROUP)) {
1552 if (!hasMixedPendingEvents(T: AMDGPU::X_CNT))
1553 applyWaitcnt(T: AMDGPU::X_CNT, Count: 0);
1554 else
1555 PendingEvents -= HWEvents::SMEM_GROUP;
1556 }
1557 if (T == AMDGPU::LOAD_CNT && hasPendingEvent(E: HWEvents::VMEM_GROUP) &&
1558 !hasPendingEvent(T: AMDGPU::STORE_CNT)) {
1559 if (!hasMixedPendingEvents(T: AMDGPU::X_CNT))
1560 applyWaitcnt(T: AMDGPU::X_CNT, Count);
1561 else if (Count == 0)
1562 PendingEvents -= HWEvents::VMEM_GROUP;
1563 }
1564}
1565
1566void WaitcntBrackets::applyWaitcnt(const AMDGPU::Waitcnt &Wait,
1567 AMDGPU::InstCounterType T) {
1568 unsigned Cnt = Wait.get(T);
1569 applyWaitcnt(T, Count: Cnt);
1570}
1571
1572// Where there are multiple types of event in the bracket of a counter,
1573// the decrement may go out of order.
1574bool WaitcntBrackets::counterOutOfOrder(AMDGPU::InstCounterType T) const {
1575 // Scalar memory read always can go out of order.
1576 if ((T == Context->SmemAccessCounter &&
1577 hasPendingEvent(E: HWEvents::SMEM_ACCESS)) ||
1578 (T == AMDGPU::X_CNT && hasPendingEvent(E: HWEvents::SMEM_GROUP)))
1579 return true;
1580
1581 if (T == AMDGPU::LOAD_CNT) {
1582
1583 // On targets without VScnt, LOAD_CNT includes all of STORE_CNT as well.
1584 // All these events use one counter and do not go out of order with respect
1585 // to each other.
1586 if (!Context->ST.hasVscnt())
1587 return false;
1588
1589 HWEvents Events = PendingEvents & Context->getWaitEvents(T);
1590
1591 // If the target does not have extended counters, VMEM_BVH/SAMPLE_READ
1592 // events are equivalent to VMEM_READ_ACCESS. We do not go out of order in
1593 // such cases.
1594 static constexpr HWEvents ExtendedImageEvents =
1595 HWEvents::VMEM_SAMPLER_READ_ACCESS | HWEvents::VMEM_BVH_READ_ACCESS;
1596 if (!Context->ST.hasExtendedWaitCounts() &&
1597 (Events & ExtendedImageEvents).any()) {
1598 Events -= ExtendedImageEvents;
1599 Events |= HWEvents::VMEM_READ_ACCESS;
1600 }
1601
1602 // GLOBAL_INV/BUFFER_INV complete in-order with VMEM_READ_ACCESS, so mixing
1603 // VMEM_INV_ACCESS with other LOAD_CNT events does not cause out-of-order
1604 // completion.
1605 Events -= HWEvents::VMEM_INV_ACCESS;
1606
1607 return Events.size() > 1;
1608 }
1609
1610 return hasMixedPendingEvents(T);
1611}
1612
1613INITIALIZE_PASS_BEGIN(SIInsertWaitcntsLegacy, DEBUG_TYPE, "SI Insert Waitcnts",
1614 false, false)
1615INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass)
1616INITIALIZE_PASS_DEPENDENCY(MachinePostDominatorTreeWrapperPass)
1617INITIALIZE_PASS_END(SIInsertWaitcntsLegacy, DEBUG_TYPE, "SI Insert Waitcnts",
1618 false, false)
1619
1620char SIInsertWaitcntsLegacy::ID = 0;
1621
1622char &llvm::SIInsertWaitcntsID = SIInsertWaitcntsLegacy::ID;
1623
1624FunctionPass *llvm::createSIInsertWaitcntsPass() {
1625 return new SIInsertWaitcntsLegacy();
1626}
1627
1628static bool updateOperandIfDifferent(MachineInstr &MI, AMDGPU::OpName OpName,
1629 unsigned NewEnc) {
1630 int OpIdx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpName);
1631 assert(OpIdx >= 0);
1632
1633 MachineOperand &MO = MI.getOperand(i: OpIdx);
1634
1635 if (NewEnc == MO.getImm())
1636 return false;
1637
1638 MO.setImm(NewEnc);
1639 return true;
1640}
1641
1642bool WaitcntGenerator::promoteSoftWaitCnt(MachineInstr *Waitcnt) const {
1643 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode: Waitcnt->getOpcode());
1644 if (Opcode == Waitcnt->getOpcode())
1645 return false;
1646
1647 Waitcnt->setDesc(TII.get(Opcode));
1648 return true;
1649}
1650
1651/// Combine consecutive S_WAITCNT and S_WAITCNT_VSCNT instructions that
1652/// precede \p It and follow \p OldWaitcntInstr and apply any extra waits
1653/// from \p Wait that were added by previous passes. Currently this pass
1654/// conservatively assumes that these preexisting waits are required for
1655/// correctness.
1656bool WaitcntGeneratorPreGFX12::applyPreexistingWaitcnt(
1657 WaitcntBrackets &ScoreBrackets, MachineInstr &OldWaitcntInstr,
1658 AMDGPU::Waitcnt &Wait, MachineBasicBlock::instr_iterator It) {
1659 assert(isNormalMode(MaxCounter));
1660
1661 bool Modified = false;
1662 MachineInstr *WaitcntInstr = nullptr;
1663 MachineInstr *WaitcntVsCntInstr = nullptr;
1664
1665 LLVM_DEBUG({
1666 dbgs() << "PreGFX12::applyPreexistingWaitcnt at: ";
1667 if (It.isEnd())
1668 dbgs() << "end of block\n";
1669 else
1670 dbgs() << *It;
1671 });
1672
1673 for (auto &II :
1674 make_early_inc_range(Range: make_range(x: OldWaitcntInstr.getIterator(), y: It))) {
1675 LLVM_DEBUG(dbgs() << "pre-existing iter: " << II);
1676 if (isNonWaitcntMetaInst(MI: II)) {
1677 LLVM_DEBUG(dbgs() << "skipped meta instruction\n");
1678 continue;
1679 }
1680
1681 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode: II.getOpcode());
1682 bool TrySimplify = Opcode != II.getOpcode() && !OptNone;
1683
1684 // Update required wait count. If this is a soft waitcnt (= it was added
1685 // by an earlier pass), it may be entirely removed.
1686 if (Opcode == AMDGPU::S_WAITCNT) {
1687 unsigned IEnc = II.getOperand(i: 0).getImm();
1688 AMDGPU::Waitcnt OldWait = AMDGPU::decodeWaitcnt(Version: IV, Encoded: IEnc);
1689 if (TrySimplify) {
1690 ScoreBrackets.simplifyWaitcnt(Wait&: OldWait);
1691 ScoreBrackets.simplifyVmemInvOnlyWait(Wait&: OldWait);
1692 }
1693 Wait = Wait.combined(Other: OldWait);
1694
1695 // Merge consecutive waitcnt of the same type by erasing multiples.
1696 if (WaitcntInstr) {
1697 DeferredErasures.erase(V: &II);
1698 II.eraseFromParent();
1699 Modified = true;
1700 } else if (!Wait.hasWaitExceptStoreCnt() && TrySimplify) {
1701 // Might or might not be redundant, depending on if there's a loop
1702 // carried dependency. Stash it away so we can erase it if nothing saves
1703 // it.
1704 DeferredErasures.insert(V: &II);
1705 } else {
1706 WaitcntInstr = &II;
1707 DeferredErasures.erase(V: &II);
1708 }
1709 } else if (Opcode == AMDGPU::S_WAITCNT_lds_direct) {
1710 assert(ST.hasVMemToLDSLoad());
1711 LLVM_DEBUG(dbgs() << "Processing S_WAITCNT_lds_direct: " << II
1712 << "Before: " << Wait << '\n';);
1713 ScoreBrackets.determineWaitForLDSDMA(T: AMDGPU::LOAD_CNT, TID: LDSDMA_BEGIN,
1714 Wait);
1715 LLVM_DEBUG(dbgs() << "After: " << Wait << '\n';);
1716
1717 // It is possible (but unlikely) that this is the only wait instruction,
1718 // in which case, we exit this loop without a WaitcntInstr to consume
1719 // `Wait`. But that works because `Wait` was passed in by reference, and
1720 // the callee eventually calls createNewWaitcnt on it. We test this
1721 // possibility in an articial MIR test since such a situation cannot be
1722 // recreated by running the memory legalizer.
1723 II.eraseFromParent();
1724 } else if (Opcode == AMDGPU::WAIT_ASYNCMARK) {
1725 unsigned N = II.getOperand(i: 0).getImm();
1726 LLVM_DEBUG(dbgs() << "Processing WAIT_ASYNCMARK: " << II << '\n';);
1727 AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(N);
1728 Wait = Wait.combined(Other: OldWait);
1729 } else {
1730 assert(Opcode == AMDGPU::S_WAITCNT_VSCNT);
1731 assert(II.getOperand(0).getReg() == AMDGPU::SGPR_NULL);
1732
1733 unsigned OldVSCnt =
1734 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm();
1735 if (TrySimplify)
1736 ScoreBrackets.simplifyWaitcnt(T: AMDGPU::STORE_CNT, Count&: OldVSCnt);
1737 Wait.set(T: AMDGPU::STORE_CNT,
1738 Val: std::min(a: Wait.get(T: AMDGPU::STORE_CNT), b: OldVSCnt));
1739
1740 if (WaitcntVsCntInstr) {
1741 DeferredErasures.erase(V: &II);
1742 II.eraseFromParent();
1743 Modified = true;
1744 } else if (!Wait.hasWaitStoreCnt() && TrySimplify) {
1745 // Loop-carried dependencies guard, see S_WAITCNT case above.
1746 DeferredErasures.insert(V: &II);
1747 } else {
1748 WaitcntVsCntInstr = &II;
1749 DeferredErasures.erase(V: &II);
1750 }
1751 }
1752 }
1753
1754 if (WaitcntInstr) {
1755 Modified |= updateOperandIfDifferent(MI&: *WaitcntInstr, OpName: AMDGPU::OpName::simm16,
1756 NewEnc: AMDGPU::encodeWaitcnt(Version: IV, Decoded: Wait));
1757 Modified |= promoteSoftWaitCnt(Waitcnt: WaitcntInstr);
1758
1759 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::LOAD_CNT);
1760 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::EXP_CNT);
1761 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::DS_CNT);
1762 Wait.set(T: AMDGPU::LOAD_CNT, Val: ~0u);
1763 Wait.set(T: AMDGPU::EXP_CNT, Val: ~0u);
1764 Wait.set(T: AMDGPU::DS_CNT, Val: ~0u);
1765
1766 LLVM_DEBUG(It.isEnd() ? dbgs() << "applied pre-existing waitcnt\n"
1767 << "New Instr at block end: "
1768 << *WaitcntInstr << '\n'
1769 : dbgs() << "applied pre-existing waitcnt\n"
1770 << "Old Instr: " << *It
1771 << "New Instr: " << *WaitcntInstr << '\n');
1772 }
1773
1774 if (WaitcntVsCntInstr) {
1775 Modified |=
1776 updateOperandIfDifferent(MI&: *WaitcntVsCntInstr, OpName: AMDGPU::OpName::simm16,
1777 NewEnc: Wait.get(T: AMDGPU::STORE_CNT));
1778 Modified |= promoteSoftWaitCnt(Waitcnt: WaitcntVsCntInstr);
1779
1780 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::STORE_CNT);
1781 Wait.set(T: AMDGPU::STORE_CNT, Val: ~0u);
1782
1783 LLVM_DEBUG(It.isEnd()
1784 ? dbgs() << "applied pre-existing waitcnt\n"
1785 << "New Instr at block end: " << *WaitcntVsCntInstr
1786 << '\n'
1787 : dbgs() << "applied pre-existing waitcnt\n"
1788 << "Old Instr: " << *It
1789 << "New Instr: " << *WaitcntVsCntInstr << '\n');
1790 }
1791
1792 return Modified;
1793}
1794
1795/// Generate S_WAITCNT and/or S_WAITCNT_VSCNT instructions for any
1796/// required counters in \p Wait
1797bool WaitcntGeneratorPreGFX12::createNewWaitcnt(
1798 MachineBasicBlock &Block, MachineBasicBlock::instr_iterator It,
1799 AMDGPU::Waitcnt Wait, const WaitcntBrackets &ScoreBrackets) {
1800 assert(isNormalMode(MaxCounter));
1801
1802 bool Modified = false;
1803 const DebugLoc &DL = Block.findDebugLoc(MBBI: It);
1804
1805 // Waits for VMcnt, LKGMcnt and/or EXPcnt are encoded together into a
1806 // single instruction while VScnt has its own instruction.
1807 if (Wait.hasWaitExceptStoreCnt()) {
1808 // If profiling expansion is enabled, emit an expanded sequence
1809 if (ExpandWaitcntProfiling) {
1810 // Check if any of the counters to be waited on are out-of-order.
1811 // If so, fall back to normal (non-expanded) behavior since expansion
1812 // would provide misleading profiling information.
1813 bool AnyOutOfOrder = false;
1814 for (auto CT : {AMDGPU::LOAD_CNT, AMDGPU::DS_CNT, AMDGPU::EXP_CNT}) {
1815 unsigned WaitCnt = Wait.get(T: CT);
1816 if (WaitCnt != ~0u && ScoreBrackets.counterOutOfOrder(T: CT)) {
1817 AnyOutOfOrder = true;
1818 break;
1819 }
1820 }
1821
1822 if (AnyOutOfOrder) {
1823 // Fall back to non-expanded wait
1824 unsigned Enc = AMDGPU::encodeWaitcnt(Version: IV, Decoded: Wait);
1825 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT)).addImm(Val: Enc);
1826 Modified = true;
1827 } else {
1828 // All counters are in-order, safe to expand
1829 for (auto CT : {AMDGPU::LOAD_CNT, AMDGPU::DS_CNT, AMDGPU::EXP_CNT}) {
1830 unsigned WaitCnt = Wait.get(T: CT);
1831 if (WaitCnt == ~0u)
1832 continue;
1833
1834 unsigned Outstanding =
1835 std::min(a: ScoreBrackets.getOutstanding(T: CT), b: getLimit(E: CT) - 1);
1836 EmitExpandedWaitcnt(Outstanding, Target: WaitCnt, EmitWaitcnt: [&](unsigned Count) {
1837 AMDGPU::Waitcnt W;
1838 W.set(T: CT, Val: Count);
1839 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT))
1840 .addImm(Val: AMDGPU::encodeWaitcnt(Version: IV, Decoded: W));
1841 });
1842 Modified = true;
1843 }
1844 }
1845 } else {
1846 // Normal behavior: emit single combined waitcnt
1847 unsigned Enc = AMDGPU::encodeWaitcnt(Version: IV, Decoded: Wait);
1848 [[maybe_unused]] auto SWaitInst =
1849 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT)).addImm(Val: Enc);
1850 Modified = true;
1851
1852 LLVM_DEBUG(dbgs() << "PreGFX12::createNewWaitcnt\n";
1853 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
1854 dbgs() << "New Instr: " << *SWaitInst << '\n');
1855 }
1856 }
1857
1858 if (Wait.hasWaitStoreCnt()) {
1859 assert(ST.hasVscnt());
1860
1861 if (ExpandWaitcntProfiling && Wait.get(T: AMDGPU::STORE_CNT) != ~0u &&
1862 !ScoreBrackets.counterOutOfOrder(T: AMDGPU::STORE_CNT)) {
1863 // Only expand if counter is not out-of-order
1864 unsigned Outstanding =
1865 std::min(a: ScoreBrackets.getOutstanding(T: AMDGPU::STORE_CNT),
1866 b: getLimit(E: AMDGPU::STORE_CNT) - 1);
1867 EmitExpandedWaitcnt(
1868 Outstanding, Target: Wait.get(T: AMDGPU::STORE_CNT), EmitWaitcnt: [&](unsigned Count) {
1869 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_VSCNT))
1870 .addReg(RegNo: AMDGPU::SGPR_NULL, Flags: RegState::Undef)
1871 .addImm(Val: Count);
1872 });
1873 Modified = true;
1874 } else {
1875 [[maybe_unused]] auto SWaitInst =
1876 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_VSCNT))
1877 .addReg(RegNo: AMDGPU::SGPR_NULL, Flags: RegState::Undef)
1878 .addImm(Val: Wait.get(T: AMDGPU::STORE_CNT));
1879 Modified = true;
1880
1881 LLVM_DEBUG(dbgs() << "PreGFX12::createNewWaitcnt\n";
1882 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
1883 dbgs() << "New Instr: " << *SWaitInst << '\n');
1884 }
1885 }
1886
1887 return Modified;
1888}
1889
1890AMDGPU::Waitcnt
1891WaitcntGeneratorPreGFX12::getAllZeroWaitcnt(bool IncludeVSCnt) const {
1892 return AMDGPU::Waitcnt(0, 0, 0, IncludeVSCnt && ST.hasVscnt() ? 0 : ~0u);
1893}
1894
1895AMDGPU::Waitcnt
1896WaitcntGeneratorGFX12Plus::getAllZeroWaitcnt(bool IncludeVSCnt) const {
1897 unsigned ExpertVal = IsExpertMode ? 0 : ~0u;
1898 return AMDGPU::Waitcnt(0, 0, 0, IncludeVSCnt ? 0 : ~0u, 0, 0, 0,
1899 ~0u /* XCNT */, ~0u /* ASYNC_CNT */,
1900 ~0u /* TENSOR_CNT */, ExpertVal, ExpertVal, ExpertVal);
1901}
1902
1903/// Combine consecutive S_WAIT_*CNT instructions that precede \p It and
1904/// follow \p OldWaitcntInstr and apply any extra waits from \p Wait that
1905/// were added by previous passes. Currently this pass conservatively
1906/// assumes that these preexisting waits are required for correctness.
1907bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
1908 WaitcntBrackets &ScoreBrackets, MachineInstr &OldWaitcntInstr,
1909 AMDGPU::Waitcnt &Wait, MachineBasicBlock::instr_iterator It) {
1910 assert(!isNormalMode(MaxCounter));
1911
1912 bool Modified = false;
1913 MachineInstr *CombinedLoadDsCntInstr = nullptr;
1914 MachineInstr *CombinedStoreDsCntInstr = nullptr;
1915 MachineInstr *WaitcntDepctrInstr = nullptr;
1916 MachineInstr *WaitInstrs[AMDGPU::NUM_EXTENDED_INST_CNTS] = {};
1917
1918 LLVM_DEBUG({
1919 dbgs() << "GFX12Plus::applyPreexistingWaitcnt at: ";
1920 if (It.isEnd())
1921 dbgs() << "end of block\n";
1922 else
1923 dbgs() << *It;
1924 });
1925
1926 // Accumulate waits that should not be simplified.
1927 AMDGPU::Waitcnt RequiredWait;
1928
1929 for (auto &II :
1930 make_early_inc_range(Range: make_range(x: OldWaitcntInstr.getIterator(), y: It))) {
1931 LLVM_DEBUG(dbgs() << "pre-existing iter: " << II);
1932 if (isNonWaitcntMetaInst(MI: II)) {
1933 LLVM_DEBUG(dbgs() << "skipped meta instruction\n");
1934 continue;
1935 }
1936
1937 // Update required wait count. If this is a soft waitcnt (= it was added
1938 // by an earlier pass), it may be entirely removed.
1939
1940 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode: II.getOpcode());
1941 bool TrySimplify = Opcode != II.getOpcode() && !OptNone;
1942
1943 // Don't crash if the programmer used legacy waitcnt intrinsics, but don't
1944 // attempt to do more than that either.
1945 if (Opcode == AMDGPU::S_WAITCNT)
1946 continue;
1947
1948 if (Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT) {
1949 unsigned OldEnc =
1950 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm();
1951 AMDGPU::Waitcnt OldWait = AMDGPU::decodeLoadcntDscnt(Version: IV, LoadcntDscnt: OldEnc);
1952 if (TrySimplify)
1953 Wait = Wait.combined(Other: OldWait);
1954 else
1955 RequiredWait = RequiredWait.combined(Other: OldWait);
1956 // Keep the first wait_loadcnt, erase the rest.
1957 if (CombinedLoadDsCntInstr == nullptr) {
1958 CombinedLoadDsCntInstr = &II;
1959 } else {
1960 II.eraseFromParent();
1961 Modified = true;
1962 }
1963 } else if (Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT) {
1964 unsigned OldEnc =
1965 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm();
1966 AMDGPU::Waitcnt OldWait = AMDGPU::decodeStorecntDscnt(Version: IV, StorecntDscnt: OldEnc);
1967 if (TrySimplify)
1968 Wait = Wait.combined(Other: OldWait);
1969 else
1970 RequiredWait = RequiredWait.combined(Other: OldWait);
1971 // Keep the first wait_storecnt, erase the rest.
1972 if (CombinedStoreDsCntInstr == nullptr) {
1973 CombinedStoreDsCntInstr = &II;
1974 } else {
1975 II.eraseFromParent();
1976 Modified = true;
1977 }
1978 } else if (Opcode == AMDGPU::S_WAITCNT_DEPCTR) {
1979 unsigned OldEnc =
1980 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm();
1981 AMDGPU::Waitcnt OldWait;
1982 // Set both counters to the decoded value from the single hardware field
1983 unsigned VaVdst = AMDGPU::DepCtr::decodeFieldVaVdst(Encoded: OldEnc);
1984 OldWait.set(T: AMDGPU::VA_VDST_RD, Val: VaVdst);
1985 OldWait.set(T: AMDGPU::VA_VDST_WR, Val: VaVdst);
1986 OldWait.set(T: AMDGPU::VM_VSRC, Val: AMDGPU::DepCtr::decodeFieldVmVsrc(Encoded: OldEnc));
1987 if (TrySimplify)
1988 ScoreBrackets.simplifyWaitcnt(Wait&: OldWait);
1989 Wait = Wait.combined(Other: OldWait);
1990 if (WaitcntDepctrInstr == nullptr) {
1991 WaitcntDepctrInstr = &II;
1992 } else {
1993 // S_WAITCNT_DEPCTR requires special care. Don't remove a
1994 // duplicate if it is waiting on things other than VA_VDST or
1995 // VM_VSRC. If that is the case, just make sure the VA_VDST and
1996 // VM_VSRC subfields of the operand are set to the "no wait"
1997 // values.
1998
1999 unsigned Enc =
2000 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm();
2001 Enc = AMDGPU::DepCtr::encodeFieldVmVsrc(Encoded: Enc, VmVsrc: ~0u);
2002 // Encode min(VA_VDST_RD, VA_VDST_WR) into the single hardware field
2003 unsigned VaVdst = std::min(a: Wait.get(T: AMDGPU::VA_VDST_RD),
2004 b: Wait.get(T: AMDGPU::VA_VDST_WR));
2005 Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Encoded: Enc, VaVdst);
2006
2007 if (Enc != (unsigned)AMDGPU::DepCtr::getDefaultDepCtrEncoding(STI: ST)) {
2008 Modified |= updateOperandIfDifferent(MI&: II, OpName: AMDGPU::OpName::simm16, NewEnc: Enc);
2009 Modified |= promoteSoftWaitCnt(Waitcnt: &II);
2010 } else {
2011 II.eraseFromParent();
2012 Modified = true;
2013 }
2014 }
2015 } else if (Opcode == AMDGPU::S_WAITCNT_lds_direct) {
2016 // Architectures higher than GFX10 do not have direct loads to
2017 // LDS, so no work required here yet.
2018 II.eraseFromParent();
2019 Modified = true;
2020 } else if (Opcode == AMDGPU::WAIT_ASYNCMARK) {
2021 // Update the Waitcnt, but don't erase the wait.asyncmark() itself. It
2022 // shows up in the assembly as a comment with the original parameter N.
2023 unsigned N = II.getOperand(i: 0).getImm();
2024 AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(N);
2025 Wait = Wait.combined(Other: OldWait);
2026 } else {
2027 std::optional<AMDGPU::InstCounterType> CT =
2028 AMDGPU::counterTypeForInstr(Opcode);
2029 assert(CT.has_value());
2030 unsigned OldCnt =
2031 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm();
2032 if (TrySimplify)
2033 Wait.add(T: CT.value(), Count: OldCnt);
2034 else
2035 RequiredWait.add(T: CT.value(), Count: OldCnt);
2036 // Keep the first wait of its kind, erase the rest.
2037 if (WaitInstrs[CT.value()] == nullptr) {
2038 WaitInstrs[CT.value()] = &II;
2039 } else {
2040 II.eraseFromParent();
2041 Modified = true;
2042 }
2043 }
2044 }
2045
2046 ScoreBrackets.simplifyWaitcnt(CheckWait: Wait.combined(Other: RequiredWait), UpdateWait&: Wait);
2047 Wait = Wait.combined(Other: RequiredWait);
2048
2049 if (CombinedLoadDsCntInstr) {
2050 // Only keep an S_WAIT_LOADCNT_DSCNT if both counters actually need
2051 // to be waited for. Otherwise, let the instruction be deleted so
2052 // the appropriate single counter wait instruction can be inserted
2053 // instead, when new S_WAIT_*CNT instructions are inserted by
2054 // createNewWaitcnt(). As a side effect, resetting the wait counts will
2055 // cause any redundant S_WAIT_LOADCNT or S_WAIT_DSCNT to be removed by
2056 // the loop below that deals with single counter instructions.
2057 //
2058 // A wait for LOAD_CNT or DS_CNT implies a wait for VM_VSRC, since
2059 // instructions that have decremented LOAD_CNT or DS_CNT on completion
2060 // will have needed to wait for their register sources to be available
2061 // first.
2062 if (Wait.get(T: AMDGPU::LOAD_CNT) != ~0u && Wait.get(T: AMDGPU::DS_CNT) != ~0u) {
2063 unsigned NewEnc = AMDGPU::encodeLoadcntDscnt(Version: IV, Decoded: Wait);
2064 Modified |= updateOperandIfDifferent(MI&: *CombinedLoadDsCntInstr,
2065 OpName: AMDGPU::OpName::simm16, NewEnc);
2066 Modified |= promoteSoftWaitCnt(Waitcnt: CombinedLoadDsCntInstr);
2067 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::LOAD_CNT);
2068 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::DS_CNT);
2069 Wait.set(T: AMDGPU::LOAD_CNT, Val: ~0u);
2070 Wait.set(T: AMDGPU::DS_CNT, Val: ~0u);
2071
2072 LLVM_DEBUG(It.isEnd() ? dbgs() << "applied pre-existing waitcnt\n"
2073 << "New Instr at block end: "
2074 << *CombinedLoadDsCntInstr << '\n'
2075 : dbgs() << "applied pre-existing waitcnt\n"
2076 << "Old Instr: " << *It << "New Instr: "
2077 << *CombinedLoadDsCntInstr << '\n');
2078 } else {
2079 CombinedLoadDsCntInstr->eraseFromParent();
2080 Modified = true;
2081 }
2082 }
2083
2084 if (CombinedStoreDsCntInstr) {
2085 // Similarly for S_WAIT_STORECNT_DSCNT.
2086 if (Wait.get(T: AMDGPU::STORE_CNT) != ~0u && Wait.get(T: AMDGPU::DS_CNT) != ~0u) {
2087 unsigned NewEnc = AMDGPU::encodeStorecntDscnt(Version: IV, Decoded: Wait);
2088 Modified |= updateOperandIfDifferent(MI&: *CombinedStoreDsCntInstr,
2089 OpName: AMDGPU::OpName::simm16, NewEnc);
2090 Modified |= promoteSoftWaitCnt(Waitcnt: CombinedStoreDsCntInstr);
2091 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::STORE_CNT);
2092 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::DS_CNT);
2093 Wait.set(T: AMDGPU::STORE_CNT, Val: ~0u);
2094 Wait.set(T: AMDGPU::DS_CNT, Val: ~0u);
2095
2096 LLVM_DEBUG(It.isEnd() ? dbgs() << "applied pre-existing waitcnt\n"
2097 << "New Instr at block end: "
2098 << *CombinedStoreDsCntInstr << '\n'
2099 : dbgs() << "applied pre-existing waitcnt\n"
2100 << "Old Instr: " << *It << "New Instr: "
2101 << *CombinedStoreDsCntInstr << '\n');
2102 } else {
2103 CombinedStoreDsCntInstr->eraseFromParent();
2104 Modified = true;
2105 }
2106 }
2107
2108 // Look for an opportunity to convert existing S_WAIT_LOADCNT,
2109 // S_WAIT_STORECNT and S_WAIT_DSCNT into new S_WAIT_LOADCNT_DSCNT
2110 // or S_WAIT_STORECNT_DSCNT. This is achieved by selectively removing
2111 // instructions so that createNewWaitcnt() will create new combined
2112 // instructions to replace them.
2113
2114 if (Wait.get(T: AMDGPU::DS_CNT) != ~0u) {
2115 // This is a vector of addresses in WaitInstrs pointing to instructions
2116 // that should be removed if they are present.
2117 SmallVector<MachineInstr **, 2> WaitsToErase;
2118
2119 // If it's known that both DScnt and either LOADcnt or STOREcnt (but not
2120 // both) need to be waited for, ensure that there are no existing
2121 // individual wait count instructions for these.
2122
2123 if (Wait.get(T: AMDGPU::LOAD_CNT) != ~0u) {
2124 WaitsToErase.push_back(Elt: &WaitInstrs[AMDGPU::LOAD_CNT]);
2125 WaitsToErase.push_back(Elt: &WaitInstrs[AMDGPU::DS_CNT]);
2126 } else if (Wait.get(T: AMDGPU::STORE_CNT) != ~0u) {
2127 WaitsToErase.push_back(Elt: &WaitInstrs[AMDGPU::STORE_CNT]);
2128 WaitsToErase.push_back(Elt: &WaitInstrs[AMDGPU::DS_CNT]);
2129 }
2130
2131 for (MachineInstr **WI : WaitsToErase) {
2132 if (!*WI)
2133 continue;
2134
2135 DeferredErasures.erase(V: *WI);
2136 (*WI)->eraseFromParent();
2137 *WI = nullptr;
2138 Modified = true;
2139 }
2140 }
2141
2142 for (auto CT : inst_counter_types(MaxCounter: AMDGPU::NUM_EXTENDED_INST_CNTS)) {
2143 if (!WaitInstrs[CT])
2144 continue;
2145
2146 unsigned NewCnt = Wait.get(T: CT);
2147 if (NewCnt != ~0u) {
2148 Modified |= updateOperandIfDifferent(MI&: *WaitInstrs[CT],
2149 OpName: AMDGPU::OpName::simm16, NewEnc: NewCnt);
2150 Modified |= promoteSoftWaitCnt(Waitcnt: WaitInstrs[CT]);
2151 DeferredErasures.erase(V: WaitInstrs[CT]);
2152
2153 ScoreBrackets.applyWaitcnt(T: CT, Count: NewCnt);
2154 Wait.clear(T: CT);
2155
2156 LLVM_DEBUG(It.isEnd()
2157 ? dbgs() << "applied pre-existing waitcnt\n"
2158 << "New Instr at block end: " << *WaitInstrs[CT]
2159 << '\n'
2160 : dbgs() << "applied pre-existing waitcnt\n"
2161 << "Old Instr: " << *It
2162 << "New Instr: " << *WaitInstrs[CT] << '\n');
2163 } else {
2164 // Defer erasure in case there's a loop-carried dependency.
2165 DeferredErasures.insert(V: WaitInstrs[CT]);
2166 }
2167 }
2168
2169 if (WaitcntDepctrInstr) {
2170 // Get the encoded Depctr immediate and override the VA_VDST and VM_VSRC
2171 // subfields with the new required values.
2172 unsigned Enc =
2173 TII.getNamedOperand(MI&: *WaitcntDepctrInstr, OperandName: AMDGPU::OpName::simm16)
2174 ->getImm();
2175 Enc = AMDGPU::DepCtr::encodeFieldVmVsrc(Encoded: Enc, VmVsrc: Wait.get(T: AMDGPU::VM_VSRC));
2176 // Encode min(VA_VDST_RD, VA_VDST_WR) into the single hardware field
2177 unsigned VaVdst =
2178 std::min(a: Wait.get(T: AMDGPU::VA_VDST_RD), b: Wait.get(T: AMDGPU::VA_VDST_WR));
2179 Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Encoded: Enc, VaVdst);
2180
2181 ScoreBrackets.applyWaitcnt(T: AMDGPU::VA_VDST_RD, Count: VaVdst);
2182 ScoreBrackets.applyWaitcnt(T: AMDGPU::VA_VDST_WR, Count: VaVdst);
2183 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::VM_VSRC);
2184 Wait.set(T: AMDGPU::VA_VDST_RD, Val: ~0u);
2185 Wait.set(T: AMDGPU::VA_VDST_WR, Val: ~0u);
2186 Wait.set(T: AMDGPU::VM_VSRC, Val: ~0u);
2187
2188 // If that new encoded Depctr immediate would actually still wait
2189 // for anything, update the instruction's operand. Otherwise it can
2190 // just be deleted.
2191 if (Enc != (unsigned)AMDGPU::DepCtr::getDefaultDepCtrEncoding(STI: ST)) {
2192 Modified |= updateOperandIfDifferent(MI&: *WaitcntDepctrInstr,
2193 OpName: AMDGPU::OpName::simm16, NewEnc: Enc);
2194 LLVM_DEBUG(It.isEnd() ? dbgs() << "applyPreexistingWaitcnt\n"
2195 << "New Instr at block end: "
2196 << *WaitcntDepctrInstr << '\n'
2197 : dbgs() << "applyPreexistingWaitcnt\n"
2198 << "Old Instr: " << *It << "New Instr: "
2199 << *WaitcntDepctrInstr << '\n');
2200 } else {
2201 WaitcntDepctrInstr->eraseFromParent();
2202 Modified = true;
2203 }
2204 }
2205
2206 return Modified;
2207}
2208
2209/// Generate S_WAIT_*CNT instructions for any required counters in \p Wait
2210bool WaitcntGeneratorGFX12Plus::createNewWaitcnt(
2211 MachineBasicBlock &Block, MachineBasicBlock::instr_iterator It,
2212 AMDGPU::Waitcnt Wait, const WaitcntBrackets &ScoreBrackets) {
2213 assert(!isNormalMode(MaxCounter));
2214
2215 bool Modified = false;
2216 const DebugLoc &DL = Block.findDebugLoc(MBBI: It);
2217
2218 // For GFX12+, we use separate wait instructions, which makes expansion
2219 // simpler
2220 if (ExpandWaitcntProfiling) {
2221 for (auto CT : inst_counter_types(MaxCounter: AMDGPU::NUM_EXTENDED_INST_CNTS)) {
2222 unsigned Count = Wait.get(T: CT);
2223 if (Count == ~0u)
2224 continue;
2225
2226 // Skip expansion for out-of-order counters - emit normal wait instead
2227 if (ScoreBrackets.counterOutOfOrder(T: CT)) {
2228 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: instrsForExtendedCounterTypes[CT]))
2229 .addImm(Val: Count);
2230 Modified = true;
2231 continue;
2232 }
2233
2234 unsigned Outstanding =
2235 std::min(a: ScoreBrackets.getOutstanding(T: CT), b: getLimit(E: CT) - 1);
2236 EmitExpandedWaitcnt(Outstanding, Target: Count, EmitWaitcnt: [&](unsigned Val) {
2237 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: instrsForExtendedCounterTypes[CT]))
2238 .addImm(Val);
2239 });
2240 Modified = true;
2241 }
2242 return Modified;
2243 }
2244
2245 // Normal behavior (no expansion)
2246 // Check for opportunities to use combined wait instructions.
2247 if (Wait.get(T: AMDGPU::DS_CNT) != ~0u) {
2248 MachineInstr *SWaitInst = nullptr;
2249
2250 if (Wait.get(T: AMDGPU::LOAD_CNT) != ~0u) {
2251 unsigned Enc = AMDGPU::encodeLoadcntDscnt(Version: IV, Decoded: Wait);
2252
2253 SWaitInst = BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAIT_LOADCNT_DSCNT))
2254 .addImm(Val: Enc);
2255
2256 Wait.set(T: AMDGPU::LOAD_CNT, Val: ~0u);
2257 Wait.set(T: AMDGPU::DS_CNT, Val: ~0u);
2258 } else if (Wait.get(T: AMDGPU::STORE_CNT) != ~0u) {
2259 unsigned Enc = AMDGPU::encodeStorecntDscnt(Version: IV, Decoded: Wait);
2260
2261 SWaitInst = BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAIT_STORECNT_DSCNT))
2262 .addImm(Val: Enc);
2263
2264 Wait.set(T: AMDGPU::STORE_CNT, Val: ~0u);
2265 Wait.set(T: AMDGPU::DS_CNT, Val: ~0u);
2266 }
2267
2268 if (SWaitInst) {
2269 Modified = true;
2270
2271 LLVM_DEBUG(dbgs() << "GFX12Plus::createNewWaitcnt\n";
2272 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
2273 dbgs() << "New Instr: " << *SWaitInst << '\n');
2274 }
2275 }
2276
2277 // Generate an instruction for any remaining counter that needs
2278 // waiting for.
2279
2280 for (auto CT : inst_counter_types(MaxCounter: AMDGPU::NUM_EXTENDED_INST_CNTS)) {
2281 unsigned Count = Wait.get(T: CT);
2282 if (Count == ~0u)
2283 continue;
2284
2285 [[maybe_unused]] auto SWaitInst =
2286 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: instrsForExtendedCounterTypes[CT]))
2287 .addImm(Val: Count);
2288
2289 Modified = true;
2290
2291 LLVM_DEBUG(dbgs() << "GFX12Plus::createNewWaitcnt\n";
2292 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
2293 dbgs() << "New Instr: " << *SWaitInst << '\n');
2294 }
2295
2296 if (Wait.hasWaitDepctr()) {
2297 assert(IsExpertMode);
2298 unsigned Enc =
2299 AMDGPU::DepCtr::encodeFieldVmVsrc(VmVsrc: Wait.get(T: AMDGPU::VM_VSRC), STI: ST);
2300 // Encode min(VA_VDST_RD, VA_VDST_WR) into the single hardware field
2301 unsigned VaVdst =
2302 std::min(a: Wait.get(T: AMDGPU::VA_VDST_RD), b: Wait.get(T: AMDGPU::VA_VDST_WR));
2303 Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Encoded: Enc, VaVdst);
2304
2305 [[maybe_unused]] auto SWaitInst =
2306 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR)).addImm(Val: Enc);
2307
2308 Modified = true;
2309
2310 LLVM_DEBUG(dbgs() << "generateWaitcnt\n";
2311 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
2312 dbgs() << "New Instr: " << *SWaitInst << '\n');
2313 }
2314
2315 return Modified;
2316}
2317
2318/// Generate s_waitcnt instruction to be placed before cur_Inst.
2319/// Instructions of a given type are returned in order,
2320/// but instructions of different types can complete out of order.
2321/// We rely on this in-order completion
2322/// and simply assign a score to the memory access instructions.
2323/// We keep track of the active "score bracket" to determine
2324/// if an access of a memory read requires an s_waitcnt
2325/// and if so what the value of each counter is.
2326/// The "score bracket" is bound by the lower bound and upper bound
2327/// scores (*_score_LB and *_score_ub respectively).
2328/// If FlushFlags.FlushVmCnt is true, we want to flush the vmcnt counter here.
2329/// If FlushFlags.FlushDsCnt is true, we want to flush the dscnt counter here
2330/// (GFX12+ only, where DS_CNT is a separate counter).
2331bool SIInsertWaitcnts::generateWaitcntInstBefore(
2332 MachineInstr &MI, WaitcntBrackets &ScoreBrackets,
2333 MachineInstr *OldWaitcntInstr, PreheaderFlushFlags FlushFlags) {
2334 LLVM_DEBUG(dbgs() << "\n*** GenerateWaitcntInstBefore: "; MI.print(dbgs()););
2335
2336 assert(!isNonWaitcntMetaInst(MI));
2337
2338 AMDGPU::Waitcnt Wait;
2339 const unsigned Opc = MI.getOpcode();
2340
2341 switch (Opc) {
2342 case AMDGPU::BUFFER_WBINVL1:
2343 case AMDGPU::BUFFER_WBINVL1_SC:
2344 case AMDGPU::BUFFER_WBINVL1_VOL:
2345 case AMDGPU::BUFFER_GL0_INV:
2346 case AMDGPU::BUFFER_GL1_INV: {
2347 // FIXME: This should have already been handled by the memory legalizer.
2348 // Removing this currently doesn't affect any lit tests, but we need to
2349 // verify that nothing was relying on this. The number of buffer invalidates
2350 // being handled here should not be expanded.
2351 Wait.set(T: AMDGPU::LOAD_CNT, Val: 0);
2352 break;
2353 }
2354 case AMDGPU::SI_RETURN_TO_EPILOG:
2355 case AMDGPU::SI_RETURN:
2356 case AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN:
2357 case AMDGPU::S_SETPC_B64_return: {
2358 // All waits must be resolved at call return.
2359 // NOTE: this could be improved with knowledge of all call sites or
2360 // with knowledge of the called routines.
2361 ReturnInsts.insert(V: &MI);
2362 AMDGPU::Waitcnt AllZeroWait =
2363 WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/false);
2364 ScoreBrackets.simplifyVmemInvOnlyWait(Wait&: AllZeroWait);
2365 Wait = AllZeroWait;
2366 break;
2367 }
2368 case AMDGPU::S_ENDPGM:
2369 case AMDGPU::S_ENDPGM_SAVED: {
2370 // In dynamic VGPR mode, we want to release the VGPRs before the wave exits.
2371 // Technically the hardware will do this on its own if we don't, but that
2372 // might cost extra cycles compared to doing it explicitly.
2373 // When not in dynamic VGPR mode, identify S_ENDPGM instructions which may
2374 // have to wait for outstanding VMEM stores. In this case it can be useful
2375 // to send a message to explicitly release all VGPRs before the stores have
2376 // completed, but it is only safe to do this if there are no outstanding
2377 // scratch stores.
2378 EndPgmInsts[&MI] =
2379 !ScoreBrackets.empty(T: AMDGPU::STORE_CNT) &&
2380 !ScoreBrackets.hasPendingEvent(E: HWEvents::SCRATCH_WRITE_ACCESS);
2381 break;
2382 }
2383 case AMDGPU::S_SENDMSG:
2384 case AMDGPU::S_SENDMSGHALT: {
2385 if (ST.hasLegacyGeometry() &&
2386 ((MI.getOperand(i: 0).getImm() & AMDGPU::SendMsg::ID_MASK_PreGFX11_) ==
2387 AMDGPU::SendMsg::ID_GS_DONE_PreGFX11)) {
2388 // Resolve vm waits before gs-done.
2389 Wait.set(T: AMDGPU::LOAD_CNT, Val: 0);
2390 break;
2391 }
2392 [[fallthrough]];
2393 }
2394 default: {
2395
2396 // Export & GDS instructions do not read the EXEC mask until after the
2397 // export is granted (which can occur well after the instruction is issued).
2398 // The shader program must flush all EXP operations on the export-count
2399 // before overwriting the EXEC mask.
2400 if (MI.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &TRI)) {
2401 // Export and GDS are tracked individually, either may trigger a waitcnt
2402 // for EXEC.
2403 if (ScoreBrackets.hasPendingEvent(E: HWEvents::EXP_GPR_LOCK) ||
2404 ScoreBrackets.hasPendingEvent(E: HWEvents::EXP_PARAM_ACCESS) ||
2405 ScoreBrackets.hasPendingEvent(E: HWEvents::EXP_POS_ACCESS) ||
2406 ScoreBrackets.hasPendingEvent(E: HWEvents::GDS_GPR_LOCK)) {
2407 Wait.set(T: AMDGPU::EXP_CNT, Val: 0);
2408 }
2409 }
2410
2411 // Wait for any pending GDS instruction to complete before any
2412 // "Always GDS" instruction.
2413 if (TII.isAlwaysGDS(Opcode: Opc) && ScoreBrackets.hasPendingGDS())
2414 Wait.add(T: AMDGPU::DS_CNT, Count: ScoreBrackets.getPendingGDSWait());
2415
2416 if (MI.isCall()) {
2417 // The function is going to insert a wait on everything in its prolog.
2418 // This still needs to be careful if the call target is a load (e.g. a GOT
2419 // load). We also need to check WAW dependency with saved PC.
2420 CallInsts.insert(V: &MI);
2421 Wait = AMDGPU::Waitcnt();
2422
2423 const MachineOperand &CallAddrOp = TII.getCalleeOperand(MI);
2424 if (CallAddrOp.isReg()) {
2425 ScoreBrackets.determineWaitForPhysReg(
2426 T: SmemAccessCounter, Reg: CallAddrOp.getReg().asMCReg(), Wait, MI);
2427
2428 if (const auto *RtnAddrOp =
2429 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::dst)) {
2430 ScoreBrackets.determineWaitForPhysReg(
2431 T: SmemAccessCounter, Reg: RtnAddrOp->getReg().asMCReg(), Wait, MI);
2432 }
2433 }
2434 } else if (Opc == AMDGPU::S_BARRIER_WAIT) {
2435 ScoreBrackets.tryClearSCCWriteEvent(Inst: &MI);
2436 } else {
2437 // FIXME: Should not be relying on memoperands.
2438 // Look at the source operands of every instruction to see if
2439 // any of them results from a previous memory operation that affects
2440 // its current usage. If so, an s_waitcnt instruction needs to be
2441 // emitted.
2442 // If the source operand was defined by a load, add the s_waitcnt
2443 // instruction.
2444 //
2445 // Two cases are handled for destination operands:
2446 // 1) If the destination operand was defined by a load, add the s_waitcnt
2447 // instruction to guarantee the right WAW order.
2448 // 2) If a destination operand that was used by a recent export/store ins,
2449 // add s_waitcnt on exp_cnt to guarantee the WAR order.
2450
2451 for (const MachineMemOperand *Memop : MI.memoperands()) {
2452 const Value *Ptr = Memop->getValue();
2453 if (Memop->isStore()) {
2454 if (auto It = SLoadAddresses.find(Val: Ptr); It != SLoadAddresses.end()) {
2455 Wait.add(T: SmemAccessCounter, Count: 0);
2456 if (PDT.dominates(A: MI.getParent(), B: It->second))
2457 SLoadAddresses.erase(I: It);
2458 }
2459 }
2460 unsigned AS = Memop->getAddrSpace();
2461 if (AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::FLAT_ADDRESS)
2462 continue;
2463 // No need to wait before load from VMEM to LDS.
2464 if (TII.mayWriteLDSThroughDMA(MI))
2465 continue;
2466
2467 // LOAD_CNT is only relevant to vgpr or LDS.
2468 unsigned TID = LDSDMA_BEGIN;
2469 if (Ptr && Memop->getAAInfo()) {
2470 const auto &LDSDMAStores = ScoreBrackets.getLDSDMAStores();
2471 for (unsigned I = 0, E = LDSDMAStores.size(); I != E; ++I) {
2472 if (MI.mayAlias(AA, Other: *LDSDMAStores[I], UseTBAA: true)) {
2473 if ((I + 1) >= NUM_LDSDMA) {
2474 // We didn't have enough slot to track this LDS DMA store, it
2475 // has been tracked using the common RegNo (FIRST_LDS_VGPR).
2476 ScoreBrackets.determineWaitForLDSDMA(T: AMDGPU::LOAD_CNT, TID,
2477 Wait);
2478 break;
2479 }
2480
2481 ScoreBrackets.determineWaitForLDSDMA(T: AMDGPU::LOAD_CNT,
2482 TID: TID + I + 1, Wait);
2483 }
2484 }
2485 } else {
2486 ScoreBrackets.determineWaitForLDSDMA(T: AMDGPU::LOAD_CNT, TID, Wait);
2487 }
2488 if (Memop->isStore()) {
2489 ScoreBrackets.determineWaitForLDSDMA(T: AMDGPU::EXP_CNT, TID, Wait);
2490 }
2491 }
2492
2493 // Loop over use and def operands.
2494 for (const MachineOperand &Op : MI.operands()) {
2495 if (!Op.isReg())
2496 continue;
2497
2498 // If the instruction does not read tied source, skip the operand.
2499 if (Op.isTied() && Op.isUse() && TII.doesNotReadTiedSource(MI))
2500 continue;
2501
2502 MCPhysReg Reg = Op.getReg().asMCReg();
2503
2504 const bool IsVGPR = TRI.isVectorRegister(MRI, Reg: Op.getReg());
2505 if (IsVGPR) {
2506 // Implicit VGPR defs and uses are never a part of the memory
2507 // instructions description and usually present to account for
2508 // super-register liveness.
2509 // TODO: Most of the other instructions also have implicit uses
2510 // for the liveness accounting only.
2511 if (Op.isImplicit() && MI.mayLoadOrStore())
2512 continue;
2513
2514 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::VA_VDST_WR, Reg, Wait,
2515 MI);
2516 if (Op.isDef()) {
2517 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::VA_VDST_RD, Reg, Wait,
2518 MI);
2519 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::VM_VSRC, Reg, Wait,
2520 MI);
2521 }
2522
2523 // RAW always needs an s_waitcnt. WAW needs an s_waitcnt unless the
2524 // previous write and this write are the same type of VMEM
2525 // instruction, in which case they are (in some architectures)
2526 // guaranteed to write their results in order anyway.
2527 // Additionally check instructions where Point Sample Acceleration
2528 // might be applied.
2529 if (Op.isUse() || !updateVMCntOnly(Inst: MI) ||
2530 ScoreBrackets.hasDifferentVGPRPendingEvents(
2531 Reg, E: AMDGPU::getSimplifiedVMEMEventsFor(Inst: MI, TII)) ||
2532 ScoreBrackets.hasPointSamplePendingVmemTypes(MI, Reg) ||
2533 !ST.hasVmemWriteVgprInOrder()) {
2534 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::LOAD_CNT, Reg, Wait,
2535 MI);
2536 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::SAMPLE_CNT, Reg, Wait,
2537 MI);
2538 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::BVH_CNT, Reg, Wait,
2539 MI);
2540 ScoreBrackets.clearVGPRPendingEvents(Reg);
2541 }
2542
2543 if (Op.isDef() ||
2544 ScoreBrackets.hasPendingEvent(E: HWEvents::EXP_LDS_ACCESS)) {
2545 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::EXP_CNT, Reg, Wait,
2546 MI);
2547 }
2548 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::DS_CNT, Reg, Wait, MI);
2549 } else if (Op.getReg() == AMDGPU::SCC) {
2550 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::KM_CNT, Reg, Wait, MI);
2551 } else {
2552 ScoreBrackets.determineWaitForPhysReg(T: SmemAccessCounter, Reg, Wait,
2553 MI);
2554 }
2555
2556 if (ST.hasWaitXcnt() && Op.isDef())
2557 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::X_CNT, Reg, Wait, MI);
2558 }
2559 }
2560 }
2561 }
2562
2563 // Ensure safety against exceptions from outstanding memory operations while
2564 // waiting for a barrier:
2565 //
2566 // * Some subtargets safely handle backing off the barrier in hardware
2567 // when an exception occurs.
2568 // * Some subtargets have an implicit S_WAITCNT 0 before barriers, so that
2569 // there can be no outstanding memory operations during the wait.
2570 // * Subtargets with split barriers don't need to back off the barrier; it
2571 // is up to the trap handler to preserve the user barrier state correctly.
2572 //
2573 // In all other cases, ensure safety by ensuring that there are no outstanding
2574 // memory operations.
2575 if (Opc == AMDGPU::S_BARRIER && !ST.hasAutoWaitcntBeforeBarrier() &&
2576 !ST.hasBackOffBarrier()) {
2577 Wait = Wait.combined(Other: WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/true));
2578 }
2579
2580 // TODO: Remove this work-around, enable the assert for Bug 457939
2581 // after fixing the scheduler. Also, the Shader Compiler code is
2582 // independent of target.
2583 if (SIInstrInfo::isCBranchVCCZRead(MI) && ST.hasReadVCCZBug() &&
2584 ScoreBrackets.hasPendingEvent(E: HWEvents::SMEM_ACCESS)) {
2585 Wait.set(T: AMDGPU::DS_CNT, Val: 0);
2586 }
2587
2588 // Verify that the wait is actually needed.
2589 ScoreBrackets.simplifyWaitcnt(Wait);
2590
2591 // It is only necessary to insert an S_WAITCNT_DEPCTR instruction that
2592 // waits on VA_VDST if the instruction it would precede is not a VALU
2593 // instruction, since hardware handles VALU->VGPR->VALU hazards in
2594 // expert scheduling mode.
2595 if (TII.isVALU(MI, /*AllowLDSDMA=*/false)) {
2596 Wait.set(T: AMDGPU::VA_VDST_RD, Val: ~0u);
2597 Wait.set(T: AMDGPU::VA_VDST_WR, Val: ~0u);
2598 }
2599
2600 // Since the translation for VMEM addresses occur in-order, we can apply the
2601 // XCnt if the current instruction is of VMEM type and has a memory
2602 // dependency with another VMEM instruction in flight.
2603 if (Wait.get(T: AMDGPU::X_CNT) != ~0u && isVmemAccess(MI)) {
2604 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::X_CNT);
2605 Wait.set(T: AMDGPU::X_CNT, Val: ~0u);
2606 }
2607
2608 // When forcing emit, we need to skip terminators because that would break the
2609 // terminators of the MBB if we emit a waitcnt between terminators.
2610 if (ForceEmitZeroFlag && !MI.isTerminator())
2611 Wait = WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/false);
2612
2613 // If we force waitcnt then update Wait accordingly.
2614 for (AMDGPU::InstCounterType T : AMDGPU::inst_counter_types()) {
2615 if (!ForceEmitWaitcnt[T])
2616 continue;
2617 Wait.set(T, Val: 0);
2618 }
2619
2620 if (FlushFlags.FlushVmCnt) {
2621 for (AMDGPU::InstCounterType T :
2622 {AMDGPU::LOAD_CNT, AMDGPU::SAMPLE_CNT, AMDGPU::BVH_CNT})
2623 Wait.set(T, Val: 0);
2624 }
2625
2626 if (FlushFlags.FlushDsCnt && ScoreBrackets.hasPendingEvent(T: AMDGPU::DS_CNT))
2627 Wait.set(T: AMDGPU::DS_CNT, Val: 0);
2628
2629 if (ForceEmitZeroLoadFlag && Wait.get(T: AMDGPU::LOAD_CNT) != ~0u)
2630 Wait.set(T: AMDGPU::LOAD_CNT, Val: 0);
2631
2632 return generateWaitcnt(Wait, It: MI.getIterator(), Block&: *MI.getParent(), ScoreBrackets,
2633 OldWaitcntInstr);
2634}
2635
2636bool SIInsertWaitcnts::generateWaitcnt(AMDGPU::Waitcnt Wait,
2637 MachineBasicBlock::instr_iterator It,
2638 MachineBasicBlock &Block,
2639 WaitcntBrackets &ScoreBrackets,
2640 MachineInstr *OldWaitcntInstr) {
2641 bool Modified = false;
2642
2643 if (OldWaitcntInstr)
2644 // Try to merge the required wait with preexisting waitcnt instructions.
2645 // Also erase redundant waitcnt.
2646 Modified =
2647 WCG->applyPreexistingWaitcnt(ScoreBrackets, OldWaitcntInstr&: *OldWaitcntInstr, Wait, It);
2648
2649 // ExpCnt can be merged into VINTERP.
2650 if (Wait.get(T: AMDGPU::EXP_CNT) != ~0u && It != Block.instr_end() &&
2651 SIInstrInfo::isVINTERP(MI: *It)) {
2652 MachineOperand *WaitExp = TII.getNamedOperand(MI&: *It, OperandName: AMDGPU::OpName::waitexp);
2653 if (Wait.get(T: AMDGPU::EXP_CNT) < WaitExp->getImm()) {
2654 WaitExp->setImm(Wait.get(T: AMDGPU::EXP_CNT));
2655 Modified = true;
2656 }
2657 // Apply ExpCnt before resetting it, so applyWaitcnt below sees all counts.
2658 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::EXP_CNT);
2659 Wait.set(T: AMDGPU::EXP_CNT, Val: ~0u);
2660
2661 LLVM_DEBUG(dbgs() << "generateWaitcnt\n"
2662 << "Update Instr: " << *It);
2663 }
2664
2665 if (WCG->createNewWaitcnt(Block, It, Wait, ScoreBrackets))
2666 Modified = true;
2667
2668 // Any counts that could have been applied to any existing waitcnt
2669 // instructions will have been done so, now deal with any remaining.
2670 ScoreBrackets.applyWaitcnt(Wait);
2671
2672 return Modified;
2673}
2674
2675bool SIInsertWaitcnts::isVmemAccess(const MachineInstr &MI) const {
2676 return (TII.isFLAT(MI) && TII.mayAccessVMEMThroughFlat(MI)) ||
2677 (TII.isVMEM(MI) && !AMDGPU::getMUBUFIsBufferInv(Opc: MI.getOpcode()));
2678}
2679
2680// Return true if the next instruction is S_ENDPGM, following fallthrough
2681// blocks if necessary.
2682bool SIInsertWaitcnts::isNextENDPGM(MachineBasicBlock::instr_iterator It,
2683 MachineBasicBlock *Block) const {
2684 auto BlockEnd = Block->getParent()->end();
2685 auto BlockIter = Block->getIterator();
2686
2687 while (true) {
2688 if (It.isEnd()) {
2689 if (++BlockIter != BlockEnd) {
2690 It = BlockIter->instr_begin();
2691 continue;
2692 }
2693
2694 return false;
2695 }
2696
2697 if (!It->isMetaInstruction())
2698 break;
2699
2700 It++;
2701 }
2702
2703 assert(!It.isEnd());
2704
2705 return It->getOpcode() == AMDGPU::S_ENDPGM;
2706}
2707
2708// Add a wait after an instruction if architecture requirements mandate one.
2709bool SIInsertWaitcnts::insertForcedWaitAfter(MachineInstr &Inst,
2710 MachineBasicBlock &Block,
2711 WaitcntBrackets &ScoreBrackets) {
2712 AMDGPU::Waitcnt Wait;
2713 bool NeedsEndPGMCheck = false;
2714
2715 if (ST.isPreciseMemoryEnabled() && Inst.mayLoadOrStore())
2716 Wait = WCG->getAllZeroWaitcnt(IncludeVSCnt: Inst.mayStore() &&
2717 !SIInstrInfo::isAtomicRet(MI: Inst));
2718
2719 if (TII.isAlwaysGDS(Opcode: Inst.getOpcode())) {
2720 Wait.set(T: AMDGPU::DS_CNT, Val: 0);
2721 NeedsEndPGMCheck = true;
2722 }
2723
2724 ScoreBrackets.simplifyWaitcnt(Wait);
2725
2726 auto SuccessorIt = std::next(x: Inst.getIterator());
2727 bool Result = generateWaitcnt(Wait, It: SuccessorIt, Block, ScoreBrackets,
2728 /*OldWaitcntInstr=*/nullptr);
2729
2730 if (Result && NeedsEndPGMCheck && isNextENDPGM(It: SuccessorIt, Block: &Block)) {
2731 BuildMI(BB&: Block, I: SuccessorIt, MIMD: Inst.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_NOP))
2732 .addImm(Val: 0);
2733 }
2734
2735 return Result;
2736}
2737
2738void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
2739 WaitcntBrackets *ScoreBrackets) {
2740
2741 HWEvents InstEvents = AMDGPU::getEventsFor(Inst, ST, IsExpertMode, TgSplit);
2742 for (HWEvents E : InstEvents)
2743 ScoreBrackets->updateByEvent(E, Inst);
2744
2745 if (TII.isDS(MI: Inst) && TII.usesLGKM_CNT(MI: Inst)) {
2746 if (TII.isAlwaysGDS(Opcode: Inst.getOpcode()) ||
2747 TII.hasModifiersSet(MI: Inst, OpName: AMDGPU::OpName::gds)) {
2748 ScoreBrackets->setPendingGDS();
2749 }
2750 } else if (TII.isFLAT(MI: Inst)) {
2751 if (Inst.mayLoadOrStore() && TII.mayAccessVMEMThroughFlat(MI: Inst) &&
2752 TII.mayAccessLDSThroughFlat(MI: Inst, TgSplit) &&
2753 !SIInstrInfo::isLDSDMA(MI: Inst)) {
2754 // Async/LDSDMA operations have FLAT encoding but do not actually use flat
2755 // pointers. They do have two operands that each access global and LDS,
2756 // thus making it appear at this point that they are using a flat pointer.
2757 // Filter them out, and for the rest, generate a dependency on flat
2758 // pointers so that both VM and LGKM counters are flushed.
2759 ScoreBrackets->setPendingFlat();
2760 }
2761 } else if (Inst.isCall()) {
2762 // Act as a wait on everything, but AsyncCnt and TensorCnt are never
2763 // included in such blanket waits.
2764 ScoreBrackets->applyWaitcnt(Wait: WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/false));
2765 ScoreBrackets->setStateOnFunctionEntryOrReturn();
2766 } else if (TII.isVINTERP(MI: Inst)) {
2767 int64_t Imm = TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::waitexp)->getImm();
2768 ScoreBrackets->applyWaitcnt(T: AMDGPU::EXP_CNT, Count: Imm);
2769 }
2770
2771 // Set XCNT to zero in the bracket for instructions that implicitly drain
2772 // XCNT.
2773 if (ST.hasWaitXcnt() && SIInstrInfo::isXcntDrain(MI: Inst))
2774 ScoreBrackets->applyWaitcnt(T: AMDGPU::X_CNT, Count: 0);
2775}
2776
2777bool WaitcntBrackets::mergeScore(const MergeInfo &M, unsigned &Score,
2778 unsigned OtherScore) {
2779 unsigned MyShifted = Score <= M.OldLB ? 0 : Score + M.MyShift;
2780 unsigned OtherShifted =
2781 OtherScore <= M.OtherLB ? 0 : OtherScore + M.OtherShift;
2782 Score = std::max(a: MyShifted, b: OtherShifted);
2783 return OtherShifted > MyShifted;
2784}
2785
2786bool WaitcntBrackets::mergeAsyncMarks(ArrayRef<MergeInfo> MergeInfos,
2787 ArrayRef<CounterValueArray> OtherMarks) {
2788 bool StrictDom = false;
2789
2790 LLVM_DEBUG(dbgs() << "Merging async marks ...");
2791 // Early exit: nothing to merge when both sides are empty.
2792 if (AsyncMarks.empty() && OtherMarks.empty()) {
2793 LLVM_DEBUG(dbgs() << " nothing to merge\n");
2794 return false;
2795 }
2796 LLVM_DEBUG(dbgs() << '\n');
2797
2798 // Determine maximum length needed after merging
2799 auto MaxSize = (unsigned)std::max(a: AsyncMarks.size(), b: OtherMarks.size());
2800 MaxSize = std::min(a: MaxSize, b: MaxAsyncMarks);
2801
2802 // Keep only the most recent marks within our limit.
2803 if (AsyncMarks.size() > MaxSize)
2804 AsyncMarks.erase(CS: AsyncMarks.begin(),
2805 CE: AsyncMarks.begin() + (AsyncMarks.size() - MaxSize));
2806
2807 // Pad with zero-filled marks if our list is shorter. Zero represents "no
2808 // pending async operations at this checkpoint" and acts as the identity
2809 // element for max() during merging. We pad at the beginning since the marks
2810 // need to be aligned in most-recent order.
2811 constexpr CounterValueArray ZeroMark{};
2812 AsyncMarks.insert(I: AsyncMarks.begin(), NumToInsert: MaxSize - AsyncMarks.size(), Elt: ZeroMark);
2813
2814 LLVM_DEBUG({
2815 dbgs() << "Before merge:\n";
2816 for (const auto &Mark : AsyncMarks) {
2817 llvm::interleaveComma(Mark, dbgs());
2818 dbgs() << '\n';
2819 }
2820 dbgs() << "Other marks:\n";
2821 for (const auto &Mark : OtherMarks) {
2822 llvm::interleaveComma(Mark, dbgs());
2823 dbgs() << '\n';
2824 }
2825 });
2826
2827 // Merge element-wise using the existing mergeScore function and the
2828 // appropriate MergeInfo for each counter type. Rebase and merge EVERY
2829 // surviving mark into the new (widened) frame, aligned by recency from the
2830 // back. A mark with no counterpart on the other side merges against the
2831 // zero/identity mark: mergeScore still applies MyShift, so the mark is
2832 // re-expressed in the new frame instead of being left with a stale
2833 // (pre-merge) score.
2834 const unsigned OtherSize = OtherMarks.size();
2835 const unsigned OurSize = AsyncMarks.size();
2836 // After the both-empty early-return above, max(AsyncMarks, OtherMarks) >= 1,
2837 // and the erase/pad steps normalize AsyncMarks to exactly MaxSize. Hence
2838 // OurSize == MaxSize >= 1 (as long as MaxAsyncMarks != 0), so the
2839 // seq_inclusive(1, OurSize) below never trips its "Begin <= End" assertion
2840 // the way seq_inclusive(1, MergeCount) could when OtherSize == 0.
2841 assert(OurSize >= 1 &&
2842 "AsyncMarks padded to MaxSize >= 1 (needs MaxAsyncMarks != 0)");
2843
2844 for (auto Idx : seq_inclusive<unsigned>(Begin: 1, End: OurSize)) {
2845 const CounterValueArray &OtherMark =
2846 Idx <= OtherSize ? OtherMarks[OtherSize - Idx] : ZeroMark;
2847 for (auto T : inst_counter_types(MaxCounter: Context->MaxCounter))
2848 StrictDom |=
2849 mergeScore(M: MergeInfos[T], Score&: AsyncMarks[OurSize - Idx][T], OtherScore: OtherMark[T]);
2850 }
2851
2852 LLVM_DEBUG({
2853 dbgs() << "After merge:\n";
2854 for (const auto &Mark : AsyncMarks) {
2855 llvm::interleaveComma(Mark, dbgs());
2856 dbgs() << '\n';
2857 }
2858 });
2859
2860 return StrictDom;
2861}
2862
2863/// Merge the pending events and associater score brackets of \p Other into
2864/// this brackets status.
2865///
2866/// Returns whether the merge resulted in a change that requires tighter waits
2867/// (i.e. the merged brackets strictly dominate the original brackets).
2868bool WaitcntBrackets::merge(const WaitcntBrackets &Other) {
2869 bool StrictDom = false;
2870
2871 // Check if "other" has keys we don't have, and create default entries for
2872 // those. If they remain empty after merging, we will clean it up after.
2873 for (auto K : Other.VMem.keys())
2874 VMem.try_emplace(Key: K);
2875 for (auto K : Other.SGPRs.keys())
2876 SGPRs.try_emplace(Key: K);
2877
2878 // Array to store MergeInfo for each counter type
2879 MergeInfo MergeInfos[AMDGPU::NUM_INST_CNTS];
2880
2881 for (auto T : inst_counter_types(MaxCounter: Context->MaxCounter)) {
2882 // Merge event flags for this counter
2883 const HWEvents &EventsForT = Context->getWaitEvents(T);
2884 const HWEvents OldEvents = PendingEvents & EventsForT;
2885 const HWEvents OtherEvents = Other.PendingEvents & EventsForT;
2886 if (!OldEvents.contains(Other: OtherEvents))
2887 StrictDom = true;
2888 PendingEvents |= OtherEvents;
2889
2890 // Merge scores for this counter
2891 const unsigned MyPending = ScoreUBs[T] - ScoreLBs[T];
2892 const unsigned OtherPending = Other.ScoreUBs[T] - Other.ScoreLBs[T];
2893 const unsigned NewUB = ScoreLBs[T] + std::max(a: MyPending, b: OtherPending);
2894 if (NewUB < ScoreLBs[T])
2895 report_fatal_error(reason: "waitcnt score overflow");
2896
2897 MergeInfo &M = MergeInfos[T];
2898 M.OldLB = ScoreLBs[T];
2899 M.OtherLB = Other.ScoreLBs[T];
2900 M.MyShift = NewUB - ScoreUBs[T];
2901 M.OtherShift = NewUB - Other.ScoreUBs[T];
2902
2903 ScoreUBs[T] = NewUB;
2904
2905 if (T == AMDGPU::LOAD_CNT)
2906 StrictDom |= mergeScore(M, Score&: LastFlatLoadCnt, OtherScore: Other.LastFlatLoadCnt);
2907
2908 if (T == AMDGPU::DS_CNT) {
2909 StrictDom |= mergeScore(M, Score&: LastFlatDsCnt, OtherScore: Other.LastFlatDsCnt);
2910 StrictDom |= mergeScore(M, Score&: LastGDS, OtherScore: Other.LastGDS);
2911 }
2912
2913 if (T == AMDGPU::KM_CNT) {
2914 StrictDom |= mergeScore(M, Score&: SCCScore, OtherScore: Other.SCCScore);
2915 if (Other.hasPendingEvent(E: HWEvents::SCC_WRITE)) {
2916 if (!(OldEvents & HWEvents::SCC_WRITE)) {
2917 PendingSCCWrite = Other.PendingSCCWrite;
2918 } else if (PendingSCCWrite != Other.PendingSCCWrite) {
2919 PendingSCCWrite = nullptr;
2920 }
2921 }
2922 }
2923
2924 for (auto &[RegID, Info] : VMem)
2925 StrictDom |= mergeScore(M, Score&: Info.Scores[T], OtherScore: Other.getVMemScore(TID: RegID, T));
2926
2927 if (isSmemCounter(T)) {
2928 for (auto &[RegID, Info] : SGPRs) {
2929 auto It = Other.SGPRs.find(Val: RegID);
2930 unsigned OtherScore = (It != Other.SGPRs.end()) ? It->second.get(T) : 0;
2931 StrictDom |= mergeScore(M, Score&: Info.get(T), OtherScore);
2932 }
2933 }
2934 }
2935
2936 for (auto &[TID, Info] : VMem) {
2937 if (auto It = Other.VMem.find(Val: TID); It != Other.VMem.end()) {
2938 HWEvents NewVGPRContext =
2939 Info.VGPRPendingEvents | It->second.VGPRPendingEvents;
2940 StrictDom |= NewVGPRContext != Info.VGPRPendingEvents;
2941 Info.VGPRPendingEvents = NewVGPRContext;
2942 }
2943 }
2944
2945 StrictDom |= mergeAsyncMarks(MergeInfos, OtherMarks: Other.AsyncMarks);
2946 for (auto T : inst_counter_types(MaxCounter: Context->MaxCounter))
2947 StrictDom |= mergeScore(M: MergeInfos[T], Score&: AsyncScore[T], OtherScore: Other.AsyncScore[T]);
2948
2949 purgeEmptyTrackingData();
2950 return StrictDom;
2951}
2952
2953static bool isWaitInstr(MachineInstr &Inst) {
2954 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode: Inst.getOpcode());
2955 return Opcode == AMDGPU::S_WAITCNT ||
2956 (Opcode == AMDGPU::S_WAITCNT_VSCNT && Inst.getOperand(i: 0).isReg() &&
2957 Inst.getOperand(i: 0).getReg() == AMDGPU::SGPR_NULL) ||
2958 Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT ||
2959 Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT ||
2960 Opcode == AMDGPU::S_WAITCNT_lds_direct ||
2961 Opcode == AMDGPU::WAIT_ASYNCMARK ||
2962 AMDGPU::counterTypeForInstr(Opcode).has_value();
2963}
2964
2965void SIInsertWaitcnts::setSchedulingMode(MachineBasicBlock &MBB,
2966 MachineBasicBlock::iterator I,
2967 bool ExpertMode) const {
2968 const unsigned EncodedReg = AMDGPU::Hwreg::HwregEncoding::encode(
2969 Values: AMDGPU::Hwreg::ID_SCHED_MODE, Values: AMDGPU::Hwreg::HwregOffset::Default, Values: 2);
2970 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_SETREG_IMM32_B32))
2971 .addImm(Val: ExpertMode ? 2 : 0)
2972 .addImm(Val: EncodedReg);
2973}
2974
2975namespace {
2976// TODO: Remove this work-around after fixing the scheduler.
2977// There are two reasons why vccz might be incorrect; see ST.hasReadVCCZBug()
2978// and ST.partialVCCWritesUpdateVCCZ().
2979// i. VCCZBug: There is a hardware bug on CI/SI where SMRD instruction may
2980// corrupt vccz bit, so when we detect that an instruction may read from
2981// a corrupt vccz bit, we need to:
2982// 1. Insert s_waitcnt lgkm(0) to wait for all outstanding SMRD
2983// operations to complete.
2984// 2. Recompute the correct value of vccz by writing the current value
2985// of vcc back to vcc.
2986// ii. Partial writes to vcc don't update vccz, so we need to recompute the
2987// correct value of vccz by reading vcc and writing it back to vcc.
2988// No waitcnt is needed in this case.
2989class VCCZWorkaround {
2990 const WaitcntBrackets &ScoreBrackets;
2991 const GCNSubtarget &ST;
2992 const SIInstrInfo &TII;
2993 const SIRegisterInfo &TRI;
2994 bool VCCZCorruptionBug = false;
2995 bool VCCZNotUpdatedByPartialWrites = false;
2996 /// vccz could be incorrect at a basic block boundary if a predecessor wrote
2997 /// to vcc and then issued an smem load, so initialize to true.
2998 bool MustRecomputeVCCZ = true;
2999
3000public:
3001 VCCZWorkaround(const WaitcntBrackets &ScoreBrackets, const GCNSubtarget &ST,
3002 const SIInstrInfo &TII, const SIRegisterInfo &TRI)
3003 : ScoreBrackets(ScoreBrackets), ST(ST), TII(TII), TRI(TRI) {
3004 VCCZCorruptionBug = ST.hasReadVCCZBug();
3005 VCCZNotUpdatedByPartialWrites = !ST.partialVCCWritesUpdateVCCZ();
3006 }
3007 /// If \p MI reads vccz and we must recompute it based on MustRecomputeVCCZ,
3008 /// then emit a vccz recompute instruction before \p MI. This needs to be
3009 /// called on every instruction in the basic block because it also tracks the
3010 /// state and updates MustRecomputeVCCZ accordingly. Returns true if it
3011 /// modified the IR.
3012 bool tryRecomputeVCCZ(MachineInstr &MI) {
3013 // No need to run this if neither bug is present.
3014 if (!VCCZCorruptionBug && !VCCZNotUpdatedByPartialWrites)
3015 return false;
3016
3017 // If MI is an SMEM and it can corrupt vccz on this target, then we need
3018 // both to emit a waitcnt and to recompute vccz.
3019 // But we don't actually emit a waitcnt here. This is done in
3020 // generateWaitcntInstBefore() because it tracks all the necessary waitcnt
3021 // state, and can either skip emitting a waitcnt if there is already one in
3022 // the IR, or emit an "optimized" combined waitcnt.
3023 // If this is an smem read, it could complete and clobber vccz at any time.
3024 MustRecomputeVCCZ |= VCCZCorruptionBug && TII.isSMRD(MI);
3025
3026 // If the target partial vcc writes don't update vccz, and MI is such an
3027 // instruction then we must recompute vccz.
3028 // Note: We are using PartiallyWritesToVCCOpt optional to avoid calling
3029 // `definesRegister()` more than needed, because it's not very cheap.
3030 std::optional<bool> PartiallyWritesToVCCOpt;
3031 auto PartiallyWritesToVCC = [](MachineInstr &MI) {
3032 return MI.definesRegister(Reg: AMDGPU::VCC_LO, /*TRI=*/nullptr) ||
3033 MI.definesRegister(Reg: AMDGPU::VCC_HI, /*TRI=*/nullptr);
3034 };
3035 if (VCCZNotUpdatedByPartialWrites) {
3036 PartiallyWritesToVCCOpt = PartiallyWritesToVCC(MI);
3037 // If this is a partial VCC write but won't update vccz, then we must
3038 // recompute vccz.
3039 MustRecomputeVCCZ |= *PartiallyWritesToVCCOpt;
3040 }
3041
3042 // If MI is a vcc write with no pending smem, or there is a pending smem
3043 // but the target does not suffer from the vccz corruption bug, then we
3044 // don't need to recompute vccz as this write will recompute it anyway.
3045 if (!ScoreBrackets.hasPendingEvent(E: HWEvents::SMEM_ACCESS) ||
3046 !VCCZCorruptionBug) {
3047 // Compute PartiallyWritesToVCCOpt if we haven't done so already.
3048 if (!PartiallyWritesToVCCOpt)
3049 PartiallyWritesToVCCOpt = PartiallyWritesToVCC(MI);
3050 bool FullyWritesToVCC = !*PartiallyWritesToVCCOpt &&
3051 MI.definesRegister(Reg: AMDGPU::VCC, /*TRI=*/nullptr);
3052 // If we write to the full vcc or we write partially and the target
3053 // updates vccz on partial writes, then vccz will be updated correctly.
3054 bool UpdatesVCCZ = FullyWritesToVCC || (!VCCZNotUpdatedByPartialWrites &&
3055 *PartiallyWritesToVCCOpt);
3056 if (UpdatesVCCZ)
3057 MustRecomputeVCCZ = false;
3058 }
3059
3060 // If MI is a branch that reads VCCZ then emit a waitcnt and a vccz
3061 // restore instruction if either is needed.
3062 if (SIInstrInfo::isCBranchVCCZRead(MI) && MustRecomputeVCCZ) {
3063 // Recompute the vccz bit. Any time a value is written to vcc, the vccz
3064 // bit is updated, so we can restore the bit by reading the value of vcc
3065 // and then writing it back to the register.
3066 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
3067 MCID: TII.get(Opcode: ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64),
3068 DestReg: TRI.getVCC())
3069 .addReg(RegNo: TRI.getVCC());
3070 MustRecomputeVCCZ = false;
3071 return true;
3072 }
3073 return false;
3074 }
3075};
3076
3077} // namespace
3078
3079// Generate s_waitcnt instructions where needed.
3080bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
3081 MachineBasicBlock &Block,
3082 WaitcntBrackets &ScoreBrackets) {
3083 bool Modified = false;
3084
3085 LLVM_DEBUG({
3086 dbgs() << "*** Begin Block: ";
3087 Block.printName(dbgs());
3088 ScoreBrackets.dump();
3089 });
3090 VCCZWorkaround VCCZW(ScoreBrackets, ST, TII, TRI);
3091
3092 // Walk over the instructions.
3093 MachineInstr *OldWaitcntInstr = nullptr;
3094
3095 // NOTE: We may append instrs after Inst while iterating.
3096 ScoreBrackets.verify();
3097 for (MachineBasicBlock::instr_iterator Iter = Block.instr_begin(),
3098 E = Block.instr_end();
3099 Iter != E; ++Iter) {
3100 MachineInstr &Inst = *Iter;
3101 if (isNonWaitcntMetaInst(MI: Inst))
3102 continue;
3103 // Track pre-existing waitcnts that were added in earlier iterations or by
3104 // the memory legalizer.
3105 if (isWaitInstr(Inst) ||
3106 (IsExpertMode && Inst.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR)) {
3107 if (!OldWaitcntInstr)
3108 OldWaitcntInstr = &Inst;
3109 continue;
3110 }
3111
3112 PreheaderFlushFlags FlushFlags;
3113 if (Block.getFirstTerminator() == Inst)
3114 FlushFlags = isPreheaderToFlush(MBB&: Block, ScoreBrackets);
3115
3116 // Generate an s_waitcnt instruction to be placed before Inst, if needed.
3117 Modified |= generateWaitcntInstBefore(MI&: Inst, ScoreBrackets, OldWaitcntInstr,
3118 FlushFlags);
3119 OldWaitcntInstr = nullptr;
3120
3121 if (Inst.getOpcode() == AMDGPU::ASYNCMARK) {
3122 // Asyncmarks record the current wait state and so should not allow
3123 // waitcnts that occur after them to be merged into waitcnts that occur
3124 // before.
3125 ScoreBrackets.recordAsyncMark(Inst);
3126 continue;
3127 }
3128
3129 if (TII.isSMRD(MI: Inst)) {
3130 for (const MachineMemOperand *Memop : Inst.memoperands()) {
3131 // No need to handle invariant loads when avoiding WAR conflicts, as
3132 // there cannot be a vector store to the same memory location.
3133 if (!Memop->isInvariant()) {
3134 const Value *Ptr = Memop->getValue();
3135 SLoadAddresses.insert(KV: std::pair(Ptr, Inst.getParent()));
3136 }
3137 }
3138 }
3139
3140 updateEventWaitcntAfter(Inst, ScoreBrackets: &ScoreBrackets);
3141
3142 // Note: insertForcedWaitAfter() may add instrs after Iter that need to be
3143 // visited by the loop.
3144 Modified |= insertForcedWaitAfter(Inst, Block, ScoreBrackets);
3145
3146 LLVM_DEBUG({
3147 Inst.print(dbgs());
3148 ScoreBrackets.dump();
3149 });
3150
3151 // If the target suffers from the vccz bugs, this may emit the necessary
3152 // vccz recompute instruction before \p Inst if needed.
3153 Modified |= VCCZW.tryRecomputeVCCZ(MI&: Inst);
3154
3155 ScoreBrackets.verify();
3156 }
3157
3158 // Flush counters at the end of the block if needed (for preheaders with no
3159 // terminator).
3160 AMDGPU::Waitcnt Wait;
3161 if (Block.getFirstTerminator() == Block.end()) {
3162 PreheaderFlushFlags FlushFlags = isPreheaderToFlush(MBB&: Block, ScoreBrackets);
3163 if (FlushFlags.FlushVmCnt) {
3164 if (ScoreBrackets.hasPendingEvent(T: AMDGPU::LOAD_CNT))
3165 Wait.set(T: AMDGPU::LOAD_CNT, Val: 0);
3166 if (ScoreBrackets.hasPendingEvent(T: AMDGPU::SAMPLE_CNT))
3167 Wait.set(T: AMDGPU::SAMPLE_CNT, Val: 0);
3168 if (ScoreBrackets.hasPendingEvent(T: AMDGPU::BVH_CNT))
3169 Wait.set(T: AMDGPU::BVH_CNT, Val: 0);
3170 }
3171 if (FlushFlags.FlushDsCnt && ScoreBrackets.hasPendingEvent(T: AMDGPU::DS_CNT))
3172 Wait.set(T: AMDGPU::DS_CNT, Val: 0);
3173 }
3174
3175 // Combine or remove any redundant waitcnts at the end of the block.
3176 Modified |= generateWaitcnt(Wait, It: Block.instr_end(), Block, ScoreBrackets,
3177 OldWaitcntInstr);
3178
3179 LLVM_DEBUG({
3180 dbgs() << "*** End Block: ";
3181 Block.printName(dbgs());
3182 ScoreBrackets.dump();
3183 });
3184
3185 return Modified;
3186}
3187
3188bool SIInsertWaitcnts::removeRedundantSoftXcnts(MachineBasicBlock &Block) {
3189 if (Block.size() <= 1)
3190 return false;
3191 // The Memory Legalizer conservatively inserts a soft xcnt before each
3192 // atomic RMW operation. However, for sequences of back-to-back atomic
3193 // RMWs, only the first s_wait_xcnt insertion is necessary. Optimize away
3194 // the redundant soft xcnts.
3195 bool Modified = false;
3196 // Remember the last atomic with a soft xcnt right before it.
3197 MachineInstr *LastAtomicWithSoftXcnt = nullptr;
3198
3199 for (MachineInstr &MI : drop_begin(RangeOrContainer&: Block)) {
3200 // Ignore last atomic if non-LDS VMEM and SMEM.
3201 bool IsLDS = TII.isDS(MI) ||
3202 (TII.isFLAT(MI) && TII.mayAccessLDSThroughFlat(MI, TgSplit));
3203 if (!IsLDS && (MI.mayLoad() ^ MI.mayStore()))
3204 LastAtomicWithSoftXcnt = nullptr;
3205
3206 bool IsAtomicRMW =
3207 SIInstrFlags::isMaybeAtomic(O: MI) && MI.mayLoad() && MI.mayStore();
3208 MachineInstr &PrevMI = *MI.getPrevNode();
3209 // This is an atomic with a soft xcnt.
3210 if (PrevMI.getOpcode() == AMDGPU::S_WAIT_XCNT_soft && IsAtomicRMW) {
3211 // If we have already found an atomic with a soft xcnt, remove this soft
3212 // xcnt as it's redundant.
3213 if (LastAtomicWithSoftXcnt) {
3214 PrevMI.eraseFromParent();
3215 Modified = true;
3216 }
3217 LastAtomicWithSoftXcnt = &MI;
3218 }
3219 }
3220 return Modified;
3221}
3222
3223// Return flags indicating which counters should be flushed in the preheader.
3224PreheaderFlushFlags
3225SIInsertWaitcnts::isPreheaderToFlush(MachineBasicBlock &MBB,
3226 const WaitcntBrackets &ScoreBrackets) {
3227 auto [Iterator, IsInserted] =
3228 PreheadersToFlush.try_emplace(Key: &MBB, Args: PreheaderFlushFlags());
3229 if (!IsInserted)
3230 return Iterator->second;
3231
3232 MachineBasicBlock *Succ = MBB.getSingleSuccessor();
3233 if (!Succ)
3234 return PreheaderFlushFlags();
3235
3236 MachineLoop *Loop = MLI.getLoopFor(BB: Succ);
3237 if (!Loop)
3238 return PreheaderFlushFlags();
3239
3240 if (Loop->getLoopPreheader() == &MBB) {
3241 Iterator->second = getPreheaderFlushFlags(ML: Loop, Brackets: ScoreBrackets);
3242 return Iterator->second;
3243 }
3244
3245 return PreheaderFlushFlags();
3246}
3247
3248bool SIInsertWaitcnts::isVMEMOrFlatVMEM(const MachineInstr &MI) const {
3249 if (SIInstrInfo::isFLAT(MI))
3250 return TII.mayAccessVMEMThroughFlat(MI);
3251 return SIInstrInfo::isVMEM(MI);
3252}
3253
3254bool SIInsertWaitcnts::isDSRead(const MachineInstr &MI) const {
3255 return SIInstrInfo::isDS(MI) && MI.mayLoad() && !MI.mayStore();
3256}
3257
3258// Check if instruction is a store to LDS that is counted via DSCNT
3259// (where that counter exists).
3260bool SIInsertWaitcnts::mayStoreIncrementingDSCNT(const MachineInstr &MI) const {
3261 return MI.mayStore() && SIInstrInfo::isDS(MI);
3262}
3263
3264// Return flags indicating which counters should be flushed in the preheader of
3265// the given loop. We currently decide to flush in the following situations:
3266// For VMEM (FlushVmCnt):
3267// 1. The loop contains vmem store(s), no vmem load and at least one use of a
3268// vgpr containing a value that is loaded outside of the loop. (Only on
3269// targets with no vscnt counter).
3270// 2. The loop contains vmem load(s), but the loaded values are not used in the
3271// loop, and at least one use of a vgpr containing a value that is loaded
3272// outside of the loop.
3273// For DS (FlushDsCnt, GFX12+ only):
3274// 3. The loop contains no DS reads, and at least one use of a vgpr containing
3275// a value that is DS read outside of the loop.
3276// 4. The loop contains DS read(s), loaded values are not used in the same
3277// iteration but in the next iteration (prefetch pattern), and at least one
3278// use of a vgpr containing a value that is DS read outside of the loop.
3279// Flushing in preheader reduces wait overhead if the wait requirement in
3280// iteration 1 would otherwise be more strict (but unfortunately preheader
3281// flush decision is taken before knowing that).
3282// 5. (Single-block loops only) The loop has DS prefetch reads with flush point
3283// tracking. Some DS reads may be used in the same iteration (creating
3284// "flush points"), but others remain unflushed at the backedge. When a DS
3285// read is consumed in the same iteration, it and all prior reads are
3286// "flushed" (FIFO order). No DS writes are allowed in the loop.
3287// TODO: Find a way to extend to multi-block loops.
3288PreheaderFlushFlags
3289SIInsertWaitcnts::getPreheaderFlushFlags(MachineLoop *ML,
3290 const WaitcntBrackets &Brackets) {
3291 PreheaderFlushFlags Flags;
3292 bool HasVMemLoad = false;
3293 bool HasVMemStore = false;
3294 bool UsesVgprVMEMLoadedOutside = false;
3295 bool UsesVgprDSReadOutside = false;
3296 bool VMemInvalidated = false;
3297 // DS optimization only applies to GFX12+ where DS_CNT is separate.
3298 // Tracking status for "no DS read in loop" or "pure DS prefetch
3299 // (use only in next iteration)".
3300 bool TrackSimpleDSOpt = ST.hasExtendedWaitCounts();
3301 DenseSet<MCRegUnit> VgprUse;
3302 DenseSet<MCRegUnit> VgprDefVMEM;
3303 DenseSet<MCRegUnit> VgprDefDS;
3304
3305 // Track DS reads for prefetch pattern with flush points (single-block only).
3306 // Keeps track of the last DS read (position counted from the top of the loop)
3307 // to each VGPR. Read is considered consumed (and thus needs flushing) if
3308 // the dest register has a use or is overwritten (by any later opertions).
3309 DenseMap<MCRegUnit, unsigned> LastDSReadPositionMap;
3310 unsigned DSReadPosition = 0;
3311 bool IsSingleBlock = ML->getNumBlocks() == 1;
3312 bool TrackDSFlushPoint = ST.hasExtendedWaitCounts() && IsSingleBlock;
3313 unsigned LastDSFlushPosition = 0;
3314
3315 for (MachineBasicBlock *MBB : ML->blocks()) {
3316 for (MachineInstr &MI : *MBB) {
3317 if (isVMEMOrFlatVMEM(MI)) {
3318 HasVMemLoad |= MI.mayLoad();
3319 HasVMemStore |= MI.mayStore();
3320 }
3321 // TODO: Can we relax DSStore check? There may be cases where
3322 // these DS stores are drained prior to the end of MBB (or loop).
3323 if (mayStoreIncrementingDSCNT(MI)) {
3324 // Early exit if none of the optimizations are feasible.
3325 // Otherwise, set tracking status appropriately and continue.
3326 if (VMemInvalidated)
3327 return Flags;
3328 TrackSimpleDSOpt = false;
3329 TrackDSFlushPoint = false;
3330 }
3331 bool IsDSRead = isDSRead(MI);
3332 if (IsDSRead)
3333 ++DSReadPosition;
3334
3335 // Helper: if RU has a pending DS read, update LastDSFlushPosition
3336 auto updateDSReadFlushTracking = [&](MCRegUnit RU) {
3337 if (!TrackDSFlushPoint)
3338 return;
3339 if (auto It = LastDSReadPositionMap.find(Val: RU);
3340 It != LastDSReadPositionMap.end()) {
3341 // RU defined by DSRead is used or overwritten. Need to complete
3342 // the read, if not already implied by a later DSRead (to any RU)
3343 // needing to complete in FIFO order.
3344 LastDSFlushPosition = std::max(a: LastDSFlushPosition, b: It->second);
3345 }
3346 };
3347
3348 for (const MachineOperand &Op : MI.all_uses()) {
3349 if (Op.isDebug() || !TRI.isVectorRegister(MRI, Reg: Op.getReg()))
3350 continue;
3351 // Vgpr use
3352 for (MCRegUnit RU : TRI.regunits(Reg: Op.getReg().asMCReg())) {
3353 // If we find a register that is loaded inside the loop, 1. and 2.
3354 // are invalidated.
3355 if (VgprDefVMEM.contains(V: RU))
3356 VMemInvalidated = true;
3357
3358 // Check for DS reads used inside the loop
3359 if (VgprDefDS.contains(V: RU))
3360 TrackSimpleDSOpt = false;
3361
3362 // Early exit if all optimizations are invalidated
3363 if (VMemInvalidated && !TrackSimpleDSOpt && !TrackDSFlushPoint)
3364 return Flags;
3365
3366 // Check for flush points (DS read used in same iteration)
3367 updateDSReadFlushTracking(RU);
3368
3369 VgprUse.insert(V: RU);
3370 // Check if this register has a pending VMEM load from outside the
3371 // loop (value loaded outside and used inside).
3372 VMEMID ID = toVMEMID(RU);
3373 if (Brackets.hasPendingVMEM(ID, T: AMDGPU::LOAD_CNT) ||
3374 Brackets.hasPendingVMEM(ID, T: AMDGPU::SAMPLE_CNT) ||
3375 Brackets.hasPendingVMEM(ID, T: AMDGPU::BVH_CNT))
3376 UsesVgprVMEMLoadedOutside = true;
3377 // Check if loaded outside the loop via DS (not VMEM/FLAT).
3378 // Only consider it a DS read if there's no pending VMEM load for
3379 // this register, since FLAT can set both counters.
3380 else if (Brackets.hasPendingVMEM(ID, T: AMDGPU::DS_CNT))
3381 UsesVgprDSReadOutside = true;
3382 }
3383 }
3384
3385 // VMem load vgpr def
3386 if (isVMEMOrFlatVMEM(MI) && MI.mayLoad()) {
3387 for (const MachineOperand &Op : MI.all_defs()) {
3388 for (MCRegUnit RU : TRI.regunits(Reg: Op.getReg().asMCReg())) {
3389 // If we find a register that is loaded inside the loop, 1. and 2.
3390 // are invalidated.
3391 if (VgprUse.contains(V: RU))
3392 VMemInvalidated = true;
3393 VgprDefVMEM.insert(V: RU);
3394 }
3395 }
3396 // Early exit if all optimizations are invalidated
3397 if (VMemInvalidated && !TrackSimpleDSOpt && !TrackDSFlushPoint)
3398 return Flags;
3399 }
3400
3401 // DS read vgpr def
3402 // Note: Unlike VMEM, we DON'T invalidate when VgprUse.contains(RegNo).
3403 // If USE comes before DEF, it's the prefetch pattern (use value from
3404 // previous iteration, read for next iteration). We should still flush
3405 // in preheader so iteration 1 doesn't need to wait inside the loop.
3406 // Only invalidate when DEF comes before USE (same-iteration consumption,
3407 // checked above when processing uses).
3408 if (IsDSRead || TrackDSFlushPoint) {
3409 for (const MachineOperand &Op : MI.all_defs()) {
3410 if (!TRI.isVectorRegister(MRI, Reg: Op.getReg()))
3411 continue;
3412 for (MCRegUnit RU : TRI.regunits(Reg: Op.getReg().asMCReg())) {
3413 // Check for overwrite of pending DS read (flush point) by any
3414 // instruction
3415 updateDSReadFlushTracking(RU);
3416 if (IsDSRead) {
3417 VgprDefDS.insert(V: RU);
3418 if (TrackDSFlushPoint)
3419 LastDSReadPositionMap[RU] = DSReadPosition;
3420 }
3421 }
3422 }
3423 }
3424 }
3425 }
3426
3427 // VMEM flush decision
3428 if (!VMemInvalidated && UsesVgprVMEMLoadedOutside &&
3429 ((!ST.hasVscnt() && HasVMemStore && !HasVMemLoad) ||
3430 (HasVMemLoad && ST.hasVmemWriteVgprInOrder())))
3431 Flags.FlushVmCnt = true;
3432
3433 // DS flush decision:
3434 // Simple DS Opt: flush if loop uses DS read values from outside
3435 // and either has no DS reads in the loop, or DS reads whose results
3436 // are not used in the loop.
3437 bool SimpleDSOpt = TrackSimpleDSOpt && UsesVgprDSReadOutside;
3438 // Prefetch with flush points: some DS reads used in same iteration,
3439 // but unflushed reads remain at backedge
3440 bool HasUnflushedDSReads = DSReadPosition > LastDSFlushPosition;
3441 bool DSFlushPointPrefetch =
3442 TrackDSFlushPoint && UsesVgprDSReadOutside && HasUnflushedDSReads;
3443
3444 if (SimpleDSOpt || DSFlushPointPrefetch)
3445 Flags.FlushDsCnt = true;
3446
3447 return Flags;
3448}
3449
3450bool SIInsertWaitcntsLegacy::runOnMachineFunction(MachineFunction &MF) {
3451 auto &MLI = getAnalysis<MachineLoopInfoWrapperPass>().getLI();
3452 auto &PDT =
3453 getAnalysis<MachinePostDominatorTreeWrapperPass>().getPostDomTree();
3454 AliasAnalysis *AA = nullptr;
3455 if (auto *AAR = getAnalysisIfAvailable<AAResultsWrapperPass>())
3456 AA = &AAR->getAAResults();
3457
3458 return SIInsertWaitcnts(MLI, PDT, AA, MF).run();
3459}
3460
3461PreservedAnalyses
3462SIInsertWaitcntsPass::run(MachineFunction &MF,
3463 MachineFunctionAnalysisManager &MFAM) {
3464 auto &MLI = MFAM.getResult<MachineLoopAnalysis>(IR&: MF);
3465 auto &PDT = MFAM.getResult<MachinePostDominatorTreeAnalysis>(IR&: MF);
3466 auto *AA = MFAM.getResult<FunctionAnalysisManagerMachineFunctionProxy>(IR&: MF)
3467 .getManager()
3468 .getCachedResult<AAManager>(IR&: MF.getFunction());
3469
3470 if (!SIInsertWaitcnts(MLI, PDT, AA, MF).run())
3471 return PreservedAnalyses::all();
3472
3473 return getMachineFunctionPassPreservedAnalyses()
3474 .preserveSet<CFGAnalyses>()
3475 .preserve<AAManager>();
3476}
3477
3478bool SIInsertWaitcnts::run() {
3479 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
3480
3481 AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(GPU: ST.getCPU());
3482
3483 // Initialize hardware limits first, as they're needed by the generators.
3484 Limits = AMDGPU::HardwareLimits(IV);
3485
3486 if (ST.hasExtendedWaitCounts()) {
3487 IsExpertMode = ST.hasExpertSchedulingMode() &&
3488 (ExpertSchedulingModeFlag.getNumOccurrences()
3489 ? ExpertSchedulingModeFlag
3490 : MF.getFunction()
3491 .getFnAttribute(Kind: "amdgpu-expert-scheduling-mode")
3492 .getValueAsBool());
3493 MaxCounter = IsExpertMode ? AMDGPU::NUM_EXPERT_INST_CNTS
3494 : AMDGPU::NUM_EXTENDED_INST_CNTS;
3495 // Initialize WCG per MF. It contains state that depends on MF attributes.
3496 WCG = std::make_unique<WaitcntGeneratorGFX12Plus>(args&: MF, args&: MaxCounter, args&: Limits,
3497 args&: IsExpertMode);
3498 } else {
3499 MaxCounter = AMDGPU::NUM_NORMAL_INST_CNTS;
3500 // Initialize WCG per MF. It contains state that depends on MF attributes.
3501 WCG = std::make_unique<WaitcntGeneratorPreGFX12>(
3502 args&: MF, args: AMDGPU::NUM_NORMAL_INST_CNTS, args&: Limits);
3503 }
3504
3505 SmemAccessCounter = getCounterFromEvent(E: HWEvents::SMEM_ACCESS);
3506
3507 bool Modified = false;
3508
3509 MachineBasicBlock &EntryBB = MF.front();
3510
3511 if (!MFI->isEntryFunction() &&
3512 !MF.getFunction().hasFnAttribute(Kind: Attribute::Naked)) {
3513 // Wait for any outstanding memory operations that the input registers may
3514 // depend on. We can't track them and it's better to do the wait after the
3515 // costly call sequence.
3516
3517 // TODO: Could insert earlier and schedule more liberally with operations
3518 // that only use caller preserved registers.
3519 MachineBasicBlock::iterator I = EntryBB.begin();
3520 while (I != EntryBB.end() && I->isMetaInstruction())
3521 ++I;
3522
3523 if (ST.hasExtendedWaitCounts()) {
3524 BuildMI(BB&: EntryBB, I, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_WAIT_LOADCNT_DSCNT))
3525 .addImm(Val: 0);
3526 for (auto CT : inst_counter_types(MaxCounter: AMDGPU::NUM_EXTENDED_INST_CNTS)) {
3527 if (CT == AMDGPU::LOAD_CNT || CT == AMDGPU::DS_CNT ||
3528 CT == AMDGPU::STORE_CNT || CT == AMDGPU::X_CNT ||
3529 CT == AMDGPU::ASYNC_CNT || CT == AMDGPU::TENSOR_CNT)
3530 continue;
3531
3532 if (!ST.hasImageInsts() &&
3533 (CT == AMDGPU::EXP_CNT || CT == AMDGPU::SAMPLE_CNT ||
3534 CT == AMDGPU::BVH_CNT))
3535 continue;
3536
3537 BuildMI(BB&: EntryBB, I, MIMD: DebugLoc(),
3538 MCID: TII.get(Opcode: instrsForExtendedCounterTypes[CT]))
3539 .addImm(Val: 0);
3540 }
3541 if (IsExpertMode) {
3542 unsigned Enc = AMDGPU::DepCtr::encodeFieldVaVdst(VaVdst: 0, STI: ST);
3543 Enc = AMDGPU::DepCtr::encodeFieldVmVsrc(Encoded: Enc, VmVsrc: 0);
3544 BuildMI(BB&: EntryBB, I, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
3545 .addImm(Val: Enc);
3546 }
3547 } else {
3548 BuildMI(BB&: EntryBB, I, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_WAITCNT)).addImm(Val: 0);
3549 }
3550
3551 auto NonKernelInitialState = std::make_unique<WaitcntBrackets>(args: this);
3552 NonKernelInitialState->setStateOnFunctionEntryOrReturn();
3553 BlockInfos[&EntryBB].Incoming = std::move(NonKernelInitialState);
3554
3555 Modified = true;
3556 }
3557
3558 // Keep iterating over the blocks in reverse post order, inserting and
3559 // updating s_waitcnt where needed, until a fix point is reached.
3560 for (auto *MBB : ReversePostOrderTraversal<MachineFunction *>(&MF))
3561 BlockInfos.try_emplace(Key: MBB);
3562
3563 std::unique_ptr<WaitcntBrackets> Brackets;
3564 bool Repeat;
3565 do {
3566 Repeat = false;
3567
3568 for (auto BII = BlockInfos.begin(), BIE = BlockInfos.end(); BII != BIE;
3569 ++BII) {
3570 MachineBasicBlock *MBB = BII->first;
3571 BlockInfo &BI = BII->second;
3572 if (!BI.Dirty)
3573 continue;
3574
3575 if (BI.Incoming) {
3576 if (!Brackets)
3577 Brackets = std::make_unique<WaitcntBrackets>(args&: *BI.Incoming);
3578 else
3579 *Brackets = *BI.Incoming;
3580 } else {
3581 if (!Brackets) {
3582 Brackets = std::make_unique<WaitcntBrackets>(args: this);
3583 } else {
3584 // Reinitialize in-place. N.B. do not do this by assigning from a
3585 // temporary because the WaitcntBrackets class is large and it could
3586 // cause this function to use an unreasonable amount of stack space.
3587 Brackets->~WaitcntBrackets();
3588 new (Brackets.get()) WaitcntBrackets(this);
3589 }
3590 }
3591
3592 if (ST.hasWaitXcnt())
3593 Modified |= removeRedundantSoftXcnts(Block&: *MBB);
3594 Modified |= insertWaitcntInBlock(MF, Block&: *MBB, ScoreBrackets&: *Brackets);
3595 BI.Dirty = false;
3596
3597 if (Brackets->hasPendingEvent()) {
3598 BlockInfo *MoveBracketsToSucc = nullptr;
3599 for (MachineBasicBlock *Succ : MBB->successors()) {
3600 auto *SuccBII = BlockInfos.find(Key: Succ);
3601 BlockInfo &SuccBI = SuccBII->second;
3602 if (!SuccBI.Incoming) {
3603 SuccBI.Dirty = true;
3604 if (SuccBII <= BII) {
3605 LLVM_DEBUG(dbgs() << "Repeat on backedge without merge\n");
3606 Repeat = true;
3607 }
3608 if (!MoveBracketsToSucc) {
3609 MoveBracketsToSucc = &SuccBI;
3610 } else {
3611 SuccBI.Incoming = std::make_unique<WaitcntBrackets>(args&: *Brackets);
3612 }
3613 } else {
3614 LLVM_DEBUG({
3615 dbgs() << "Try to merge ";
3616 MBB->printName(dbgs());
3617 dbgs() << " into ";
3618 Succ->printName(dbgs());
3619 dbgs() << '\n';
3620 });
3621 if (SuccBI.Incoming->merge(Other: *Brackets)) {
3622 SuccBI.Dirty = true;
3623 if (SuccBII <= BII) {
3624 LLVM_DEBUG(dbgs() << "Repeat on backedge with merge\n");
3625 Repeat = true;
3626 }
3627 }
3628 }
3629 }
3630 if (MoveBracketsToSucc)
3631 MoveBracketsToSucc->Incoming = std::move(Brackets);
3632 }
3633 }
3634 } while (Repeat);
3635
3636 Modified |= WCG->eraseDeferredWaitcnts();
3637
3638 if (ST.hasScalarStores()) {
3639 SmallVector<MachineBasicBlock *, 4> EndPgmBlocks;
3640 bool HaveScalarStores = false;
3641
3642 for (MachineBasicBlock &MBB : MF) {
3643 for (MachineInstr &MI : MBB) {
3644 if (!HaveScalarStores && TII.isScalarStore(MI))
3645 HaveScalarStores = true;
3646
3647 if (MI.getOpcode() == AMDGPU::S_ENDPGM ||
3648 MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG)
3649 EndPgmBlocks.push_back(Elt: &MBB);
3650 }
3651 }
3652
3653 if (HaveScalarStores) {
3654 // If scalar writes are used, the cache must be flushed or else the next
3655 // wave to reuse the same scratch memory can be clobbered.
3656 //
3657 // Insert s_dcache_wb at wave termination points if there were any scalar
3658 // stores, and only if the cache hasn't already been flushed. This could
3659 // be improved by looking across blocks for flushes in postdominating
3660 // blocks from the stores but an explicitly requested flush is probably
3661 // very rare.
3662 for (MachineBasicBlock *MBB : EndPgmBlocks) {
3663 bool SeenDCacheWB = false;
3664
3665 for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
3666 I != E; ++I) {
3667 if (I->getOpcode() == AMDGPU::S_DCACHE_WB)
3668 SeenDCacheWB = true;
3669 else if (TII.isScalarStore(MI: *I))
3670 SeenDCacheWB = false;
3671
3672 // FIXME: It would be better to insert this before a waitcnt if any.
3673 if ((I->getOpcode() == AMDGPU::S_ENDPGM ||
3674 I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) &&
3675 !SeenDCacheWB) {
3676 Modified = true;
3677 BuildMI(BB&: *MBB, I, MIMD: I->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_DCACHE_WB));
3678 }
3679 }
3680 }
3681 }
3682 }
3683
3684 if (IsExpertMode) {
3685 // Enable expert scheduling on function entry. To satisfy ABI requirements
3686 // and to allow calls between function with different expert scheduling
3687 // settings, disable it around calls and before returns.
3688
3689 MachineBasicBlock::iterator I = EntryBB.begin();
3690 while (I != EntryBB.end() && I->isMetaInstruction())
3691 ++I;
3692 setSchedulingMode(MBB&: EntryBB, I, ExpertMode: true);
3693
3694 for (MachineInstr *MI : CallInsts) {
3695 MachineBasicBlock &MBB = *MI->getParent();
3696 setSchedulingMode(MBB, I: MI, ExpertMode: false);
3697 setSchedulingMode(MBB, I: std::next(x: MI->getIterator()), ExpertMode: true);
3698 }
3699
3700 for (MachineInstr *MI : ReturnInsts)
3701 setSchedulingMode(MBB&: *MI->getParent(), I: MI, ExpertMode: false);
3702
3703 Modified = true;
3704 }
3705
3706 // Deallocate the VGPRs before previously identified S_ENDPGM instructions.
3707 // This is done in different ways depending on how the VGPRs were allocated
3708 // (i.e. whether we're in dynamic VGPR mode or not).
3709 // Skip deallocation if kernel is waveslot limited vs VGPR limited. A short
3710 // waveslot limited kernel runs slower with the deallocation.
3711 if (!WCG->isOptNone() && MFI->isDynamicVGPREnabled()) {
3712 for (auto [MI, _] : EndPgmInsts) {
3713 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
3714 MCID: TII.get(Opcode: AMDGPU::S_ALLOC_VGPR))
3715 .addImm(Val: 0);
3716 Modified = true;
3717 }
3718 } else if (!WCG->isOptNone() &&
3719 ST.getGeneration() >= AMDGPUSubtarget::GFX11 &&
3720 (MF.getFrameInfo().hasCalls() ||
3721 ST.getOccupancyWithNumVGPRs(
3722 VGPRs: TRI.getNumUsedPhysRegs(MRI, RC: AMDGPU::VGPR_32RegClass),
3723 /*IsDynamicVGPR=*/DynamicVGPRBlockSize: false) < ST.getMaxWavesPerEU())) {
3724 for (auto [MI, Flag] : EndPgmInsts) {
3725 if (Flag) {
3726 if (ST.requiresNopBeforeDeallocVGPRs()) {
3727 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
3728 MCID: TII.get(Opcode: AMDGPU::S_NOP))
3729 .addImm(Val: 0);
3730 }
3731 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
3732 MCID: TII.get(Opcode: AMDGPU::S_SENDMSG))
3733 .addImm(Val: AMDGPU::SendMsg::ID_DEALLOC_VGPRS_GFX11Plus);
3734 Modified = true;
3735 }
3736 }
3737 }
3738
3739 if (MFI->isEntryFunction()) {
3740 MachineBasicBlock::iterator InsertPt = EntryBB.begin();
3741
3742 if (ST.hasWaitXcnt()) {
3743 // Set REPLAY_MODE (bit 25) in MODE register to enable multi-group XNACK
3744 // replay. This aligns hardware behavior with the compiler's s_wait_xcnt
3745 // insertion logic, which assumes multi-group mode by default.
3746 unsigned RegEncoding =
3747 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 25, Values: 1);
3748 BuildMI(BB&: EntryBB, I: InsertPt, MIMD: DebugLoc(),
3749 MCID: TII.get(Opcode: AMDGPU::S_SETREG_IMM32_B32))
3750 .addImm(Val: 1)
3751 .addImm(Val: RegEncoding);
3752 Modified = true;
3753 }
3754
3755 if (ST.hasRequiresInitialUnclausedVmem()) {
3756 // Hardware entrypoints must begin with a specific sequence:
3757 // S_MOV_B64 S[64:65], 0
3758 // V_NOP
3759 // GLOBAL_PREFETCH_B8 V0, S[64:65] SCOPE:SCOPE_SE TH:TH_LOAD_RT
3760 BuildMI(BB&: EntryBB, I: InsertPt, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B64),
3761 DestReg: AMDGPU::SGPR64_SGPR65)
3762 .addImm(Val: 0);
3763 BuildMI(BB&: EntryBB, I: InsertPt, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_NOP_e32));
3764 BuildMI(BB&: EntryBB, I: InsertPt, MIMD: DebugLoc(),
3765 MCID: TII.get(Opcode: AMDGPU::GLOBAL_PREFETCH_B8_SADDR))
3766 .addReg(RegNo: AMDGPU::SGPR64_SGPR65)
3767 .addReg(RegNo: AMDGPU::VGPR0, Flags: RegState::Undef)
3768 .addImm(Val: 0)
3769 .addImm(Val: AMDGPU::CPol::SCOPE_SE | AMDGPU::CPol::TH_RT);
3770 Modified = true;
3771 }
3772 }
3773
3774 return Modified;
3775}
3776